@uniweb/build 1.10.0 → 1.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/prerender.js +23 -18
- package/src/site/seo-files.js +38 -155
- package/src/uwx/backfill.js +41 -14
- package/src/uwx/link-records.js +1 -0
- package/src/uwx/yaml-edit.js +41 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@uniweb/build",
|
|
3
|
-
"version": "1.10.
|
|
3
|
+
"version": "1.10.2",
|
|
4
4
|
"description": "Build tooling for the Uniweb Component Web Platform",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -57,14 +57,14 @@
|
|
|
57
57
|
"sharp": "^0.35.5",
|
|
58
58
|
"yaml": "^2.5.0",
|
|
59
59
|
"@uniweb/content-reader": "~1.5.7",
|
|
60
|
-
"@uniweb/
|
|
61
|
-
"@uniweb/
|
|
62
|
-
"@uniweb/theming": "~1.2.1",
|
|
60
|
+
"@uniweb/projections": "~1.2.1",
|
|
61
|
+
"@uniweb/semantic-parser": "~1.13.0",
|
|
63
62
|
"@uniweb/schemas": "~1.6.5",
|
|
64
|
-
"@uniweb/
|
|
63
|
+
"@uniweb/theming": "~1.2.1",
|
|
64
|
+
"@uniweb/content-writer": "~1.3.1"
|
|
65
65
|
},
|
|
66
66
|
"optionalDependencies": {
|
|
67
|
-
"@uniweb/runtime": "~1.2.
|
|
67
|
+
"@uniweb/runtime": "~1.2.3"
|
|
68
68
|
},
|
|
69
69
|
"peerDependencies": {
|
|
70
70
|
"vite": "^5.0.0 || ^6.0.0 || ^7.0.0",
|
|
@@ -73,8 +73,8 @@
|
|
|
73
73
|
"@tailwindcss/vite": "^4.0.0",
|
|
74
74
|
"@vitejs/plugin-react": "^4.0.0 || ^5.0.0",
|
|
75
75
|
"vite-plugin-svgr": "^4.0.0",
|
|
76
|
-
"@uniweb/
|
|
77
|
-
"@uniweb/
|
|
76
|
+
"@uniweb/core": "~1.4.7",
|
|
77
|
+
"@uniweb/runtime": "~1.2.3"
|
|
78
78
|
},
|
|
79
79
|
"peerDependenciesMeta": {
|
|
80
80
|
"vite": {
|
package/src/prerender.js
CHANGED
|
@@ -34,7 +34,7 @@ import { getAdapter } from './hosts/index.js'
|
|
|
34
34
|
import { detectCiContext } from './hosts/detect-ci-context.js'
|
|
35
35
|
import { stripBasePath } from './site/extension-urls.js'
|
|
36
36
|
import { isSiteRelative, resolveExtensionDist } from './site/emit-extensions.js'
|
|
37
|
-
import { crawlerFiles,
|
|
37
|
+
import { crawlerFiles, sitemapEntriesOf } from './site/seo-files.js'
|
|
38
38
|
import { absoluteUrl } from '@uniweb/core/site-address'
|
|
39
39
|
|
|
40
40
|
/**
|
|
@@ -879,7 +879,7 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
879
879
|
|
|
880
880
|
// Pre-render each locale
|
|
881
881
|
const renderedFiles = []
|
|
882
|
-
// Each
|
|
882
|
+
// Each locale's sitemap entries, for the pages it wrote: the sitemap lists what this build serves.
|
|
883
883
|
const sitemapEntries = []
|
|
884
884
|
// A redirect stub names its target as the target's own pages do: absolutely, on the site's address.
|
|
885
885
|
const stubCanonical = (target) => absoluteUrl(defaultSiteContent.config?.url, target) || target
|
|
@@ -908,6 +908,10 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
908
908
|
// Store fetchedData on siteContent for runtime DataStore pre-population
|
|
909
909
|
siteContent.fetchedData = fetchedData
|
|
910
910
|
|
|
911
|
+
// The pages as authored, each parametric page its template: what the sitemap reads, with the
|
|
912
|
+
// record pages written from it (`./site/seo-files.js`).
|
|
913
|
+
const authoredPages = siteContent.pages
|
|
914
|
+
|
|
911
915
|
// Expand dynamic pages (e.g., /blog/:slug → /blog/post-1, /blog/post-2)
|
|
912
916
|
if (siteContent.pages?.some(p => p.isDynamic)) {
|
|
913
917
|
onProgress('Expanding dynamic routes...')
|
|
@@ -998,7 +1002,9 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
998
1002
|
* ⭐ A PAGE RENDERS AGAINST WHAT IT CARRIES — the entries its HTML embeds, which are all the
|
|
999
1003
|
* browser's store holds when it boots on that page (`initUniweb` hydrates from them). The
|
|
1000
1004
|
* browser renders over the prerendered page, so a render that reads more paints content the
|
|
1001
|
-
* browser does not have, and the browser replaces it with a loading state at boot.
|
|
1005
|
+
* browser does not have, and the browser replaces it with a loading state at boot. Each page
|
|
1006
|
+
* hands them to `renderer.render` as `fetchedData`, as a host's isolate does; the 404 page,
|
|
1007
|
+
* rendered on its own below, holds them the same way.
|
|
1002
1008
|
*
|
|
1003
1009
|
* ⛔ Until 2026-10-25 the store was hydrated once for the whole locale and then took every
|
|
1004
1010
|
* page's answers in turn, so a page could render data that only other pages carried. A
|
|
@@ -1007,11 +1013,9 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
1007
1013
|
* the record pages before it had fetched, carried none of them, and showed "loading…" at
|
|
1008
1014
|
* boot (measured).
|
|
1009
1015
|
*/
|
|
1010
|
-
const holdOnly = (carried) => {
|
|
1011
|
-
website.dataStore.clear()
|
|
1012
|
-
hydrateDataStore(website, carried)
|
|
1013
|
-
}
|
|
1014
1016
|
|
|
1017
|
+
// The route of each page written, as its head states it (`Page#getNavRoute`).
|
|
1018
|
+
const written = new Set()
|
|
1015
1019
|
for (const page of website.pages) {
|
|
1016
1020
|
// Skip dynamic template pages — they exist in the content for runtime
|
|
1017
1021
|
// route matching but can't be pre-rendered (no concrete route)
|
|
@@ -1109,7 +1113,7 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
1109
1113
|
const scopeRoutes = new Set([page.route, page.parent?.route, page.dynamicContext?.templateRoute].filter(Boolean))
|
|
1110
1114
|
// The same call `injectBuildData` makes below for this page's HTML, so the render holds what
|
|
1111
1115
|
// the HTML embeds — the page's own data step's answers among them.
|
|
1112
|
-
|
|
1116
|
+
const carried = scopeFetchedData(siteContent.fetchedData, splitContent ? scopeRoutes : null, page.route)
|
|
1113
1117
|
|
|
1114
1118
|
onProgress(`Rendering ${outputRoute}...`)
|
|
1115
1119
|
|
|
@@ -1120,7 +1124,7 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
1120
1124
|
// is how the two lanes drift. This lane passes the Page it already holds; the
|
|
1121
1125
|
// isolate passes a route. See `runtime/src/page-renderer.js` for what it leaves
|
|
1122
1126
|
// to the host.
|
|
1123
|
-
const result = renderer.render(page)
|
|
1127
|
+
const result = renderer.render(page, { fetchedData: carried })
|
|
1124
1128
|
|
|
1125
1129
|
if (result.outcome === 'failed') {
|
|
1126
1130
|
const { type, message } = result.error
|
|
@@ -1165,10 +1169,15 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
1165
1169
|
renderedFiles.push(outputPath)
|
|
1166
1170
|
onProgress(` → ${outputPath.replace(distDir, 'dist')}`)
|
|
1167
1171
|
// The page whose head was written — a content-less folder renders its index child.
|
|
1168
|
-
|
|
1169
|
-
if (entry) sitemapEntries.push(entry)
|
|
1172
|
+
written.add((result.page || page).getNavRoute())
|
|
1170
1173
|
}
|
|
1171
1174
|
|
|
1175
|
+
sitemapEntries.push(...sitemapEntriesOf({ ...siteContent, pages: authoredPages }, {
|
|
1176
|
+
locale,
|
|
1177
|
+
records: (siteContent.pages || []).filter((page) => page.dynamicContext),
|
|
1178
|
+
written,
|
|
1179
|
+
}))
|
|
1180
|
+
|
|
1172
1181
|
// Write 404.html — shared logic from @uniweb/runtime/ssr
|
|
1173
1182
|
const fallbackBaseHtml = injectBuildData(htmlShell, siteContent, {
|
|
1174
1183
|
splitContent,
|
|
@@ -1176,7 +1185,8 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
1176
1185
|
scopeRoutes: new Set(), // SPA fallback carries site-level fetched data only
|
|
1177
1186
|
})
|
|
1178
1187
|
// The 404 page renders against what 404.html carries, as every page above does.
|
|
1179
|
-
|
|
1188
|
+
website.forgetData()
|
|
1189
|
+
hydrateDataStore(website, scopeFetchedData(siteContent.fetchedData, splitContent ? new Set() : null, null))
|
|
1180
1190
|
const { html: notFoundHtml, hasNotFoundPage } = generate404Html({
|
|
1181
1191
|
baseHtml: fallbackBaseHtml,
|
|
1182
1192
|
website,
|
|
@@ -1213,12 +1223,7 @@ export async function prerenderSite(siteDir, options = {}) {
|
|
|
1213
1223
|
|
|
1214
1224
|
// ⭐ The crawler files, from the pages just written (`./site/seo-files.js`). A site's own
|
|
1215
1225
|
// `public/robots.txt` or `public/sitemap.xml` is its author's, and is left as it is.
|
|
1216
|
-
const crawl = crawlerFiles({
|
|
1217
|
-
address: defaultSiteContent.config?.url || null,
|
|
1218
|
-
base: defaultSiteContent.config?.base || '/',
|
|
1219
|
-
entries: sitemapEntries,
|
|
1220
|
-
robots: defaultSiteContent.config?.seo?.robots,
|
|
1221
|
-
})
|
|
1226
|
+
const crawl = crawlerFiles({ content: defaultSiteContent, entries: sitemapEntries })
|
|
1222
1227
|
for (const [file, body] of [['sitemap.xml', crawl.sitemap], ['robots.txt', crawl.robots]]) {
|
|
1223
1228
|
if (!body) continue
|
|
1224
1229
|
if (existsSync(join(siteDir, 'public', file))) {
|
package/src/site/seo-files.js
CHANGED
|
@@ -1,171 +1,56 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `sitemap.xml` and `robots.txt` — the two files a crawler reads about a statically served site,
|
|
3
|
-
* written by the prerender
|
|
3
|
+
* written by the prerender once it has written the pages.
|
|
4
4
|
*
|
|
5
|
-
* ⭐
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
5
|
+
* ⭐ BOTH ARE `@uniweb/projections`', made from the site's content alone (`sitemapEntries`,
|
|
6
|
+
* `renderSitemap`, `renderRobotsTxt`), so a host serving the site from its content serves the same
|
|
7
|
+
* files. A page's entry is the canonical and the alternates its `<head>` states, by the rules
|
|
8
|
+
* `Page#getHeadMeta` states them with (guarded by projections' `tests/sitemap-heads.test.js`).
|
|
9
|
+
* ⛔ Until 2026-10-25 the build read each entry off the rendered page instead, and until earlier
|
|
10
|
+
* that day the plugin wrote the sitemap at bundle time, before parametric pages were expanded: it
|
|
11
|
+
* listed `/books/:slug` and no book, at URLs that left the base path out.
|
|
11
12
|
*
|
|
12
|
-
* ⭐
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*/
|
|
16
|
-
|
|
17
|
-
/** The three defined Content Signals, in the order they are emitted. */
|
|
18
|
-
const CONTENT_SIGNAL_KEYS = ['search', 'ai-input', 'ai-train']
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Format the `Content-Signal:` directive from `seo.robots.contentSignals`.
|
|
22
|
-
*
|
|
23
|
-
* Content Signals express what a site permits its content to be *used for*,
|
|
24
|
-
* which is a different axis from `Disallow:` — that governs fetching, this
|
|
25
|
-
* governs use after fetching. The three defined signals:
|
|
26
|
-
*
|
|
27
|
-
* - `search` — appear in search results
|
|
28
|
-
* - `ai-input` — be retrieved at inference time (RAG, grounding)
|
|
29
|
-
* - `ai-train` — be used to train a model
|
|
30
|
-
*
|
|
31
|
-
* **Emitted only when declared.** There is no default: a preference the site
|
|
32
|
-
* owner did not state is not ours to assert, in either direction. An absent
|
|
33
|
-
* signal means "unstated", which is not the same as `no`.
|
|
34
|
-
*
|
|
35
|
-
* Unknown keys are ignored rather than passed through — the vocabulary is a
|
|
36
|
-
* closed set, and forwarding an invented signal would produce a directive no
|
|
37
|
-
* crawler honors while reading as though it were doing something.
|
|
38
|
-
*
|
|
39
|
-
* @param {Object|null} signals - e.g. `{ search: true, 'ai-input': true, 'ai-train': false }`
|
|
40
|
-
* @returns {string} The directive line, or `''` when nothing is declared
|
|
41
|
-
*/
|
|
42
|
-
export function formatContentSignals(signals) {
|
|
43
|
-
if (!signals || typeof signals !== 'object') return ''
|
|
44
|
-
|
|
45
|
-
const parts = []
|
|
46
|
-
for (const key of CONTENT_SIGNAL_KEYS) {
|
|
47
|
-
if (!(key in signals)) continue
|
|
48
|
-
const value = signals[key]
|
|
49
|
-
const yes = value === true || value === 'yes'
|
|
50
|
-
const no = value === false || value === 'no'
|
|
51
|
-
if (!yes && !no) continue
|
|
52
|
-
parts.push(`${key}=${yes ? 'yes' : 'no'}`)
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
return parts.length ? `Content-Signal: ${parts.join(', ')}` : ''
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
/**
|
|
59
|
-
* `robots.txt` — the site's rules, and where its sitemap is.
|
|
13
|
+
* ⭐ THE SITEMAP LISTS THE PAGES A STATIC HOST WILL ANSWER WITH: the pages this build wrote, each
|
|
14
|
+
* record's own among them. A page it wrote no file for — one whose render failed — is answered with
|
|
15
|
+
* the not-found page, so it is left out, where a host that renders on request lists it.
|
|
60
16
|
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* @param {Object} [options]
|
|
64
|
-
* @param {string|null} [options.sitemap] - the sitemap's absolute URL, when there is one
|
|
65
|
-
* @returns {string}
|
|
17
|
+
* ⭐ A site served under a base path — a GitHub project site — gets no `robots.txt`, since a crawler
|
|
18
|
+
* reads it only at a host's root, and the build says so.
|
|
66
19
|
*/
|
|
67
|
-
export function generateRobotsTxt(robots = {}, { sitemap = null } = {}) {
|
|
68
|
-
const { disallow = [], allow = [], crawlDelay = null, additionalSitemaps = [], contentSignals = null } = robots || {}
|
|
69
20
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const signals = formatContentSignals(contentSignals)
|
|
73
|
-
if (signals) {
|
|
74
|
-
// Inside the User-agent group, before the rules — the directive applies to
|
|
75
|
-
// the group it sits in.
|
|
76
|
-
content += `${signals}\n`
|
|
77
|
-
}
|
|
78
|
-
for (const path of allow) content += `Allow: ${path}\n`
|
|
79
|
-
for (const path of disallow) content += `Disallow: ${path}\n`
|
|
80
|
-
if (crawlDelay) content += `Crawl-delay: ${crawlDelay}\n`
|
|
81
|
-
|
|
82
|
-
const sitemaps = [sitemap, ...additionalSitemaps].filter(Boolean)
|
|
83
|
-
if (sitemaps.length) content += `\n${sitemaps.map((url) => `Sitemap: ${url}`).join('\n')}\n`
|
|
84
|
-
return content
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/** Whether `seo.robots` says anything a crawler would act on. */
|
|
88
|
-
function hasRules(robots) {
|
|
89
|
-
if (!robots || typeof robots !== 'object') return false
|
|
90
|
-
return Boolean(
|
|
91
|
-
robots.allow?.length ||
|
|
92
|
-
robots.disallow?.length ||
|
|
93
|
-
robots.crawlDelay ||
|
|
94
|
-
robots.additionalSitemaps?.length ||
|
|
95
|
-
formatContentSignals(robots.contentSignals)
|
|
96
|
-
)
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* A rendered page's sitemap entry — its canonical URL, with its alternates — or null when it is not
|
|
101
|
-
* one to list: kept out of search (`noindex`), naming another URL as its canonical, or on a site
|
|
102
|
-
* whose address is unknown.
|
|
103
|
-
*
|
|
104
|
-
* @param {Object} page - a Page of the Website the prerender rendered it from
|
|
105
|
-
* @returns {{ loc: string, alternates: Array, lastmod: string|null, changefreq: string|null, priority: (string|number)|null }|null}
|
|
106
|
-
*/
|
|
107
|
-
export function sitemapEntry(page) {
|
|
108
|
-
const meta = page?.getHeadMeta?.()
|
|
109
|
-
if (!meta?.canonical || meta.robots?.includes('noindex')) return null
|
|
110
|
-
const own = page.website?.getPageUrl?.(page)
|
|
111
|
-
if (!own || meta.canonical !== own) return null
|
|
112
|
-
return {
|
|
113
|
-
loc: own,
|
|
114
|
-
alternates: meta.alternates || [],
|
|
115
|
-
// A record's page has no file of its own, so no date of its own: none is better than its template's.
|
|
116
|
-
lastmod: !page.dynamicContext && page.lastModified ? String(page.lastModified).split('T')[0] : null,
|
|
117
|
-
changefreq: page.seo?.changefreq || null,
|
|
118
|
-
priority: page.seo?.priority ?? null,
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
/** Escape a value for XML text or a double-quoted attribute. */
|
|
123
|
-
function escapeXml(str) {
|
|
124
|
-
return String(str)
|
|
125
|
-
.replace(/&/g, '&')
|
|
126
|
-
.replace(/</g, '<')
|
|
127
|
-
.replace(/>/g, '>')
|
|
128
|
-
.replace(/"/g, '"')
|
|
129
|
-
.replace(/'/g, ''')
|
|
130
|
-
}
|
|
21
|
+
import { renderRobotsTxt, renderSitemap, sitemapEntries } from '@uniweb/projections'
|
|
131
22
|
|
|
132
23
|
/**
|
|
133
|
-
*
|
|
134
|
-
*
|
|
24
|
+
* One language's sitemap entries, for the pages this build wrote: `sitemapEntries` for its content
|
|
25
|
+
* and its record pages, less any page the build wrote no file for.
|
|
135
26
|
*
|
|
136
|
-
* @param {
|
|
137
|
-
* @
|
|
27
|
+
* @param {Object} content - the language's content, as authored: each parametric page its template
|
|
28
|
+
* @param {Object} options
|
|
29
|
+
* @param {string} options.locale - the language
|
|
30
|
+
* @param {Object[]} [options.records] - its record pages, as expanded (`expandDynamicPages`)
|
|
31
|
+
* @param {Set<string>} options.written - the route of each page written, as its head states it
|
|
32
|
+
* (`Page#getNavRoute`)
|
|
33
|
+
* @returns {Array} `sitemapEntries` results
|
|
138
34
|
*/
|
|
139
|
-
export function
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
const lines = [` <loc>${escapeXml(entry.loc)}</loc>`]
|
|
143
|
-
if (entry.lastmod) lines.push(` <lastmod>${escapeXml(entry.lastmod)}</lastmod>`)
|
|
144
|
-
if (entry.changefreq) lines.push(` <changefreq>${escapeXml(entry.changefreq)}</changefreq>`)
|
|
145
|
-
if (entry.priority !== null && entry.priority !== undefined) lines.push(` <priority>${escapeXml(entry.priority)}</priority>`)
|
|
146
|
-
for (const { hreflang, url } of entry.alternates || []) {
|
|
147
|
-
lines.push(` <xhtml:link rel="alternate" hreflang="${escapeXml(hreflang)}" href="${escapeXml(url)}" />`)
|
|
148
|
-
}
|
|
149
|
-
return ` <url>\n${lines.join('\n')}\n </url>`
|
|
150
|
-
})
|
|
151
|
-
const xmlns = alternated ? ' xmlns:xhtml="http://www.w3.org/1999/xhtml"' : ''
|
|
152
|
-
return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"${xmlns}>\n${urls.join('\n')}\n</urlset>\n`
|
|
35
|
+
export function sitemapEntriesOf(content, { locale, records = [], written }) {
|
|
36
|
+
const entries = sitemapEntries(content, { locale, records: records.map((page) => ({ type: 'record', route: page.route })) })
|
|
37
|
+
return entries.filter((entry) => written.has(entry.route))
|
|
153
38
|
}
|
|
154
39
|
|
|
155
40
|
/**
|
|
156
41
|
* The site's crawler files, for what this build serves: the sitemap when the site's address is
|
|
157
|
-
* known and it has pages to list, `robots.txt`
|
|
158
|
-
* sitemap to name.
|
|
42
|
+
* known and it has pages to list, and `robots.txt` (`renderRobotsTxt`), naming that sitemap.
|
|
159
43
|
*
|
|
160
44
|
* @param {Object} options
|
|
161
|
-
* @param {
|
|
162
|
-
*
|
|
163
|
-
* @param {Array} options.entries - `
|
|
164
|
-
* @param {Object} [options.robots] - `seo.robots`
|
|
45
|
+
* @param {Object} options.content - the site's content: its address (`config.url`), the base it is
|
|
46
|
+
* built for (`config.base`) and its rules (`config.seo.robots`)
|
|
47
|
+
* @param {Array} options.entries - every language's entries (`sitemapEntriesOf`)
|
|
165
48
|
* @returns {{ sitemap: string|null, robots: string|null, notes: string[] }} what to write, and why
|
|
166
49
|
* anything was not
|
|
167
50
|
*/
|
|
168
|
-
export function crawlerFiles({
|
|
51
|
+
export function crawlerFiles({ content, entries = [] }) {
|
|
52
|
+
const config = content?.config || {}
|
|
53
|
+
const address = config.url || null
|
|
169
54
|
const notes = []
|
|
170
55
|
let sitemap = null
|
|
171
56
|
if (!address) {
|
|
@@ -175,16 +60,14 @@ export function crawlerFiles({ address, base = '/', entries = [], robots = {} })
|
|
|
175
60
|
} else {
|
|
176
61
|
sitemap = renderSitemap(entries)
|
|
177
62
|
}
|
|
178
|
-
const
|
|
179
|
-
const
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
if (sitemapUrl || hasRules(robots)) robotsTxt = generateRobotsTxt(robots, { sitemap: sitemapUrl })
|
|
183
|
-
} else {
|
|
63
|
+
const robots = renderRobotsTxt(content, { sitemap: Boolean(sitemap) })
|
|
64
|
+
const base = config.base || '/'
|
|
65
|
+
if (base !== '/') {
|
|
66
|
+
const sitemapUrl = sitemap ? new URL('sitemap.xml', address).href : null
|
|
184
67
|
notes.push(
|
|
185
68
|
`No robots.txt: crawlers read it only at a host's root, and this site is served under ${base}.` +
|
|
186
69
|
(sitemapUrl ? ` Submit ${sitemapUrl} to search engines directly.` : '')
|
|
187
70
|
)
|
|
188
71
|
}
|
|
189
|
-
return { sitemap, robots
|
|
72
|
+
return { sitemap, robots, notes }
|
|
190
73
|
}
|
package/src/uwx/backfill.js
CHANGED
|
@@ -4,13 +4,19 @@
|
|
|
4
4
|
// `$id` with no `$uuid` on first sync; the backend mints `$uuid` and returns it;
|
|
5
5
|
// this writes it into the record in its source file (docs/reference/entity-content.md).
|
|
6
6
|
//
|
|
7
|
-
// Write-back
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
7
|
+
// ⭐ Write-back ADDS ONE LINE: `$uuid` as the record's leading key, in a YAML file or a
|
|
8
|
+
// markdown file's frontmatter (`yaml-edit.js::setYamlKeyFirst`), and changes nothing else
|
|
9
|
+
// the author wrote — comments, quotes, blank lines, line breaks, `[a, b]` lists. A re-sync
|
|
10
|
+
// with the same `$uuid` is byte-identical. It reads from the SOURCE file (not the backend's
|
|
11
|
+
// finalized document), so field values round-trip untouched. A file that cannot be edited
|
|
12
|
+
// in place (an alias, a flow-style map) is re-rendered whole and the caller is warned; JSON,
|
|
13
|
+
// which holds no comments, is re-rendered as before.
|
|
14
|
+
//
|
|
15
|
+
// ⛔ Until 2026-10-25 every file was re-rendered whole, which the first push of a site's
|
|
16
|
+
// records made into a commit of every record file: on a site's first publish, measured on
|
|
17
|
+
// 16 hand-written records, every comment was deleted, every long line refolded, every
|
|
18
|
+
// `[a, b]` list exploded and every blank line between keys removed — the price of "one
|
|
19
|
+
// uniform path", written down as accepted before `yaml-edit.js` could edit in place.
|
|
14
20
|
//
|
|
15
21
|
// ⛔ A PUSH NEVER RENDERS THE BACKEND'S DOCUMENT OVER THE AUTHOR'S FILE. It did, for
|
|
16
22
|
// a record's first push, from 2026-05-29 to 2026-09-23 ("variant A"), and that
|
|
@@ -34,6 +40,7 @@ import yaml from 'js-yaml'
|
|
|
34
40
|
import { YAML_OPTIONS } from '../utils/yaml-schema.js'
|
|
35
41
|
import { proseMirrorToMarkdown } from '@uniweb/content-writer'
|
|
36
42
|
import { parseFrontmatter } from './entity-source.js'
|
|
43
|
+
import { setYamlKeyFirst } from './yaml-edit.js'
|
|
37
44
|
import { isProseMirrorField, isOpenMapSection, openMapFromRows } from './data-schema.js'
|
|
38
45
|
import { recordFileLayout, contentBodyTarget } from './record-layout.js'
|
|
39
46
|
import { unwrapLocalizedContent } from './locale-sync.js'
|
|
@@ -63,13 +70,16 @@ function withUuidFirst(obj, uuid) {
|
|
|
63
70
|
}
|
|
64
71
|
|
|
65
72
|
/**
|
|
66
|
-
* Insert `$uuid` into a single-record source file
|
|
67
|
-
*
|
|
68
|
-
*
|
|
73
|
+
* Insert `$uuid` into a single-record source file as its leading key, editing nothing else
|
|
74
|
+
* in a YAML file or a markdown file's frontmatter (`setYamlKeyFirst`): an author's comments,
|
|
75
|
+
* quotes, blank lines and line breaks stay as written. A file that cannot be edited in place
|
|
76
|
+
* (an alias, a flow-style map) is re-rendered whole, and the result says so. Idempotent: the
|
|
77
|
+
* same `$uuid` leaves the file untouched.
|
|
69
78
|
*
|
|
70
79
|
* @param {string} filePath
|
|
71
80
|
* @param {string} uuid
|
|
72
|
-
* @returns {{ status: 'updated'|'unchanged'|'deferred'|'error', message?: string }}
|
|
81
|
+
* @returns {{ status: 'updated'|'unchanged'|'deferred'|'error', message?: string, warning?: string }}
|
|
82
|
+
* `warning` when the file was re-rendered whole
|
|
73
83
|
*/
|
|
74
84
|
export function backfillUuid(filePath, uuid) {
|
|
75
85
|
const dot = filePath.lastIndexOf('.')
|
|
@@ -83,6 +93,7 @@ export function backfillUuid(filePath, uuid) {
|
|
|
83
93
|
}
|
|
84
94
|
|
|
85
95
|
let next
|
|
96
|
+
let rewritten = false
|
|
86
97
|
if (ext === '.json') {
|
|
87
98
|
let obj
|
|
88
99
|
try {
|
|
@@ -104,17 +115,32 @@ export function backfillUuid(filePath, uuid) {
|
|
|
104
115
|
if (Array.isArray(obj)) {
|
|
105
116
|
return { status: 'deferred', message: 'array-form YAML (many records) not handled in v1' }
|
|
106
117
|
}
|
|
107
|
-
next =
|
|
118
|
+
next = setYamlKeyFirst(text, '$uuid', uuid)
|
|
119
|
+
if (next === null) {
|
|
120
|
+
next = yaml.dump(withUuidFirst(obj && typeof obj === 'object' ? obj : {}, uuid))
|
|
121
|
+
rewritten = true
|
|
122
|
+
}
|
|
108
123
|
} else if (ext === '.md') {
|
|
109
124
|
const { frontmatter, body } = parseFrontmatter(text, filePath)
|
|
110
|
-
|
|
125
|
+
// Split as `parseFrontmatter` splits, so the frontmatter edited is the one it reads and
|
|
126
|
+
// everything around it stays byte for byte.
|
|
127
|
+
const parts = text.split('---\n')
|
|
128
|
+
const edited = text.trimStart().startsWith('---') && parts.length >= 3 ? setYamlKeyFirst(parts[1], '$uuid', uuid) : null
|
|
129
|
+
if (edited !== null) next = [parts[0], edited, ...parts.slice(2)].join('---\n')
|
|
130
|
+
else {
|
|
131
|
+
next = `---\n${yaml.dump(withUuidFirst(frontmatter, uuid))}---\n${body}`
|
|
132
|
+
// A file with no frontmatter gains one, which loses nothing.
|
|
133
|
+
rewritten = text.trimStart().startsWith('---')
|
|
134
|
+
}
|
|
111
135
|
} else {
|
|
112
136
|
return { status: 'deferred', message: `${ext || '(no extension)'} back-fill is not yet implemented` }
|
|
113
137
|
}
|
|
114
138
|
|
|
115
139
|
if (next === text) return { status: 'unchanged' }
|
|
116
140
|
writeFileSync(filePath, next)
|
|
117
|
-
return
|
|
141
|
+
return rewritten
|
|
142
|
+
? { status: 'updated', warning: 'rewritten whole to record its $uuid — it could not be edited in place, so its comments and layout are gone' }
|
|
143
|
+
: { status: 'updated' }
|
|
118
144
|
}
|
|
119
145
|
|
|
120
146
|
/**
|
|
@@ -473,6 +499,7 @@ export function backfillEntityUuids({ index, finalized }) {
|
|
|
473
499
|
|
|
474
500
|
// The uuid, into the author's own file — never the returned document (header ⛔).
|
|
475
501
|
const res = backfillUuid(entry.sourceFile, uuid)
|
|
502
|
+
if (res.warning) warnings.push(`${entry.sourceFile}: ${res.warning}`)
|
|
476
503
|
if (res.status === 'updated') updated.push(entry.sourceFile)
|
|
477
504
|
else if (res.status === 'unchanged') unchanged.push(entry.sourceFile)
|
|
478
505
|
else if (res.status === 'deferred') deferred.push({ index: i, id: entry.id, file: entry.sourceFile, reason: res.message })
|
package/src/uwx/link-records.js
CHANGED
|
@@ -132,6 +132,7 @@ export function backfillLinkUuids({ links = [], folderDoc }) {
|
|
|
132
132
|
warnings.push(`${link.sourceFile}: ${res.message}`)
|
|
133
133
|
continue
|
|
134
134
|
}
|
|
135
|
+
if (res.warning) warnings.push(`${link.sourceFile}: ${res.warning}`)
|
|
135
136
|
if (res.status === 'updated') updated.push(link.sourceFile)
|
|
136
137
|
mapped[item.$uuid] = item.$uuid
|
|
137
138
|
}
|
package/src/uwx/yaml-edit.js
CHANGED
|
@@ -289,6 +289,47 @@ export function setYamlKey(text, key, value) {
|
|
|
289
289
|
return before ? editYamlText(text, before, { ...before, [key]: value }) : null
|
|
290
290
|
}
|
|
291
291
|
|
|
292
|
+
/**
|
|
293
|
+
* Set top-level `key` to `value` as the map's FIRST key — a line of its own before the first
|
|
294
|
+
* key, so after any comments that head the file — editing nothing else. A key already there is
|
|
295
|
+
* set where it stands (`editYamlText`).
|
|
296
|
+
*
|
|
297
|
+
* For an identity a sync writes into a file an author wrote (`$uuid`, `backfill.js`), which leads
|
|
298
|
+
* a record by convention, as a record a pull writes has it.
|
|
299
|
+
*
|
|
300
|
+
* @param {string} text - the file's contents (a record's frontmatter, for a markdown record)
|
|
301
|
+
* @param {string} key
|
|
302
|
+
* @param {*} value
|
|
303
|
+
* @returns {string|null} the new contents; null when the text cannot be edited in place — not a
|
|
304
|
+
* block map, an alias or merge key, a parse error, or a result that reads differently
|
|
305
|
+
*/
|
|
306
|
+
export function setYamlKeyFirst(text, key, value) {
|
|
307
|
+
const before = readMap(text)
|
|
308
|
+
if (!before) return null
|
|
309
|
+
if (Object.hasOwn(before, key)) return editYamlText(text, before, { ...before, [key]: value })
|
|
310
|
+
let doc
|
|
311
|
+
try {
|
|
312
|
+
doc = parseDocument(text)
|
|
313
|
+
} catch {
|
|
314
|
+
return null
|
|
315
|
+
}
|
|
316
|
+
if (doc.errors.length || usesReferences(doc)) return null
|
|
317
|
+
const line = yaml.dump({ [key]: value }, YAML_DUMP_OPTS)
|
|
318
|
+
let out
|
|
319
|
+
if (doc.contents === null || doc.contents === undefined) {
|
|
320
|
+
// Comments and nothing else: the key goes after them.
|
|
321
|
+
out = text + (text && !text.endsWith('\n') ? '\n' : '') + line
|
|
322
|
+
} else {
|
|
323
|
+
if (!isMap(doc.contents) || doc.contents.flow || !doc.contents.items[0]?.key?.range) return null
|
|
324
|
+
const first = doc.contents.items[0].key
|
|
325
|
+
if (columnOf(text, first) !== 0) return null
|
|
326
|
+
const at = lineStart(text, first.range[0])
|
|
327
|
+
out = text.slice(0, at) + line + text.slice(at)
|
|
328
|
+
}
|
|
329
|
+
const read = readMap(out)
|
|
330
|
+
return read && sameValue(read, { [key]: value, ...before }) ? out : null
|
|
331
|
+
}
|
|
332
|
+
|
|
292
333
|
/**
|
|
293
334
|
* Replace entries of the top-level list `key` — each `from` entry becomes its `to` — where
|
|
294
335
|
* they stand, in the quotes they were written in. An entry that only CONTAINS `from` is
|