@uniweb/build 0.15.4 → 0.15.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -7
- package/src/search/collections.js +2 -54
- package/src/search/extract.js +2 -335
- package/src/search/generate.js +8 -108
- package/src/search/index.js +8 -22
- package/src/site/build-site-data.js +80 -3
- package/src/site/content-collector.js +22 -4
- package/src/site/data-ball.js +43 -3
- package/src/site/plugin.js +130 -16
- package/src/uwx/collections-project.js +0 -1
- package/src/uwx/entity-document.js +34 -2
- package/src/uwx/index.js +9 -0
- package/src/uwx/site-diff.js +294 -0
- package/src/uwx/site-project.js +15 -4
- package/src/uwx/site.js +58 -11
- package/src/uwx/sync-package.js +81 -3
|
@@ -1164,11 +1164,9 @@ async function processPage(pagePath, pageName, siteRoot, { isIndex = false, pare
|
|
|
1164
1164
|
}
|
|
1165
1165
|
}
|
|
1166
1166
|
|
|
1167
|
-
// Create items with .name property for applyWildcardOrder
|
|
1168
|
-
const allItems = [...discoveredMap.keys()].map(name => ({ name }))
|
|
1169
|
-
const ordered = applyWildcardOrder(allItems, sectionsParsed)
|
|
1170
|
-
|
|
1171
1167
|
// Collect subsection configs from the original array (e.g., { features: [a, b] })
|
|
1168
|
+
// BEFORE ordering, because the children they name must not also be treated as
|
|
1169
|
+
// top-level sections.
|
|
1172
1170
|
const subsectionConfigs = new Map()
|
|
1173
1171
|
for (const item of [...sectionsParsed.before, ...sectionsParsed.after]) {
|
|
1174
1172
|
if (typeof item === 'object' && item !== null) {
|
|
@@ -1179,6 +1177,26 @@ async function processPage(pagePath, pageName, siteRoot, { isIndex = false, pare
|
|
|
1179
1177
|
}
|
|
1180
1178
|
}
|
|
1181
1179
|
|
|
1180
|
+
// A child named in `nest`-style config is already placed, so it must not fall
|
|
1181
|
+
// into the auto-discovered "rest" as well. `@`-prefixed files are excluded from
|
|
1182
|
+
// discovery above, but a child does NOT have to be `@`-prefixed: `uniweb pull`
|
|
1183
|
+
// writes every section — nested or not — as a flat `<stableId>.md` in the page
|
|
1184
|
+
// dir. Without this, an inclusive list flattens exactly the nesting the list
|
|
1185
|
+
// exists to preserve, promoting each child to a sibling of its own parent.
|
|
1186
|
+
const claimedChildren = new Set()
|
|
1187
|
+
for (const children of subsectionConfigs.values()) {
|
|
1188
|
+
for (const child of children || []) {
|
|
1189
|
+
const name = typeof child === 'string' ? child : Object.keys(child || {})[0]
|
|
1190
|
+
if (name) claimedChildren.add(name)
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
// Create items with .name property for applyWildcardOrder
|
|
1195
|
+
const allItems = [...discoveredMap.keys()]
|
|
1196
|
+
.filter(name => !claimedChildren.has(name))
|
|
1197
|
+
.map(name => ({ name }))
|
|
1198
|
+
const ordered = applyWildcardOrder(allItems, sectionsParsed)
|
|
1199
|
+
|
|
1182
1200
|
// Process sections in wildcard-expanded order
|
|
1183
1201
|
const sections = []
|
|
1184
1202
|
let sectionIndex = 1
|
package/src/site/data-ball.js
CHANGED
|
@@ -56,7 +56,7 @@ function collectionOf(relPath) {
|
|
|
56
56
|
* @returns {Promise<{ data: Object, search: Object }|null>} null when there is nothing
|
|
57
57
|
* to deliver (no schema-less data AND no search index).
|
|
58
58
|
*/
|
|
59
|
-
export async function assembleDataBall(distDir, schemalessNames = []) {
|
|
59
|
+
export async function assembleDataBall(distDir, schemalessNames = [], { projections = false } = {}) {
|
|
60
60
|
const schemaless = new Set(schemalessNames)
|
|
61
61
|
const allData = await readJsonTree(join(distDir, 'data'))
|
|
62
62
|
const data = {}
|
|
@@ -64,8 +64,48 @@ export async function assembleDataBall(distDir, schemalessNames = []) {
|
|
|
64
64
|
if (schemaless.has(collectionOf(relPath))) data[relPath] = value
|
|
65
65
|
}
|
|
66
66
|
const search = await readJsonTree(join(distDir, '_search'))
|
|
67
|
-
|
|
68
|
-
|
|
67
|
+
|
|
68
|
+
// Agent projections ride the same ball — same shape of thing (precomputed
|
|
69
|
+
// at publish, opaque to the backend, served as-is), just text rather than
|
|
70
|
+
// JSON. Off by default: what the artifacts are called and how they are
|
|
71
|
+
// addressed on the serving side is the backend's to specify, and shipping
|
|
72
|
+
// an unrecognized key into a payload the backend unwraps is not something
|
|
73
|
+
// to decide unilaterally. Flip this on once that contract is agreed.
|
|
74
|
+
const projectionFiles = projections ? await readTextTree(distDir) : {}
|
|
75
|
+
|
|
76
|
+
const empty =
|
|
77
|
+
Object.keys(data).length === 0 &&
|
|
78
|
+
Object.keys(search).length === 0 &&
|
|
79
|
+
Object.keys(projectionFiles).length === 0
|
|
80
|
+
if (empty) return null
|
|
81
|
+
|
|
82
|
+
return projections ? { data, search, projections: projectionFiles } : { data, search }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Collect the agent projections from a built dist/ — `llms.txt` plus every
|
|
87
|
+
* `.md` page projection, keyed by their served path.
|
|
88
|
+
*
|
|
89
|
+
* Deliberately excludes `node_modules`-shaped noise and anything under the
|
|
90
|
+
* bundle's own asset directories; a site's `dist/` at this point holds only
|
|
91
|
+
* what the link lane emitted.
|
|
92
|
+
*
|
|
93
|
+
* @param {string} distDir
|
|
94
|
+
* @returns {Promise<Object<string, string>>}
|
|
95
|
+
*/
|
|
96
|
+
async function readTextTree(distDir) {
|
|
97
|
+
if (!existsSync(distDir)) return {}
|
|
98
|
+
const out = {}
|
|
99
|
+
const entries = await readdir(distDir, { withFileTypes: true, recursive: true })
|
|
100
|
+
for (const entry of entries) {
|
|
101
|
+
if (!entry.isFile()) continue
|
|
102
|
+
const isProjection = entry.name === 'llms.txt' || entry.name.endsWith('.md')
|
|
103
|
+
if (!isProjection) continue
|
|
104
|
+
const full = join(entry.parentPath || entry.path, entry.name)
|
|
105
|
+
const rel = relative(distDir, full).split(sep).join('/')
|
|
106
|
+
out[rel] = await readFile(full, 'utf8')
|
|
107
|
+
}
|
|
108
|
+
return out
|
|
69
109
|
}
|
|
70
110
|
|
|
71
111
|
// --- local media in the ball -------------------------------------------------
|
package/src/site/plugin.js
CHANGED
|
@@ -34,6 +34,15 @@ import { resolve, join } from 'node:path'
|
|
|
34
34
|
import { watch, existsSync } from 'node:fs'
|
|
35
35
|
import { readFile, readdir } from 'node:fs/promises'
|
|
36
36
|
import { resolveDefaultLocale } from '@uniweb/core'
|
|
37
|
+
import {
|
|
38
|
+
renderSiteIndex,
|
|
39
|
+
renderPageMarkdown,
|
|
40
|
+
resolveAgentsConfig,
|
|
41
|
+
selectIndexablePages,
|
|
42
|
+
pageMarkdownFilename,
|
|
43
|
+
applyRouteTranslation,
|
|
44
|
+
INDEX_FILENAME
|
|
45
|
+
} from '@uniweb/projections'
|
|
37
46
|
import { collectSiteContent } from './content-collector.js'
|
|
38
47
|
import { processAssets, rewriteSiteContentPaths } from './asset-processor.js'
|
|
39
48
|
import { processAdvancedAssets } from './advanced-processors.js'
|
|
@@ -187,23 +196,12 @@ async function processDevSectionFetches(sections, fetchOptions) {
|
|
|
187
196
|
import { generateSearchIndex, isSearchEnabled, getSearchIndexFilename } from '../search/index.js'
|
|
188
197
|
import { mergeTranslations } from '../i18n/merge.js'
|
|
189
198
|
|
|
190
|
-
|
|
191
|
-
*
|
|
192
|
-
*
|
|
199
|
+
/*
|
|
200
|
+
* `applyRouteTranslation` now comes from `@uniweb/projections` (imported
|
|
201
|
+
* above). It used to be defined here; the agent index builds localized URLs
|
|
202
|
+
* from the same rule, and two copies would eventually disagree about where a
|
|
203
|
+
* localized page lives.
|
|
193
204
|
*/
|
|
194
|
-
function applyRouteTranslation(route, locale, routeTranslations) {
|
|
195
|
-
const localeMap = routeTranslations?.[locale]
|
|
196
|
-
if (!localeMap) return route
|
|
197
|
-
// Exact match
|
|
198
|
-
if (localeMap[route]) return localeMap[route]
|
|
199
|
-
// Prefix match
|
|
200
|
-
for (const [canonical, translated] of Object.entries(localeMap)) {
|
|
201
|
-
if (route.startsWith(canonical + '/')) {
|
|
202
|
-
return translated + route.slice(canonical.length)
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
return route
|
|
206
|
-
}
|
|
207
205
|
|
|
208
206
|
/**
|
|
209
207
|
* Restrict authored `seo.locales` hreflang entries to locales the payload
|
|
@@ -577,6 +575,76 @@ export function siteContentPlugin(options = {}) {
|
|
|
577
575
|
})
|
|
578
576
|
}
|
|
579
577
|
|
|
578
|
+
/**
|
|
579
|
+
* Options shared by every projection call, so the dev middleware and the
|
|
580
|
+
* build emit cannot drift apart in how they build URLs.
|
|
581
|
+
*/
|
|
582
|
+
function projectionOptions(content) {
|
|
583
|
+
const defaultLocale = resolveDefaultLocale(content.config)
|
|
584
|
+
return {
|
|
585
|
+
baseUrl: seoOptions.baseUrl,
|
|
586
|
+
basePath,
|
|
587
|
+
locale: content.config?.activeLocale || defaultLocale,
|
|
588
|
+
defaultLocale
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* Emit `llms.txt` and the per-page `.md` projections.
|
|
594
|
+
*
|
|
595
|
+
* Called with the Rollup plugin context, so `this.emitFile` is available.
|
|
596
|
+
* Sits beside the sitemap and search-index emits because it is the same kind
|
|
597
|
+
* of thing: an artifact derived from the site's content alone.
|
|
598
|
+
*
|
|
599
|
+
* Unlike the SEO artifacts, this does NOT silently skip when `seo.baseUrl`
|
|
600
|
+
* is unset — it falls back to root-relative links and `uniweb doctor` warns.
|
|
601
|
+
* An agent that arrived via the index can still follow a relative link, and
|
|
602
|
+
* a silently-absent index is the failure mode the whole feature exists to
|
|
603
|
+
* prevent.
|
|
604
|
+
*
|
|
605
|
+
* @param {Object} content - Final site content for this locale
|
|
606
|
+
*/
|
|
607
|
+
function emitProjections(content) {
|
|
608
|
+
const agents = resolveAgentsConfig(content?.config)
|
|
609
|
+
if (!agents.index && !agents.markdown) return
|
|
610
|
+
if (!content?.pages?.length) return
|
|
611
|
+
|
|
612
|
+
const options = projectionOptions(content)
|
|
613
|
+
const localeDir =
|
|
614
|
+
options.locale && options.defaultLocale && options.locale !== options.defaultLocale
|
|
615
|
+
? `${options.locale}/`
|
|
616
|
+
: ''
|
|
617
|
+
|
|
618
|
+
if (agents.index) {
|
|
619
|
+
this.emitFile({
|
|
620
|
+
type: 'asset',
|
|
621
|
+
fileName: `${localeDir}${INDEX_FILENAME}`,
|
|
622
|
+
source: renderSiteIndex(content, { ...options, exclude: agents.exclude })
|
|
623
|
+
})
|
|
624
|
+
console.log(`[site-content] Generated ${localeDir}${INDEX_FILENAME}`)
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
if (agents.markdown) {
|
|
628
|
+
const pages = selectIndexablePages(content.pages, { exclude: agents.exclude })
|
|
629
|
+
let emitted = 0
|
|
630
|
+
for (const page of pages) {
|
|
631
|
+
const markdown = renderPageMarkdown(page)
|
|
632
|
+
if (!markdown) continue
|
|
633
|
+
const route =
|
|
634
|
+
options.locale !== options.defaultLocale
|
|
635
|
+
? applyRouteTranslation(page.route, options.locale, content.config?.i18n?.routeTranslations)
|
|
636
|
+
: page.route
|
|
637
|
+
this.emitFile({
|
|
638
|
+
type: 'asset',
|
|
639
|
+
fileName: `${localeDir}${pageMarkdownFilename(route)}`,
|
|
640
|
+
source: markdown
|
|
641
|
+
})
|
|
642
|
+
emitted++
|
|
643
|
+
}
|
|
644
|
+
if (emitted) console.log(`[site-content] Generated ${emitted} page markdown projections`)
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
580
648
|
return {
|
|
581
649
|
name: 'uniweb:site-content',
|
|
582
650
|
|
|
@@ -928,6 +996,47 @@ export function siteContentPlugin(options = {}) {
|
|
|
928
996
|
}
|
|
929
997
|
}
|
|
930
998
|
|
|
999
|
+
// Serve the agent projections in dev mode.
|
|
1000
|
+
//
|
|
1001
|
+
// Without this the artifacts can only be inspected by deploying, and
|
|
1002
|
+
// `uniweb doctor`'s baseUrl warning describes something invisible
|
|
1003
|
+
// locally. Both routes reuse the build's generators, so what a
|
|
1004
|
+
// developer sees here is what ships.
|
|
1005
|
+
if (siteContent) {
|
|
1006
|
+
const agents = resolveAgentsConfig(siteContent.config)
|
|
1007
|
+
const url = req.url.split('?')[0]
|
|
1008
|
+
|
|
1009
|
+
const indexMatch = url.match(new RegExp(`^(?:\\/(${LOCALE_RE}))?\\/${INDEX_FILENAME}$`))
|
|
1010
|
+
if (indexMatch && agents.index) {
|
|
1011
|
+
const localized = (indexMatch[1] ? await getTranslatedContent(indexMatch[1]) : null) || siteContent
|
|
1012
|
+
res.setHeader('Content-Type', 'text/plain; charset=utf-8')
|
|
1013
|
+
res.end(
|
|
1014
|
+
renderSiteIndex(localized, {
|
|
1015
|
+
...projectionOptions(localized),
|
|
1016
|
+
locale: indexMatch[1] || projectionOptions(localized).locale,
|
|
1017
|
+
exclude: agents.exclude
|
|
1018
|
+
})
|
|
1019
|
+
)
|
|
1020
|
+
return
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
const markdownMatch = url.match(new RegExp(`^(?:\\/(${LOCALE_RE}))?\\/(.+)\\.md$`))
|
|
1024
|
+
if (markdownMatch && agents.markdown) {
|
|
1025
|
+
const localized = (markdownMatch[1] ? await getTranslatedContent(markdownMatch[1]) : null) || siteContent
|
|
1026
|
+
const requested = markdownMatch[2] === 'index' ? '/' : `/${markdownMatch[2]}`
|
|
1027
|
+
const pages = selectIndexablePages(localized.pages, { exclude: agents.exclude })
|
|
1028
|
+
const page = pages.find(p => p.route === requested)
|
|
1029
|
+
if (page) {
|
|
1030
|
+
const markdown = renderPageMarkdown(page)
|
|
1031
|
+
if (markdown) {
|
|
1032
|
+
res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
|
|
1033
|
+
res.end(markdown)
|
|
1034
|
+
return
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
|
|
931
1040
|
// Handle localized collection data (e.g., /fr/data/articles.json)
|
|
932
1041
|
const localeDataMatch = req.url.match(new RegExp(`^\\/(${LOCALE_RE})\\/data\\/(.+\\.json)$`))
|
|
933
1042
|
if (localeDataMatch) {
|
|
@@ -1234,6 +1343,11 @@ export function siteContentPlugin(options = {}) {
|
|
|
1234
1343
|
|
|
1235
1344
|
console.log(`[site-content] Generated ${searchFilename} (${searchIndex.count} entries)`)
|
|
1236
1345
|
}
|
|
1346
|
+
|
|
1347
|
+
// Agent-readable projections: the index (discovery) and per-page
|
|
1348
|
+
// markdown (retrieval). Free and on by default; a site opts out under
|
|
1349
|
+
// `agents:` in site.yml.
|
|
1350
|
+
emitProjections.call(this, finalContent)
|
|
1237
1351
|
},
|
|
1238
1352
|
|
|
1239
1353
|
closeBundle() {
|
|
@@ -43,7 +43,6 @@ import { createTranslationCollector, writeLocaleTranslations, writeFreeformTrans
|
|
|
43
43
|
import { buildFreeformCollectionPath } from '../i18n/freeform.js'
|
|
44
44
|
|
|
45
45
|
// Single-record source extensions we scan + place (BibTeX is multi-record → out).
|
|
46
|
-
const SINGLE_RECORD_EXTS = ['.md', '.yml', '.yaml', '.json']
|
|
47
46
|
const EXT_FOR_FORMAT = { md: '.md', yaml: '.yml', json: '.json' }
|
|
48
47
|
|
|
49
48
|
function formatForExt(ext) {
|
|
@@ -68,7 +68,7 @@ export function emitEntitySyncPackage({
|
|
|
68
68
|
for (const entity of entities) {
|
|
69
69
|
const data = toJsonBuffer(entity.document)
|
|
70
70
|
files.push({ name: entity.file, data })
|
|
71
|
-
|
|
71
|
+
const entry = {
|
|
72
72
|
kind: 'entity',
|
|
73
73
|
// `$id` as the handle label — NOT the identity key on the sync lane (the
|
|
74
74
|
// backend reads identity from the body) and never uuid-parsed for entities.
|
|
@@ -83,7 +83,39 @@ export function emitEntitySyncPackage({
|
|
|
83
83
|
updated_at: null,
|
|
84
84
|
file: entity.file,
|
|
85
85
|
sha256: sha256Hex(data),
|
|
86
|
-
}
|
|
86
|
+
}
|
|
87
|
+
// Optimistic-concurrency precondition (the push gate). When present, the
|
|
88
|
+
// backend compares it against the entity's stored version and refuses the
|
|
89
|
+
// WHOLE package atomically if it has moved (409 `reason: "stale_base"`),
|
|
90
|
+
// before any write. OMITTING it is the force path — the backend then falls
|
|
91
|
+
// back to the `collision` policy. Note `collision=force` does NOT override a
|
|
92
|
+
// present token: the token wins, so `--force` must DROP this, never flip
|
|
93
|
+
// `collision`. The value is the opaque RFC3339 `version` the backend stamped
|
|
94
|
+
// on the entity — read from the pull manifest's top-level `version` or from a
|
|
95
|
+
// prior push's `finalized[].version`, and never parsed or synthesized here.
|
|
96
|
+
//
|
|
97
|
+
// TOP-LEVEL on the entry, beside `sha256` — not nested under an `extra`
|
|
98
|
+
// object. The backend's Rust struct has an `extra` field but it is
|
|
99
|
+
// `#[serde(flatten)]`, so those keys serialize beside the others and `extra`
|
|
100
|
+
// never appears on the wire in either direction. Its pull manifests emit a
|
|
101
|
+
// top-level `version` for the same reason.
|
|
102
|
+
//
|
|
103
|
+
// The backend correlates this to an entity by the BODY's `$uuid`, not by
|
|
104
|
+
// `entries[].uuid` — which stays the `$id` handle above. Don't be tempted to
|
|
105
|
+
// write a real uuid into that field to "help": it would make the field mean
|
|
106
|
+
// two different things depending on sync state, and the backend has no use
|
|
107
|
+
// for it (it reads identity from the body everywhere else too).
|
|
108
|
+
if (entity.baseVersion) entry.base_version = entity.baseVersion
|
|
109
|
+
// Per-ITEM preconditions, keyed by each record's `$uuid`. The entity token
|
|
110
|
+
// gates the whole document, so a stale base on any one record refuses the
|
|
111
|
+
// entire push — which fires on the common case of two people editing different
|
|
112
|
+
// sections. These let the backend accept the records we may touch and refuse
|
|
113
|
+
// only the ones that genuinely moved. Same rules as the entity token: opaque,
|
|
114
|
+
// top-level on the entry (no `extra` wrapper), omitted = unconditional.
|
|
115
|
+
if (entity.itemBaseVersions && Object.keys(entity.itemBaseVersions).length) {
|
|
116
|
+
entry.item_base_versions = entity.itemBaseVersions
|
|
117
|
+
}
|
|
118
|
+
entries.push(entry)
|
|
87
119
|
}
|
|
88
120
|
|
|
89
121
|
const manifest = buildManifest({
|
package/src/uwx/index.js
CHANGED
|
@@ -68,6 +68,15 @@ export {
|
|
|
68
68
|
localeFilePath,
|
|
69
69
|
} from './locale-sync.js'
|
|
70
70
|
export { emitSyncPackages } from './sync-package.js'
|
|
71
|
+
export {
|
|
72
|
+
diffSiteUnits,
|
|
73
|
+
describeSiteDiff,
|
|
74
|
+
computeUnitHashes,
|
|
75
|
+
collectSiteUnits,
|
|
76
|
+
walkSiteUnits,
|
|
77
|
+
collectUnitUuids,
|
|
78
|
+
stampUnitUuids,
|
|
79
|
+
} from './site-diff.js'
|
|
71
80
|
export {
|
|
72
81
|
collectionRecordsToEntities,
|
|
73
82
|
buildCollectionEntities,
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
// File-level diff of two `@uniweb/site-content` documents.
|
|
2
|
+
//
|
|
3
|
+
// The push staleness gate is ENTITY-grained: a site's whole page tree is one
|
|
4
|
+
// entity, so a refusal can only say "the document moved". That leaves the user
|
|
5
|
+
// with "pull and lose yours, or force and lose theirs" and no way to judge either.
|
|
6
|
+
// This turns the refusal into an account of which units diverged and which side
|
|
7
|
+
// moved them.
|
|
8
|
+
//
|
|
9
|
+
// THE UNIT IS THE FILE, NOT THE PAGE. A site's content is already split one
|
|
10
|
+
// section per file, so two people editing different sections — even of the SAME
|
|
11
|
+
// page — have not conflicted. Diffing at page granularity reports that as a
|
|
12
|
+
// conflict, and a false conflict is not a harmless imprecision: the only moves it
|
|
13
|
+
// leaves are pulling (loses their work) or forcing (loses the other side's), so it
|
|
14
|
+
// actively pushes people toward the destructive one. Three unit kinds, matching
|
|
15
|
+
// exactly what someone would open:
|
|
16
|
+
//
|
|
17
|
+
// site.yml site info (name, theme, foundation ref)
|
|
18
|
+
// pages/<route>/page.yml page metadata (title, slug, section order, …)
|
|
19
|
+
// pages/<route>/<section>.md one section — including nested `$children`,
|
|
20
|
+
// which the projector also writes flat here
|
|
21
|
+
// layout/<section>.md a layout section
|
|
22
|
+
//
|
|
23
|
+
// Units are DISJOINT: a page's hash excludes its sections, a section's excludes its
|
|
24
|
+
// children, so a change lands in exactly one unit and is never double-reported.
|
|
25
|
+
// Naming comes from the projector's own helpers (`pageDirName`,
|
|
26
|
+
// `safeStableIdFilename`) rather than a local copy, so a label always names a file
|
|
27
|
+
// that really exists.
|
|
28
|
+
//
|
|
29
|
+
// It needs TWO bases, and that is the non-obvious part.
|
|
30
|
+
//
|
|
31
|
+
// Our document and the backend's are not byte-comparable renderings of the same
|
|
32
|
+
// unit: the backend's copy carries fields we don't emit (`params`,
|
|
33
|
+
// `theme_override`, its own `$uuid`) and serializes in its own key order. So a hash
|
|
34
|
+
// taken on our side and one taken on theirs differ for a unit NEITHER side touched.
|
|
35
|
+
// A single base therefore validates exactly one comparison and silently corrupts
|
|
36
|
+
// the other — with a local-representation base every unit looks "changed
|
|
37
|
+
// upstream", which is worse than saying nothing.
|
|
38
|
+
//
|
|
39
|
+
// So each side is compared only against a base in its OWN representation:
|
|
40
|
+
// local vs localBase → did WE change it
|
|
41
|
+
// remote vs remoteBase → did THEY change it
|
|
42
|
+
// and local-vs-remote is never compared directly. Both bases are captured at the
|
|
43
|
+
// same two moments (a successful push, a pull) from sources already to hand: our
|
|
44
|
+
// emitted document, and the backend's `finalized[].document` / the pulled document.
|
|
45
|
+
//
|
|
46
|
+
// Either base may be missing; the affected side degrades to "unknown" and is
|
|
47
|
+
// reported as unattributed rather than guessed at.
|
|
48
|
+
|
|
49
|
+
import { entityContentHash } from './collections.js'
|
|
50
|
+
import { recordStableId, safeStableIdFilename, pageDirName } from './site-project.js'
|
|
51
|
+
import { LOCALIZED_FIELD_ASSUMPTION } from './localize.js'
|
|
52
|
+
|
|
53
|
+
// A unit's own content, with the nested collections that are their own units
|
|
54
|
+
// removed — so editing a section never also marks its page (or its parent
|
|
55
|
+
// section) as changed.
|
|
56
|
+
const ownContent = (record, ...childKeys) => {
|
|
57
|
+
const copy = { ...record }
|
|
58
|
+
for (const k of childKeys) delete copy[k]
|
|
59
|
+
return copy
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Every diffable unit of a site-content document, keyed by the repo-relative path
|
|
64
|
+
* the projector would write it to.
|
|
65
|
+
*
|
|
66
|
+
* Paths use the conventional `pages/` and `layout/` roots. A site that relocates
|
|
67
|
+
* them via `info.paths` still gets stable, unique keys — only the displayed prefix
|
|
68
|
+
* would differ from disk, which is a labelling nicety, not a correctness issue.
|
|
69
|
+
*
|
|
70
|
+
* @returns {Map<string, object>} path → the unit's own record
|
|
71
|
+
*/
|
|
72
|
+
export function collectSiteUnits(doc, sourceLocale = LOCALIZED_FIELD_ASSUMPTION.defaultSourceLocale) {
|
|
73
|
+
const units = new Map()
|
|
74
|
+
walkSiteUnits(doc, (path, record, kind) => {
|
|
75
|
+
units.set(
|
|
76
|
+
path,
|
|
77
|
+
kind === 'page' ? ownContent(record, 'page_sections', '$children')
|
|
78
|
+
: kind === 'info' ? ownContent(record)
|
|
79
|
+
: ownContent(record, '$children')
|
|
80
|
+
)
|
|
81
|
+
}, sourceLocale)
|
|
82
|
+
return units
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Walk every unit of a site-content document, handing the callback the path and the
|
|
87
|
+
* LIVE record (not a copy — mutating it mutates the document).
|
|
88
|
+
*
|
|
89
|
+
* The single traversal behind hashing, uuid harvesting, and uuid stamping, so those
|
|
90
|
+
* three can never disagree about what a unit is or where it lives. A path that
|
|
91
|
+
* differs between them would silently mis-key a cache.
|
|
92
|
+
*
|
|
93
|
+
* @param {(path: string, record: object, kind: 'info'|'page'|'section') => void} cb
|
|
94
|
+
*/
|
|
95
|
+
export function walkSiteUnits(doc, cb, sourceLocale = LOCALIZED_FIELD_ASSUMPTION.defaultSourceLocale) {
|
|
96
|
+
const walkSections = (sections, dir) => {
|
|
97
|
+
for (const record of sections || []) {
|
|
98
|
+
const id = recordStableId(record)
|
|
99
|
+
// Anonymous and id-less: the projector cannot place it either, so there is
|
|
100
|
+
// no file to name and nothing to attribute.
|
|
101
|
+
if (id) cb(`${dir}/${safeStableIdFilename(id)}.md`, record, 'section')
|
|
102
|
+
walkSections(record.$children, dir)
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const walkPages = (pages, parentDir) => {
|
|
106
|
+
for (const record of pages || []) {
|
|
107
|
+
const dirName = pageDirName(record, sourceLocale)
|
|
108
|
+
if (!dirName) continue
|
|
109
|
+
const dir = `${parentDir}/${dirName}`
|
|
110
|
+
cb(`${dir}/${record.mode === 'folder' ? 'folder.yml' : 'page.yml'}`, record, 'page')
|
|
111
|
+
if (record.mode !== 'folder') walkSections(record.page_sections, dir)
|
|
112
|
+
walkPages(record.$children, dir)
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
// `info` is an item too — it carries its own `$uuid` and its own per-item token,
|
|
116
|
+
// and it holds the site's name, theme and foundation ref. Omitting it left those
|
|
117
|
+
// ungated (an upstream theme change would not be caught) and invisible in the
|
|
118
|
+
// diff. It projects to several files (site.yml, theme.yml, head.html); `site.yml`
|
|
119
|
+
// is the label, since that is where its identity-bearing fields live.
|
|
120
|
+
if (doc?.info && typeof doc.info === 'object') cb('site.yml', doc.info, 'info')
|
|
121
|
+
walkPages(doc?.pages, 'pages')
|
|
122
|
+
walkSections(doc?.layout_sections, 'layout')
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Per-unit content hashes: `{ <path>: <sha256> }`. */
|
|
126
|
+
export function computeUnitHashes(doc, sourceLocale) {
|
|
127
|
+
const out = {}
|
|
128
|
+
for (const [path, record] of collectSiteUnits(doc, sourceLocale)) out[path] = entityContentHash(record)
|
|
129
|
+
return out
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Harvest per-item identity from a document the BACKEND produced (a pull, or a push
|
|
134
|
+
* response's `finalized[].document`): `{ <path>: <$uuid> }`.
|
|
135
|
+
*
|
|
136
|
+
* This is the map that has to be echoed back on the next push. Without it the
|
|
137
|
+
* backend cannot match our items — a uuid-less record in a `multi` section (which
|
|
138
|
+
* `pages`, `page_sections` and `layout_sections` all are) is read as new, so it is
|
|
139
|
+
* inserted and its stored counterpart is deleted. That silently replaces every
|
|
140
|
+
* page and section identity on every push, which in turn destroys the per-item
|
|
141
|
+
* handles the app holds for its own concurrency. Identity is not decoration here.
|
|
142
|
+
*/
|
|
143
|
+
export function collectUnitUuids(doc, sourceLocale) {
|
|
144
|
+
const out = {}
|
|
145
|
+
walkSiteUnits(doc, (path, record) => {
|
|
146
|
+
if (typeof record?.$uuid === 'string' && record.$uuid) out[path] = record.$uuid
|
|
147
|
+
}, sourceLocale)
|
|
148
|
+
return out
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Stamp known `$uuid`s onto a document we are about to push, so the backend matches
|
|
153
|
+
* our items instead of re-minting them.
|
|
154
|
+
*
|
|
155
|
+
* A unit the map doesn't know is left alone: that is genuinely new content on its
|
|
156
|
+
* first push, and minting is the only coherent reading. The map's absence entirely
|
|
157
|
+
* is the dangerous case — see `collectUnitUuids` — and the caller is responsible for
|
|
158
|
+
* not pushing blind (the backend also refuses an all-blank `multi` section).
|
|
159
|
+
*
|
|
160
|
+
* @returns {{ stamped: number, unknown: number }}
|
|
161
|
+
*/
|
|
162
|
+
export function stampUnitUuids(doc, pathToUuid = {}, sourceLocale) {
|
|
163
|
+
let stamped = 0
|
|
164
|
+
let unknown = 0
|
|
165
|
+
const seen = new Set()
|
|
166
|
+
const collisions = []
|
|
167
|
+
walkSiteUnits(doc, (path, record) => {
|
|
168
|
+
// Two records can resolve to ONE path when their stable ids collide — most
|
|
169
|
+
// easily `1-hero.md` and `hero.md` in the same page dir, since the numeric
|
|
170
|
+
// prefix is stripped. Stamping both with the same uuid produces a package the
|
|
171
|
+
// backend rejects outright ("a `$uuid` must be unique within the entity"), so
|
|
172
|
+
// only the first occurrence takes the identity; the rest push as new. The
|
|
173
|
+
// collision is reported because it is an authoring problem either way — the
|
|
174
|
+
// projector writes `<stableId>.md`, so a pull would collapse the two files
|
|
175
|
+
// into one.
|
|
176
|
+
if (seen.has(path)) {
|
|
177
|
+
collisions.push(path)
|
|
178
|
+
unknown++
|
|
179
|
+
return
|
|
180
|
+
}
|
|
181
|
+
seen.add(path)
|
|
182
|
+
const uuid = pathToUuid[path]
|
|
183
|
+
if (uuid) {
|
|
184
|
+
record.$uuid = uuid
|
|
185
|
+
stamped++
|
|
186
|
+
} else {
|
|
187
|
+
unknown++
|
|
188
|
+
}
|
|
189
|
+
}, sourceLocale)
|
|
190
|
+
return { stamped, unknown, collisions }
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Compare the local and remote site-content documents unit by unit, attributing
|
|
195
|
+
* each side's changes against a base in that side's own representation.
|
|
196
|
+
*
|
|
197
|
+
* @param {object} localDoc - the document we were about to push
|
|
198
|
+
* @param {object} remoteDoc - the backend's current document
|
|
199
|
+
* @param {object} [bases]
|
|
200
|
+
* @param {Object<string,string>} [bases.local] - unit hashes of OUR document as of
|
|
201
|
+
* the last sync. Enables "did we change it".
|
|
202
|
+
* @param {Object<string,string>} [bases.remote] - unit hashes of THEIR document as
|
|
203
|
+
* of the last sync. Enables "did they change it". Never compare this against
|
|
204
|
+
* a locally-derived hash.
|
|
205
|
+
* @returns {{
|
|
206
|
+
* knowsLocal: boolean, knowsRemote: boolean,
|
|
207
|
+
* changedUpstream: string[], changedLocally: string[], changedBoth: string[],
|
|
208
|
+
* changedUnattributed: string[],
|
|
209
|
+
* addedUpstream: string[], addedLocally: string[], identical: string[],
|
|
210
|
+
* }} each list holding repo-relative paths, sorted.
|
|
211
|
+
*/
|
|
212
|
+
export function diffSiteUnits(localDoc, remoteDoc, bases = {}) {
|
|
213
|
+
const localBase = bases.local || {}
|
|
214
|
+
const remoteBase = bases.remote || {}
|
|
215
|
+
const knowsLocal = Object.keys(localBase).length > 0
|
|
216
|
+
const knowsRemote = Object.keys(remoteBase).length > 0
|
|
217
|
+
|
|
218
|
+
const local = collectSiteUnits(localDoc)
|
|
219
|
+
const remote = collectSiteUnits(remoteDoc)
|
|
220
|
+
|
|
221
|
+
const out = {
|
|
222
|
+
knowsLocal, knowsRemote,
|
|
223
|
+
changedUpstream: [], changedLocally: [], changedBoth: [], changedUnattributed: [],
|
|
224
|
+
addedUpstream: [], addedLocally: [], identical: [],
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
for (const path of new Set([...local.keys(), ...remote.keys()])) {
|
|
228
|
+
const l = local.get(path)
|
|
229
|
+
const r = remote.get(path)
|
|
230
|
+
// Present on one side only. Set membership is representation-independent, so
|
|
231
|
+
// these are sound even with no bases at all — and `addedUpstream` is the
|
|
232
|
+
// dangerous one: forcing does not revert it, it deletes it.
|
|
233
|
+
if (l && !r) { out.addedLocally.push(path); continue }
|
|
234
|
+
if (r && !l) { out.addedUpstream.push(path); continue }
|
|
235
|
+
|
|
236
|
+
// Each side against its OWN base. A missing base entry is UNKNOWN, not unchanged.
|
|
237
|
+
const weChanged = knowsLocal && localBase[path] ? entityContentHash(l) !== localBase[path] : null
|
|
238
|
+
const theyChanged = knowsRemote && remoteBase[path] ? entityContentHash(r) !== remoteBase[path] : null
|
|
239
|
+
|
|
240
|
+
// An UNKNOWN side is only worth reporting when the other side doesn't already
|
|
241
|
+
// settle the question. If the remote is known to sit at its base, this unit is
|
|
242
|
+
// not contested no matter what we did to it — pushing our version is the whole
|
|
243
|
+
// point — so "differs, side unknown" would be both untrue and noise. That
|
|
244
|
+
// combination is the normal state right after a pull (which clears the local
|
|
245
|
+
// base), which is exactly when a refusal is most likely, so getting it wrong
|
|
246
|
+
// buried the one contested file under a list of perfectly fine ones.
|
|
247
|
+
if (weChanged === true && theyChanged === true) out.changedBoth.push(path)
|
|
248
|
+
else if (theyChanged === true) out.changedUpstream.push(path)
|
|
249
|
+
else if (weChanged === true) out.changedLocally.push(path)
|
|
250
|
+
else if (weChanged === null && theyChanged === null) out.changedUnattributed.push(path)
|
|
251
|
+
else out.identical.push(path)
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
for (const k of Object.keys(out)) if (Array.isArray(out[k])) out[k].sort()
|
|
255
|
+
return out
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Render a diff as the lines a CLI shows after a staleness refusal, ordered by the
|
|
260
|
+
* cost of getting each one wrong. Returns `[]` when there is nothing to say.
|
|
261
|
+
*/
|
|
262
|
+
export function describeSiteDiff(diff, { limit = 8 } = {}) {
|
|
263
|
+
const lines = []
|
|
264
|
+
// Never let a long list silently truncate into something that reads complete.
|
|
265
|
+
const list = (xs) =>
|
|
266
|
+
xs.length > limit ? `${xs.slice(0, limit).join(', ')} … and ${xs.length - limit} more` : xs.join(', ')
|
|
267
|
+
|
|
268
|
+
if (diff.addedUpstream.length) lines.push(`Added upstream — forcing DELETES these: ${list(diff.addedUpstream)}`)
|
|
269
|
+
if (diff.changedBoth.length) lines.push(`Changed on both sides: ${list(diff.changedBoth)}`)
|
|
270
|
+
if (diff.changedUpstream.length) lines.push(`Changed upstream — forcing discards these: ${list(diff.changedUpstream)}`)
|
|
271
|
+
if (diff.changedLocally.length) lines.push(`Changed by you — pulling discards these: ${list(diff.changedLocally)}`)
|
|
272
|
+
if (diff.changedUnattributed.length) lines.push(`Differs, side unknown: ${list(diff.changedUnattributed)}`)
|
|
273
|
+
if (diff.addedLocally.length) lines.push(`Added by you (not upstream yet): ${list(diff.addedLocally)}`)
|
|
274
|
+
|
|
275
|
+
// The good news is worth saying — but only when BOTH sides are actually known.
|
|
276
|
+
// Without the local base we cannot see our own edits, so "nothing was changed on
|
|
277
|
+
// both sides" would be a claim about evidence we don't have: the user could pull
|
|
278
|
+
// on the strength of it and lose the very edit they were trying to push. A
|
|
279
|
+
// reassurance that can't be backed is worse than no reassurance.
|
|
280
|
+
if (
|
|
281
|
+
lines.length && diff.knowsLocal && diff.knowsRemote &&
|
|
282
|
+
!diff.changedBoth.length && !diff.changedUnattributed.length
|
|
283
|
+
) {
|
|
284
|
+
lines.push('No unit was changed on both sides — pulling should merge cleanly.')
|
|
285
|
+
}
|
|
286
|
+
// Say which half of the attribution is missing rather than under-reporting silently.
|
|
287
|
+
if (lines.length && !diff.knowsRemote) {
|
|
288
|
+
lines.push("No record of the backend's last state, so upstream edits to existing units are not listed.")
|
|
289
|
+
}
|
|
290
|
+
if (lines.length && !diff.knowsLocal) {
|
|
291
|
+
lines.push('No record of your last sync, so your own edits are not listed.')
|
|
292
|
+
}
|
|
293
|
+
return lines
|
|
294
|
+
}
|