@uniweb/build 0.15.14 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -6
- package/src/i18n/collections.js +23 -52
- package/src/i18n/data-strings.js +114 -0
- package/src/i18n/extract.js +150 -52
- package/src/i18n/merge.js +34 -0
- package/src/index.js +1 -0
- package/src/prerender.js +2 -2
- package/src/site/build-site-data.js +23 -4
- package/src/site/collection-processor.js +85 -7
- package/src/site/content-collector.js +4 -3
- package/src/site/data-ball.js +2 -1
- package/src/site/data-fetcher.js +2 -2
- package/src/site/plugin.js +112 -15
- package/src/uwx/site.js +2 -2
- package/src/validate-data.js +167 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@uniweb/build",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.1",
|
|
4
4
|
"description": "Build tooling for the Uniweb Component Web Platform",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -59,13 +59,14 @@
|
|
|
59
59
|
"js-yaml": "^4.1.0",
|
|
60
60
|
"sharp": "^0.35.3",
|
|
61
61
|
"yaml": "^2.5.0",
|
|
62
|
-
"@uniweb/projections": "0.1
|
|
62
|
+
"@uniweb/projections": "0.2.1",
|
|
63
63
|
"@uniweb/theming": "0.1.15",
|
|
64
|
-
"@uniweb/content-writer": "0.
|
|
64
|
+
"@uniweb/content-writer": "0.3.1"
|
|
65
65
|
},
|
|
66
66
|
"optionalDependencies": {
|
|
67
|
-
"@uniweb/
|
|
68
|
-
"@uniweb/
|
|
67
|
+
"@uniweb/content-reader": "1.2.0",
|
|
68
|
+
"@uniweb/semantic-parser": "1.2.0",
|
|
69
|
+
"@uniweb/runtime": "0.9.0",
|
|
69
70
|
"@uniweb/schemas": "0.2.4"
|
|
70
71
|
},
|
|
71
72
|
"peerDependencies": {
|
|
@@ -75,7 +76,7 @@
|
|
|
75
76
|
"@tailwindcss/vite": "^4.0.0",
|
|
76
77
|
"@vitejs/plugin-react": "^4.0.0 || ^5.0.0",
|
|
77
78
|
"vite-plugin-svgr": "^4.0.0",
|
|
78
|
-
"@uniweb/core": "0.
|
|
79
|
+
"@uniweb/core": "0.8.0"
|
|
79
80
|
},
|
|
80
81
|
"peerDependenciesMeta": {
|
|
81
82
|
"vite": {
|
package/src/i18n/collections.js
CHANGED
|
@@ -14,51 +14,23 @@ import { readFile, writeFile, readdir, mkdir } from 'fs/promises'
|
|
|
14
14
|
import { existsSync } from 'fs'
|
|
15
15
|
import { join } from 'path'
|
|
16
16
|
import { pathToFileURL } from 'url'
|
|
17
|
+
import { DATA_DIR } from '@uniweb/core'
|
|
17
18
|
import { computeHash } from './hash.js'
|
|
18
19
|
import { loadFreeformCollectionItem } from './freeform.js'
|
|
20
|
+
// The heuristic judgement about which strings inside structured data are prose.
|
|
21
|
+
// It lives in its own module because the page lane needs exactly the same
|
|
22
|
+
// answer for a tagged data block's payload — a `label` is prose and an `href`
|
|
23
|
+
// is not, wherever the value came from. Moved rather than copied: two tuned
|
|
24
|
+
// denylists would drift, and drift here is silent.
|
|
25
|
+
import {
|
|
26
|
+
NON_TRANSLATABLE_TYPES,
|
|
27
|
+
HEURISTIC_SKIP_FIELDS,
|
|
28
|
+
MAX_HEURISTIC_DEPTH,
|
|
29
|
+
isStructuralString,
|
|
30
|
+
} from './data-strings.js'
|
|
19
31
|
|
|
20
32
|
export const COLLECTIONS_DIR = 'collections'
|
|
21
33
|
|
|
22
|
-
// ---------------------------------------------------------------------------
|
|
23
|
-
// Constants
|
|
24
|
-
// ---------------------------------------------------------------------------
|
|
25
|
-
|
|
26
|
-
/** Types that are never translatable regardless of schema */
|
|
27
|
-
const NON_TRANSLATABLE_TYPES = new Set([
|
|
28
|
-
'number', 'boolean', 'date', 'datetime', 'url', 'email', 'image'
|
|
29
|
-
])
|
|
30
|
-
|
|
31
|
-
/** Field names skipped by the heuristic extractor (structural, not human-readable) */
|
|
32
|
-
const HEURISTIC_SKIP_FIELDS = new Set([
|
|
33
|
-
'slug', 'id', 'type', 'status', 'href', 'url', 'src', 'icon',
|
|
34
|
-
'target', 'email', 'phone', 'orcid', 'doi', 'arxiv', 'isbn',
|
|
35
|
-
'pmid', 'bibtex', 'pdf', 'code', 'data', 'slides', 'video',
|
|
36
|
-
'repository', 'caseStudy', 'website', 'avatar', 'image',
|
|
37
|
-
'thumbnail', 'currency', 'order', 'hidden', 'current',
|
|
38
|
-
'featured', 'published', 'allDay', 'remote', 'hybrid',
|
|
39
|
-
'noindex', 'corresponding', 'required', 'virtual',
|
|
40
|
-
'lastModified', 'date', 'updated', 'posted', 'submitted',
|
|
41
|
-
'accepted', 'startDate', 'endDate', 'deadline',
|
|
42
|
-
'readTime', 'citations', 'capacity', 'volume', 'issue', 'pages',
|
|
43
|
-
'time', 'timezone',
|
|
44
|
-
])
|
|
45
|
-
|
|
46
|
-
/** String patterns that indicate non-translatable values */
|
|
47
|
-
const HEURISTIC_SKIP_PATTERNS = [
|
|
48
|
-
/^https?:\/\//, // URLs
|
|
49
|
-
/^mailto:/, // mailto links
|
|
50
|
-
/^[^\s@]+@[^\s@]+\.[^\s@]+$/, // email addresses
|
|
51
|
-
/^\d{4}-\d{2}-\d{2}/, // ISO dates
|
|
52
|
-
/^#[0-9a-fA-F]{3,8}$/, // hex colors
|
|
53
|
-
/^[\w./\\-]+\.\w{2,4}$/, // file paths (e.g., ./logo.svg, /img/hero.jpg)
|
|
54
|
-
/^[A-Z]{3}$/, // currency codes (USD, EUR)
|
|
55
|
-
/^\d+(\.\d+)?$/, // plain numbers as strings
|
|
56
|
-
/^\d{1,2}:\d{2}(:\d{2})?$/, // times (09:00, 14:30:00)
|
|
57
|
-
]
|
|
58
|
-
|
|
59
|
-
/** Max recursion depth for heuristic extraction */
|
|
60
|
-
const MAX_HEURISTIC_DEPTH = 5
|
|
61
|
-
|
|
62
34
|
// ---------------------------------------------------------------------------
|
|
63
35
|
// Schema resolution
|
|
64
36
|
// ---------------------------------------------------------------------------
|
|
@@ -85,8 +57,14 @@ async function resolveSchema(collectionName, siteRoot) {
|
|
|
85
57
|
|
|
86
58
|
let schema = null
|
|
87
59
|
|
|
88
|
-
// 1. Companion schema file
|
|
89
|
-
|
|
60
|
+
// 1. Companion schema file, beside the collection it describes.
|
|
61
|
+
//
|
|
62
|
+
// This used to be looked up in `public/<DATA_DIR>/` — next to the compiled
|
|
63
|
+
// output rather than the source. That directory is the build's, and asking
|
|
64
|
+
// an author to write into it was the one remaining place the framework
|
|
65
|
+
// contradicted its own rule that `collections/` is the only way to provide
|
|
66
|
+
// structured data. The schema describes the source, so it lives with it.
|
|
67
|
+
const companionPath = join(siteRoot, 'collections', `${collectionName}.schema.js`)
|
|
90
68
|
if (existsSync(companionPath)) {
|
|
91
69
|
try {
|
|
92
70
|
const mod = await import(pathToFileURL(companionPath).href)
|
|
@@ -290,13 +268,6 @@ function extractFromItemHeuristic(data, pathPrefix, context, units, depth) {
|
|
|
290
268
|
}
|
|
291
269
|
}
|
|
292
270
|
|
|
293
|
-
/**
|
|
294
|
-
* Check if a string value looks structural (not human-readable).
|
|
295
|
-
*/
|
|
296
|
-
function isStructuralString(value) {
|
|
297
|
-
return HEURISTIC_SKIP_PATTERNS.some(pattern => pattern.test(value))
|
|
298
|
-
}
|
|
299
|
-
|
|
300
271
|
// ---------------------------------------------------------------------------
|
|
301
272
|
// Schema-guided translation
|
|
302
273
|
// ---------------------------------------------------------------------------
|
|
@@ -412,7 +383,7 @@ function translateItemHeuristic(data, context, translations, depth) {
|
|
|
412
383
|
* @returns {Promise<Object>} Manifest with translation units
|
|
413
384
|
*/
|
|
414
385
|
export async function extractCollectionContent(siteRoot, options = {}) {
|
|
415
|
-
const dataDir = join(siteRoot, 'public',
|
|
386
|
+
const dataDir = join(siteRoot, 'public', DATA_DIR)
|
|
416
387
|
|
|
417
388
|
if (!existsSync(dataDir)) {
|
|
418
389
|
return { version: '1.0', units: {} }
|
|
@@ -577,7 +548,7 @@ export async function buildLocalizedCollections(siteRoot, options = {}) {
|
|
|
577
548
|
freeformEnabled = true
|
|
578
549
|
} = options
|
|
579
550
|
|
|
580
|
-
const dataDir = join(siteRoot, 'public',
|
|
551
|
+
const dataDir = join(siteRoot, 'public', DATA_DIR)
|
|
581
552
|
|
|
582
553
|
if (!existsSync(dataDir)) {
|
|
583
554
|
return {}
|
|
@@ -615,7 +586,7 @@ export async function buildLocalizedCollections(siteRoot, options = {}) {
|
|
|
615
586
|
const hasFreeform = freeformEnabled && existsSync(freeformDir)
|
|
616
587
|
|
|
617
588
|
// Create locale data directory
|
|
618
|
-
const localeDataDir = join(outputDir, locale,
|
|
589
|
+
const localeDataDir = join(outputDir, locale, DATA_DIR)
|
|
619
590
|
await mkdir(localeDataDir, { recursive: true })
|
|
620
591
|
|
|
621
592
|
outputs[locale] = {}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which strings inside STRUCTURED DATA are human-readable prose.
|
|
3
|
+
*
|
|
4
|
+
* A tagged data block (```yaml:nav, ```json:pricing) carries an arbitrary shape,
|
|
5
|
+
* so there is no element to key a translation unit to — only a tree of values,
|
|
6
|
+
* most of which are machinery. A nav's `label` is prose; its `href` and `icon`
|
|
7
|
+
* are not. Guessing wrong in one direction leaves a site half-translated; in the
|
|
8
|
+
* other it rewrites a URL into another language and breaks the link.
|
|
9
|
+
*
|
|
10
|
+
* The judgement was already made and tuned for collections, which have exactly
|
|
11
|
+
* this problem. This module is that judgement, moved somewhere both lanes can
|
|
12
|
+
* reach rather than copied — the copy is how the two would drift, and the whole
|
|
13
|
+
* point is that a `label` means the same thing in a collection record and in a
|
|
14
|
+
* data block.
|
|
15
|
+
*
|
|
16
|
+
* An ALLOWLIST of shapes is impossible here (the data is author-defined), so
|
|
17
|
+
* this is necessarily a denylist, and it is deliberately conservative: a missed
|
|
18
|
+
* skip shows up as a translatable string an author can leave alone, while a
|
|
19
|
+
* missed field shows up as untranslated content nobody notices.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** Types that are never translatable regardless of schema. */
|
|
23
|
+
export const NON_TRANSLATABLE_TYPES = new Set([
|
|
24
|
+
'number', 'boolean', 'date', 'datetime', 'url', 'email', 'image'
|
|
25
|
+
])
|
|
26
|
+
|
|
27
|
+
/** Field names skipped by the heuristic extractor (structural, not human-readable). */
|
|
28
|
+
export const HEURISTIC_SKIP_FIELDS = new Set([
|
|
29
|
+
'slug', 'id', 'type', 'status', 'href', 'url', 'src', 'icon',
|
|
30
|
+
'target', 'email', 'phone', 'orcid', 'doi', 'arxiv', 'isbn',
|
|
31
|
+
'pmid', 'bibtex', 'pdf', 'code', 'data', 'slides', 'video',
|
|
32
|
+
'repository', 'caseStudy', 'website', 'avatar', 'image',
|
|
33
|
+
'thumbnail', 'currency', 'order', 'hidden', 'current',
|
|
34
|
+
'featured', 'published', 'allDay', 'remote', 'hybrid',
|
|
35
|
+
'noindex', 'corresponding', 'required', 'virtual',
|
|
36
|
+
'lastModified', 'date', 'updated', 'posted', 'submitted',
|
|
37
|
+
'accepted', 'startDate', 'endDate', 'deadline',
|
|
38
|
+
'readTime', 'citations', 'capacity', 'volume', 'issue', 'pages',
|
|
39
|
+
'time', 'timezone',
|
|
40
|
+
])
|
|
41
|
+
|
|
42
|
+
/** String patterns that indicate non-translatable values. */
|
|
43
|
+
export const HEURISTIC_SKIP_PATTERNS = [
|
|
44
|
+
/^https?:\/\//, // URLs
|
|
45
|
+
/^mailto:/, // mailto links
|
|
46
|
+
/^[^\s@]+@[^\s@]+\.[^\s@]+$/, // email addresses
|
|
47
|
+
/^\d{4}-\d{2}-\d{2}/, // ISO dates
|
|
48
|
+
/^#[0-9a-fA-F]{3,8}$/, // hex colors
|
|
49
|
+
/^[\w./\\-]+\.\w{2,4}$/, // file paths (e.g., ./logo.svg, /img/hero.jpg)
|
|
50
|
+
/^[A-Z]{3}$/, // currency codes (USD, EUR)
|
|
51
|
+
/^\d+(\.\d+)?$/, // plain numbers as strings
|
|
52
|
+
/^\d{1,2}:\d{2}(:\d{2})?$/, // times (09:00, 14:30:00)
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
/** Max recursion depth for heuristic extraction. */
|
|
56
|
+
export const MAX_HEURISTIC_DEPTH = 5
|
|
57
|
+
|
|
58
|
+
/** Whether a string value looks structural rather than human-readable. */
|
|
59
|
+
export function isStructuralString(value) {
|
|
60
|
+
return HEURISTIC_SKIP_PATTERNS.some((pattern) => pattern.test(value))
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Walk structured data and hand every translatable-looking STRING to `visit`.
|
|
65
|
+
*
|
|
66
|
+
* One walker for both directions, the same discipline the document walk uses:
|
|
67
|
+
* extraction returns nothing and collects, application returns a replacement and
|
|
68
|
+
* the walker writes it back. Two copies of a rule this fiddly is how the
|
|
69
|
+
* extracted set and the applied set end up different, which fails as content
|
|
70
|
+
* that appears in the manifest, gets translated by a human, and still renders in
|
|
71
|
+
* the source language.
|
|
72
|
+
*
|
|
73
|
+
* Mutates `data` in place when `visit` returns a different string, so callers
|
|
74
|
+
* that must not mutate should pass a clone.
|
|
75
|
+
*
|
|
76
|
+
* @param {*} data - any parsed YAML/JSON value
|
|
77
|
+
* @param {(value: string, path: string) => string|void} visit
|
|
78
|
+
* @param {string} [path] - dotted path, for the unit's `field` metadata
|
|
79
|
+
* @param {number} [depth]
|
|
80
|
+
*/
|
|
81
|
+
export function visitDataStrings(data, visit, path = '', depth = 0) {
|
|
82
|
+
if (!data || typeof data !== 'object' || depth > MAX_HEURISTIC_DEPTH) return
|
|
83
|
+
|
|
84
|
+
const isArray = Array.isArray(data)
|
|
85
|
+
const keys = isArray ? data.map((_, i) => i) : Object.keys(data)
|
|
86
|
+
|
|
87
|
+
for (const key of keys) {
|
|
88
|
+
const value = data[key]
|
|
89
|
+
if (value === undefined || value === null) continue
|
|
90
|
+
|
|
91
|
+
const fieldPath = isArray
|
|
92
|
+
? `${path}[${key}]`
|
|
93
|
+
: path
|
|
94
|
+
? `${path}.${key}`
|
|
95
|
+
: String(key)
|
|
96
|
+
|
|
97
|
+
if (typeof value === 'string') {
|
|
98
|
+
// A skip-list entry applies to a NAMED field. Inside an array the key is
|
|
99
|
+
// an index and carries no meaning, so only the value patterns apply —
|
|
100
|
+
// otherwise a list of prose strings would be skipped by position.
|
|
101
|
+
if (!isArray && HEURISTIC_SKIP_FIELDS.has(key)) continue
|
|
102
|
+
if (isStructuralString(value)) continue
|
|
103
|
+
if (!value.trim()) continue
|
|
104
|
+
|
|
105
|
+
const replacement = visit(value, fieldPath)
|
|
106
|
+
if (typeof replacement === 'string' && replacement !== value) {
|
|
107
|
+
data[key] = replacement
|
|
108
|
+
}
|
|
109
|
+
} else if (typeof value === 'object') {
|
|
110
|
+
visitDataStrings(value, visit, fieldPath, depth + 1)
|
|
111
|
+
}
|
|
112
|
+
// numbers and booleans are never prose
|
|
113
|
+
}
|
|
114
|
+
}
|
package/src/i18n/extract.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { resolveDefaultLocale } from '@uniweb/core'
|
|
9
9
|
import { computeHash, stripInlineTags } from './hash.js'
|
|
10
|
+
import { visitDataStrings } from './data-strings.js'
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Extract all translatable units from site content
|
|
@@ -130,6 +131,7 @@ function extractFromSection(section, pageRoute, units) {
|
|
|
130
131
|
|
|
131
132
|
if (section.content?.type === 'doc') {
|
|
132
133
|
extractFromProseMirrorDoc(section.content, context, units)
|
|
134
|
+
extractFromDataBlocks(section.content, context, units)
|
|
133
135
|
}
|
|
134
136
|
|
|
135
137
|
// Recursively process subsections
|
|
@@ -138,6 +140,61 @@ function extractFromSection(section, pageRoute, units) {
|
|
|
138
140
|
}
|
|
139
141
|
}
|
|
140
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Extract the human-readable strings inside a section's TAGGED DATA BLOCKS.
|
|
145
|
+
*
|
|
146
|
+
* A ```yaml:nav block's link labels, a ```yaml:pricing block's plan names — the
|
|
147
|
+
* strings an author most expects to see translated, and which stayed in the
|
|
148
|
+
* source language on every multilingual site until 2026-07-30 because nothing
|
|
149
|
+
* in this file mentioned `dataBlock`.
|
|
150
|
+
*
|
|
151
|
+
* Which strings count is not this file's judgement to make: it is the same
|
|
152
|
+
* question collections answer for a record's fields, so the same detection runs
|
|
153
|
+
* here (`data-strings.js`).
|
|
154
|
+
*
|
|
155
|
+
* DELIBERATELY SEPARATE from `extractFromProseMirrorDoc`, and not folded into
|
|
156
|
+
* it. That function is also `extractUnitsFromDoc`, which publishes the
|
|
157
|
+
* structural-keying contract pinned by tests/i18n/structural-keying-vectors.json
|
|
158
|
+
* — units keyed by a BLOCK ELEMENT's text. A data string is not a block
|
|
159
|
+
* element's text and has no place in that map. The consequence is real rather
|
|
160
|
+
* than tidy: `deriveStructuralMap` recovers a pulled translation by walking
|
|
161
|
+
* block elements, so a data string translated on the sync wire would be
|
|
162
|
+
* invisible to it — neither captured in the map nor counted as divergence — and
|
|
163
|
+
* would be silently lost on the next pull. So this lane is the BUILD lane only
|
|
164
|
+
* (the manifest, and `dist/{locale}/`), which is where the reported bug lives.
|
|
165
|
+
* Carrying data-block translations across the sync wire needs a representation
|
|
166
|
+
* that contract does not have yet.
|
|
167
|
+
*/
|
|
168
|
+
function extractFromDataBlocks(doc, context, units) {
|
|
169
|
+
for (const node of dataBlockNodes(doc)) {
|
|
170
|
+
const { tag, data } = node.attrs || {}
|
|
171
|
+
if (!tag || !data || typeof data !== 'object') continue
|
|
172
|
+
|
|
173
|
+
visitDataStrings(data, (value, fieldPath) => {
|
|
174
|
+
addUnit(units, value, `data.${tag}.${fieldPath}`, context)
|
|
175
|
+
})
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Every tagged data block in a doc, including any nested inside a container.
|
|
181
|
+
*/
|
|
182
|
+
function dataBlockNodes(doc) {
|
|
183
|
+
const out = []
|
|
184
|
+
const walk = (nodes) => {
|
|
185
|
+
for (const node of nodes || []) {
|
|
186
|
+
if (!node) continue
|
|
187
|
+
if (node.type === 'dataBlock') out.push(node)
|
|
188
|
+
else if (CONTAINER_BLOCKS.has(node.type)) walk(node.content)
|
|
189
|
+
else if (node.type === 'bulletList' || node.type === 'orderedList') {
|
|
190
|
+
for (const listItem of node.content || []) walk(listItem.content)
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
walk(doc?.content)
|
|
195
|
+
return out
|
|
196
|
+
}
|
|
197
|
+
|
|
141
198
|
/**
|
|
142
199
|
* Extract translatable strings from ProseMirror document
|
|
143
200
|
* @param {Object} doc - ProseMirror document
|
|
@@ -165,28 +222,103 @@ function extractFromProseMirrorDoc(doc, context, units) {
|
|
|
165
222
|
let headingIndex = { h1: 0, h2: 0, h3: 0, h4: 0 }
|
|
166
223
|
let paragraphIndex = 0
|
|
167
224
|
|
|
168
|
-
|
|
225
|
+
visitTranslatableBlocks(doc.content, (node, listIndex) => {
|
|
169
226
|
if (node.type === 'heading') {
|
|
170
227
|
const text = elementText(node)
|
|
171
|
-
if (!text)
|
|
228
|
+
if (!text) return
|
|
172
229
|
|
|
173
230
|
const level = node.attrs?.level || 1
|
|
174
231
|
const field = getHeadingField(level, headingIndex)
|
|
175
232
|
headingIndex[`h${level}`]++
|
|
176
233
|
|
|
177
234
|
addUnit(units, text, field, context)
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
235
|
+
return
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Whole-element keying: ONE unit per paragraph, with link text kept INLINE
|
|
239
|
+
// (not split into a separate link.label unit). The conformance gate is
|
|
240
|
+
// tests/i18n/structural-keying-vectors.json.
|
|
241
|
+
const text = elementText(node)
|
|
242
|
+
if (!text) return
|
|
243
|
+
|
|
244
|
+
if (listIndex !== null) {
|
|
245
|
+
addUnit(units, text, `list.${listIndex}`, context)
|
|
246
|
+
return
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const field = paragraphIndex === 0 ? 'paragraph' : `paragraph.${paragraphIndex}`
|
|
250
|
+
addUnit(units, text, field, context)
|
|
251
|
+
paragraphIndex++
|
|
252
|
+
})
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Block node types whose children carry translatable prose, so the walk has to
|
|
257
|
+
* descend into them.
|
|
258
|
+
*
|
|
259
|
+
* WHY THIS EXISTS. The walk used to handle four node types and recurse into
|
|
260
|
+
* nothing, so a string inside ANY container was invisible to translation: a
|
|
261
|
+
* callout's body, a table cell, a blockquote. On a multilingual site those
|
|
262
|
+
* silently stayed in the source language, and nothing reported it. That
|
|
263
|
+
* predates concept blocks — adding one more prose container without fixing the
|
|
264
|
+
* walk would have inherited the hole in the place it hurts most, since a
|
|
265
|
+
* concept block is prose, which is the thing translation exists for.
|
|
266
|
+
*
|
|
267
|
+
* An explicit set rather than "recurse into anything with content", because the
|
|
268
|
+
* denylist version is the dangerous one: `codeBlock` also has content, and
|
|
269
|
+
* extracting source code as translatable prose would be worse than missing it.
|
|
270
|
+
*
|
|
271
|
+
* KEEP IN SYNC with the editor's own container list. It maintains a second
|
|
272
|
+
* implementation of this walk over the same documents and already recursed
|
|
273
|
+
* containers when this one did not, so the two disagreed — the shared vectors
|
|
274
|
+
* in tests/i18n/structural-keying-vectors.json are the only thing pinning them
|
|
275
|
+
* together, and they are kept in step by hand.
|
|
276
|
+
*/
|
|
277
|
+
export const CONTAINER_BLOCKS = new Set([
|
|
278
|
+
'concept_block', // ```md:<tag> — a concept's body is authored prose
|
|
279
|
+
'inset_block', // ```@Component{params} — a callout's body is authored prose
|
|
280
|
+
'blockquote',
|
|
281
|
+
'table',
|
|
282
|
+
'tableRow',
|
|
283
|
+
'tableCell',
|
|
284
|
+
])
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Walk a content array and hand every translatable BLOCK element to `visit`, in
|
|
288
|
+
* document order, descending into containers.
|
|
289
|
+
*
|
|
290
|
+
* ONE walker with two consumers, deliberately. Extraction (into the manifest)
|
|
291
|
+
* and resolution (applying a translation) used to be two separate walks over
|
|
292
|
+
* the same four node types. Two copies of one rule is the shape that fails
|
|
293
|
+
* halfway: teach only the extractor about a container and its strings reach the
|
|
294
|
+
* manifest but are never applied; teach only the resolver and there is nothing
|
|
295
|
+
* to apply. Neither half fails loudly. They cannot drift now because there is
|
|
296
|
+
* only one of them.
|
|
297
|
+
*
|
|
298
|
+
* @param {Array} nodes - a content array
|
|
299
|
+
* @param {(node: Object, listIndex: number|null) => void} visit
|
|
300
|
+
*/
|
|
301
|
+
function visitTranslatableBlocks(nodes, visit) {
|
|
302
|
+
for (const node of nodes || []) {
|
|
303
|
+
if (!node) continue
|
|
304
|
+
|
|
305
|
+
if (node.type === 'heading' || node.type === 'paragraph') {
|
|
306
|
+
visit(node, null)
|
|
188
307
|
} else if (node.type === 'bulletList' || node.type === 'orderedList') {
|
|
189
|
-
|
|
308
|
+
// A list item's index is part of its unit's field name, so lists keep
|
|
309
|
+
// their own branch rather than folding into the container recursion.
|
|
310
|
+
;(node.content || []).forEach((listItem, index) => {
|
|
311
|
+
if (listItem.type !== 'listItem') return
|
|
312
|
+
for (const child of listItem.content || []) {
|
|
313
|
+
if (child.type === 'paragraph') {
|
|
314
|
+
visit(child, index)
|
|
315
|
+
} else if (CONTAINER_BLOCKS.has(child.type)) {
|
|
316
|
+
visitTranslatableBlocks([child], visit)
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
})
|
|
320
|
+
} else if (CONTAINER_BLOCKS.has(node.type)) {
|
|
321
|
+
visitTranslatableBlocks(node.content, visit)
|
|
190
322
|
}
|
|
191
323
|
}
|
|
192
324
|
}
|
|
@@ -203,27 +335,6 @@ function getHeadingField(level, index) {
|
|
|
203
335
|
return `heading.h${level}.${index[`h${level}`]}`
|
|
204
336
|
}
|
|
205
337
|
|
|
206
|
-
/**
|
|
207
|
-
* Extract from list items — one whole-element unit per list item (link text
|
|
208
|
-
* stays inline, same rule as paragraphs; vectors G and H).
|
|
209
|
-
*/
|
|
210
|
-
function extractFromList(listNode, context, units) {
|
|
211
|
-
if (!listNode.content) return
|
|
212
|
-
|
|
213
|
-
listNode.content.forEach((listItem, index) => {
|
|
214
|
-
if (listItem.type === 'listItem' && listItem.content) {
|
|
215
|
-
for (const child of listItem.content) {
|
|
216
|
-
if (child.type === 'paragraph') {
|
|
217
|
-
const text = elementText(child)
|
|
218
|
-
if (text) {
|
|
219
|
-
addUnit(units, text, `list.${index}`, context)
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
})
|
|
225
|
-
}
|
|
226
|
-
|
|
227
338
|
/**
|
|
228
339
|
* A block element's cleaned source text — the WHOLE-ELEMENT translation key.
|
|
229
340
|
* ALL inline marks (bold, italic, link, span, …) flatten into the text: link
|
|
@@ -259,27 +370,14 @@ function collectInlineText(node) {
|
|
|
259
370
|
|
|
260
371
|
/**
|
|
261
372
|
* The translatable block elements of a content doc, in document order, with the
|
|
262
|
-
* SAME coverage as extraction above
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
* them via elementText.
|
|
373
|
+
* SAME coverage as extraction above — because it is the same walk. Shared by the
|
|
374
|
+
* merge resolver (push) and the pull-side structural-map derivation so all paths
|
|
375
|
+
* walk identically and keys never drift. Returns the element nodes themselves —
|
|
376
|
+
* callers read `.type`/`.content` and key them via elementText.
|
|
267
377
|
*/
|
|
268
378
|
export function blockElements(doc) {
|
|
269
379
|
const out = []
|
|
270
|
-
|
|
271
|
-
if (node.type === 'heading' || node.type === 'paragraph') {
|
|
272
|
-
out.push(node)
|
|
273
|
-
} else if (node.type === 'bulletList' || node.type === 'orderedList') {
|
|
274
|
-
for (const listItem of node.content || []) {
|
|
275
|
-
if (listItem.type === 'listItem' && listItem.content) {
|
|
276
|
-
for (const child of listItem.content) {
|
|
277
|
-
if (child.type === 'paragraph') out.push(child)
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
}
|
|
380
|
+
visitTranslatableBlocks(doc?.content, (node) => out.push(node))
|
|
283
381
|
return out
|
|
284
382
|
}
|
|
285
383
|
|
package/src/i18n/merge.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import { computeHash } from './hash.js'
|
|
16
16
|
import { loadFreeformTranslation } from './freeform.js'
|
|
17
17
|
import { elementText, blockElements } from './extract.js'
|
|
18
|
+
import { visitDataStrings } from './data-strings.js'
|
|
18
19
|
|
|
19
20
|
// Inline-markdown → ProseMirror inline fragment, for resolving a whole-element
|
|
20
21
|
// translation VALUE (which carries marks/links/icons as inline markdown). Same
|
|
@@ -227,6 +228,7 @@ function translateSectionSync(section, pageRoute, translations, fallbackToSource
|
|
|
227
228
|
|
|
228
229
|
if (section.content?.type === 'doc') {
|
|
229
230
|
translateProseMirrorDoc(section.content, context, translations, fallbackToSource)
|
|
231
|
+
translateDataBlocks(section.content, context, translations, fallbackToSource)
|
|
230
232
|
}
|
|
231
233
|
|
|
232
234
|
// Recursively translate subsections
|
|
@@ -259,6 +261,7 @@ async function translateSectionAsync(section, page, translations, options) {
|
|
|
259
261
|
// Fall back to hash-based translation
|
|
260
262
|
if (section.content?.type === 'doc') {
|
|
261
263
|
translateProseMirrorDoc(section.content, context, translations, fallbackToSource)
|
|
264
|
+
translateDataBlocks(section.content, context, translations, fallbackToSource)
|
|
262
265
|
}
|
|
263
266
|
}
|
|
264
267
|
|
|
@@ -302,6 +305,37 @@ function applyElementTranslation(node, context, translations, fallbackToSource)
|
|
|
302
305
|
return false
|
|
303
306
|
}
|
|
304
307
|
|
|
308
|
+
/**
|
|
309
|
+
* Translate the human-readable strings inside a section's TAGGED DATA BLOCKS,
|
|
310
|
+
* in place. The other half of the fix in `extract.js` — a manifest entry nobody
|
|
311
|
+
* applies is worse than no entry, because a translator has already done the work.
|
|
312
|
+
*
|
|
313
|
+
* Called from the two BUILD-lane section walks only, deliberately, and NOT from
|
|
314
|
+
* `resolveDocForLocale`. See `extractFromDataBlocks` for why: the sync wire's
|
|
315
|
+
* structural map is derived by walking block elements, so a translated data
|
|
316
|
+
* payload there would be neither captured nor flagged as divergent, and would be
|
|
317
|
+
* dropped on the next pull. Losing a translation silently is worse than not
|
|
318
|
+
* carrying one yet.
|
|
319
|
+
*/
|
|
320
|
+
function translateDataBlocks(doc, context, translations, fallbackToSource) {
|
|
321
|
+
const walk = (nodes) => {
|
|
322
|
+
for (const node of nodes || []) {
|
|
323
|
+
if (!node) continue
|
|
324
|
+
if (node.type === 'dataBlock') {
|
|
325
|
+
const data = node.attrs?.data
|
|
326
|
+
if (data && typeof data === 'object') {
|
|
327
|
+
visitDataStrings(data, (value) =>
|
|
328
|
+
lookupTranslation(value, context, translations, fallbackToSource)
|
|
329
|
+
)
|
|
330
|
+
}
|
|
331
|
+
} else if (Array.isArray(node.content)) {
|
|
332
|
+
walk(node.content)
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
walk(doc?.content)
|
|
337
|
+
}
|
|
338
|
+
|
|
305
339
|
/**
|
|
306
340
|
* Resolve ONE ProseMirror content doc for a single target locale: a deep clone of
|
|
307
341
|
* the source doc with each whole-element translated (inline content replaced from
|
package/src/index.js
CHANGED
package/src/prerender.js
CHANGED
|
@@ -11,7 +11,7 @@ import { readFile, writeFile, mkdir } from 'node:fs/promises'
|
|
|
11
11
|
import { existsSync, readdirSync, statSync } from 'node:fs'
|
|
12
12
|
import { join, dirname, resolve } from 'node:path'
|
|
13
13
|
import { pathToFileURL } from 'node:url'
|
|
14
|
-
import { resolveDefaultLocale } from '@uniweb/core'
|
|
14
|
+
import { resolveDefaultLocale, isDataUrl } from '@uniweb/core'
|
|
15
15
|
import { executeFetch, mergeDataIntoContent } from './site/data-fetcher.js'
|
|
16
16
|
import { shouldSplitContent } from './site/split-content.js'
|
|
17
17
|
import { FONT_LINKS_MARKER } from './site/head-markers.js'
|
|
@@ -81,7 +81,7 @@ async function executeAllFetches(siteContent, siteDir, onProgress, localeInfo) {
|
|
|
81
81
|
localeInfo.distDir
|
|
82
82
|
|
|
83
83
|
function localizeFetch(config) {
|
|
84
|
-
if (!isNonDefaultLocale || !config.path
|
|
84
|
+
if (!isNonDefaultLocale || !isDataUrl(config.path)) return config
|
|
85
85
|
return { ...config, path: `/${localeInfo.locale}${config.path}` }
|
|
86
86
|
}
|
|
87
87
|
|
|
@@ -21,7 +21,7 @@ import { writeFile, readFile, mkdir, cp } from 'node:fs/promises'
|
|
|
21
21
|
import { existsSync } from 'node:fs'
|
|
22
22
|
import { join, resolve, dirname } from 'node:path'
|
|
23
23
|
|
|
24
|
-
import { resolveDefaultLocale } from '@uniweb/core'
|
|
24
|
+
import { resolveDefaultLocale, DATA_DIR } from '@uniweb/core'
|
|
25
25
|
import { collectSiteContent } from './content-collector.js'
|
|
26
26
|
import { processCollections, writeCollectionFiles } from './collection-processor.js'
|
|
27
27
|
import { processAssets, rewriteSiteContentPaths } from './asset-processor.js'
|
|
@@ -35,7 +35,9 @@ import {
|
|
|
35
35
|
renderPageMarkdown,
|
|
36
36
|
resolveAgentsConfig,
|
|
37
37
|
selectIndexablePages,
|
|
38
|
+
selectIndexBranches,
|
|
38
39
|
pageMarkdownFilename,
|
|
40
|
+
branchIndexFilename,
|
|
39
41
|
INDEX_FILENAME
|
|
40
42
|
} from '@uniweb/projections'
|
|
41
43
|
|
|
@@ -133,8 +135,8 @@ export async function buildSiteData({
|
|
|
133
135
|
)
|
|
134
136
|
await writeCollectionFiles(resolvedSiteRoot, collections, siteContent.config.collections)
|
|
135
137
|
|
|
136
|
-
const publicDataDir = join(resolvedSiteRoot, 'public',
|
|
137
|
-
const distDataDir = join(resolvedDistDir,
|
|
138
|
+
const publicDataDir = join(resolvedSiteRoot, 'public', DATA_DIR)
|
|
139
|
+
const distDataDir = join(resolvedDistDir, DATA_DIR)
|
|
138
140
|
if (existsSync(publicDataDir)) {
|
|
139
141
|
await cp(publicDataDir, distDataDir, { recursive: true })
|
|
140
142
|
}
|
|
@@ -228,7 +230,7 @@ export async function buildSiteData({
|
|
|
228
230
|
const collectionIndexes = []
|
|
229
231
|
for (const [collName, collConfig] of Object.entries(collections)) {
|
|
230
232
|
if (!collConfig.search?.enabled || !collConfig.route) continue
|
|
231
|
-
const cascadeFile = join(resolvedDistDir,
|
|
233
|
+
const cascadeFile = join(resolvedDistDir, DATA_DIR, `${collName}.json`)
|
|
232
234
|
if (!existsSync(cascadeFile)) continue
|
|
233
235
|
let collectionData
|
|
234
236
|
try {
|
|
@@ -297,6 +299,23 @@ async function writeProjections(siteContent, distDir) {
|
|
|
297
299
|
if (agents.index) {
|
|
298
300
|
const index = renderSiteIndex(siteContent, { ...options, exclude: agents.exclude })
|
|
299
301
|
await writeFile(join(distDir, INDEX_FILENAME), index)
|
|
302
|
+
|
|
303
|
+
// Additive scoped indexes; the root one above stays complete. See
|
|
304
|
+
// `selectIndexBranches` for why this is not a delegation.
|
|
305
|
+
if (agents.branchIndexes) {
|
|
306
|
+
const branches = selectIndexBranches(siteContent.pages, {
|
|
307
|
+
exclude: agents.exclude,
|
|
308
|
+
minPages: agents.branchMinPages
|
|
309
|
+
})
|
|
310
|
+
for (const branch of branches) {
|
|
311
|
+
const target = join(distDir, branchIndexFilename(branch.route))
|
|
312
|
+
await mkdir(dirname(target), { recursive: true })
|
|
313
|
+
await writeFile(
|
|
314
|
+
target,
|
|
315
|
+
renderSiteIndex(siteContent, { ...options, exclude: agents.exclude, branch: branch.route })
|
|
316
|
+
)
|
|
317
|
+
}
|
|
318
|
+
}
|
|
300
319
|
}
|
|
301
320
|
|
|
302
321
|
if (!agents.markdown) return
|
|
@@ -32,11 +32,12 @@
|
|
|
32
32
|
* await writeCollectionFiles(siteDir, collections)
|
|
33
33
|
*/
|
|
34
34
|
|
|
35
|
-
import { readFile, readdir, stat, writeFile, mkdir, copyFile } from 'node:fs/promises'
|
|
36
|
-
import { join, basename, extname, dirname, relative, resolve } from 'node:path'
|
|
35
|
+
import { readFile, readdir, stat, writeFile, mkdir, copyFile, rm } from 'node:fs/promises'
|
|
36
|
+
import { join, basename, extname, dirname, relative, resolve, sep } from 'node:path'
|
|
37
37
|
import { existsSync } from 'node:fs'
|
|
38
38
|
import yaml from 'js-yaml'
|
|
39
39
|
import { parseBibtex } from '@citestyle/bibtex'
|
|
40
|
+
import { DATA_DIR } from '@uniweb/core'
|
|
40
41
|
import { applyFilter, applySort } from './data-fetcher.js'
|
|
41
42
|
import { resolveAssetPath, walkContentAssets, isLocalAssetPath } from './assets.js'
|
|
42
43
|
|
|
@@ -642,6 +643,62 @@ export async function processCollections(siteDir, collectionsConfig, collections
|
|
|
642
643
|
return results
|
|
643
644
|
}
|
|
644
645
|
|
|
646
|
+
/**
|
|
647
|
+
* Reconcile a deferred collection's per-record directory with the records it
|
|
648
|
+
* should hold this run — delete the `<slug>.json` files that are no longer
|
|
649
|
+
* backed by a record.
|
|
650
|
+
*
|
|
651
|
+
* Why this is not optional. `public/data/` is a persistent, normally-committed
|
|
652
|
+
* directory, so anything written there survives until something removes it.
|
|
653
|
+
* Without this, unpublishing a record (`published: false`, which the build
|
|
654
|
+
* honours automatically) or deleting its source file drops it from the cascade
|
|
655
|
+
* listing — it vanishes from the site — while its per-record file stays on
|
|
656
|
+
* disk with the full body, gets committed, and gets deployed. The author has
|
|
657
|
+
* every reason to believe the content is gone. It is still fetchable at a URL
|
|
658
|
+
* that was public a moment ago.
|
|
659
|
+
*
|
|
660
|
+
* `public/data/` is the build's output directory and nothing else — authors
|
|
661
|
+
* provide structured data through `collections/`, which is the only supported
|
|
662
|
+
* way. So `<name>/` is entirely ours and the reconciliation is total: anything
|
|
663
|
+
* in it that this run did not write is stale by definition. `expected` is
|
|
664
|
+
* empty when a collection stops declaring `deferred:`, which correctly clears
|
|
665
|
+
* a directory that will otherwise never be written again.
|
|
666
|
+
*
|
|
667
|
+
* NOT covered: a collection removed from `site.yml` entirely. There is no
|
|
668
|
+
* declaration left to reconcile against, so pruning it would mean the build
|
|
669
|
+
* asserting ownership of a directory on a name match alone. That needs the
|
|
670
|
+
* ownership question answered on purpose, not as a side effect of this.
|
|
671
|
+
*
|
|
672
|
+
* @param {string} dataDir - `public/data/`, the containing output directory
|
|
673
|
+
* @param {string} name - the declared collection name
|
|
674
|
+
* @param {Set<string>} expected - filenames this run wrote, e.g. `hello.json`
|
|
675
|
+
* @returns {Promise<string[]>} the entry names removed
|
|
676
|
+
*/
|
|
677
|
+
async function pruneOrphanedRecords(dataDir, name, expected) {
|
|
678
|
+
const recordsDir = join(dataDir, name)
|
|
679
|
+
|
|
680
|
+
// This routine deletes, and `name` reaches it from site.yml. A name that
|
|
681
|
+
// resolves outside the output directory would make the traversal somebody
|
|
682
|
+
// else's files, so refuse rather than trust the caller.
|
|
683
|
+
const contained = resolve(recordsDir)
|
|
684
|
+
if (contained !== resolve(dataDir, name) || !contained.startsWith(resolve(dataDir) + sep)) {
|
|
685
|
+
console.warn(
|
|
686
|
+
`[collection-processor] Refusing to prune "${name}" — it does not resolve ` +
|
|
687
|
+
`inside ${dataDir}`
|
|
688
|
+
)
|
|
689
|
+
return []
|
|
690
|
+
}
|
|
691
|
+
if (!existsSync(recordsDir)) return []
|
|
692
|
+
|
|
693
|
+
const removed = []
|
|
694
|
+
for (const entry of await readdir(recordsDir, { withFileTypes: true })) {
|
|
695
|
+
if (expected.has(entry.name)) continue
|
|
696
|
+
await rm(join(recordsDir, entry.name), { recursive: true, force: true })
|
|
697
|
+
removed.push(entry.isDirectory() ? `${entry.name}/` : entry.name)
|
|
698
|
+
}
|
|
699
|
+
return removed
|
|
700
|
+
}
|
|
701
|
+
|
|
645
702
|
/**
|
|
646
703
|
* Write collection data to JSON files in public/data/
|
|
647
704
|
*
|
|
@@ -660,7 +717,7 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
|
|
|
660
717
|
return
|
|
661
718
|
}
|
|
662
719
|
|
|
663
|
-
const dataDir = join(siteDir, 'public',
|
|
720
|
+
const dataDir = join(siteDir, 'public', DATA_DIR)
|
|
664
721
|
await mkdir(dataDir, { recursive: true })
|
|
665
722
|
|
|
666
723
|
for (const [name, items] of Object.entries(collections)) {
|
|
@@ -677,13 +734,15 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
|
|
|
677
734
|
const recordsDir = join(dataDir, name)
|
|
678
735
|
await mkdir(recordsDir, { recursive: true })
|
|
679
736
|
|
|
680
|
-
|
|
737
|
+
const written = new Set()
|
|
681
738
|
for (const item of items) {
|
|
682
739
|
if (!item || typeof item !== 'object' || !item.slug) continue
|
|
683
|
-
const
|
|
684
|
-
await writeFile(
|
|
685
|
-
|
|
740
|
+
const filename = `${item.slug}.json`
|
|
741
|
+
await writeFile(join(recordsDir, filename), JSON.stringify(item, null, 2))
|
|
742
|
+
written.add(filename)
|
|
686
743
|
}
|
|
744
|
+
const perRecordCount = written.size
|
|
745
|
+
const pruned = await pruneOrphanedRecords(dataDir, name, written)
|
|
687
746
|
|
|
688
747
|
const stripped = items.map((item) => {
|
|
689
748
|
if (!item || typeof item !== 'object') return item
|
|
@@ -697,10 +756,29 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
|
|
|
697
756
|
`[collection-processor] Generated ${cascadePath} (${items.length} items, ` +
|
|
698
757
|
`deferred: [${deferred.join(', ')}]) + ${perRecordCount} per-record files`
|
|
699
758
|
)
|
|
759
|
+
if (pruned.length > 0) {
|
|
760
|
+
// A deletion is always worth naming. These files were public a moment
|
|
761
|
+
// ago, so "which ones went" is the question an author will have.
|
|
762
|
+
console.log(
|
|
763
|
+
`[collection-processor] Removed ${pruned.length} stale per-record ` +
|
|
764
|
+
`file(s) from ${recordsDir}: ${pruned.join(', ')}`
|
|
765
|
+
)
|
|
766
|
+
}
|
|
700
767
|
} else {
|
|
701
768
|
const filepath = join(dataDir, `${name}.json`)
|
|
702
769
|
await writeFile(filepath, JSON.stringify(items, null, 2))
|
|
703
770
|
console.log(`[collection-processor] Generated ${filepath} (${items.length} items)`)
|
|
771
|
+
|
|
772
|
+
// This collection is not deferred, so it has no per-record files. If it
|
|
773
|
+
// used to, the directory is still there and will never be written again
|
|
774
|
+
// — every file in it is stale. Same reconciliation, empty expected set.
|
|
775
|
+
const pruned = await pruneOrphanedRecords(dataDir, name, new Set())
|
|
776
|
+
if (pruned.length > 0) {
|
|
777
|
+
console.log(
|
|
778
|
+
`[collection-processor] Removed ${pruned.length} per-record file(s) ` +
|
|
779
|
+
`from ${join(dataDir, name)} — "${name}" no longer declares deferred:`
|
|
780
|
+
)
|
|
781
|
+
}
|
|
704
782
|
}
|
|
705
783
|
}
|
|
706
784
|
}
|
|
@@ -2152,9 +2152,10 @@ export async function collectSiteContent(sitePath, options = {}) {
|
|
|
2152
2152
|
// base prefix while the hydrated browser routes were fine.
|
|
2153
2153
|
//
|
|
2154
2154
|
// Only a real base is written. At '/' the field stays absent, because in
|
|
2155
|
-
// shell mode `config.base` is the SERVING layer's channel
|
|
2156
|
-
//
|
|
2157
|
-
//
|
|
2155
|
+
// shell mode `config.base` is the SERVING layer's channel — the host injects
|
|
2156
|
+
// whatever subpath it serves the site under, and a build-time '/' would be a
|
|
2157
|
+
// meaningless value sitting in its slot. The build deliberately does not model
|
|
2158
|
+
// what that subpath looks like; that is the host's shape, not ours.
|
|
2158
2159
|
if (base && base !== '/') {
|
|
2159
2160
|
siteConfig.base = base
|
|
2160
2161
|
}
|
package/src/site/data-ball.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
import { existsSync } from 'node:fs'
|
|
20
20
|
import { readFile, readdir } from 'node:fs/promises'
|
|
21
21
|
import { join, relative, sep } from 'node:path'
|
|
22
|
+
import { DATA_DIR } from '@uniweb/core'
|
|
22
23
|
import { isLocalAssetPath } from './assets.js'
|
|
23
24
|
|
|
24
25
|
// Walk a dist subdir for *.json → { "<posix-relpath>": <parsedJson> }. Unparseable
|
|
@@ -58,7 +59,7 @@ function collectionOf(relPath) {
|
|
|
58
59
|
*/
|
|
59
60
|
export async function assembleDataBall(distDir, schemalessNames = []) {
|
|
60
61
|
const schemaless = new Set(schemalessNames)
|
|
61
|
-
const allData = await readJsonTree(join(distDir,
|
|
62
|
+
const allData = await readJsonTree(join(distDir, DATA_DIR))
|
|
62
63
|
const data = {}
|
|
63
64
|
for (const [relPath, value] of Object.entries(allData)) {
|
|
64
65
|
if (schemaless.has(collectionOf(relPath))) data[relPath] = value
|
package/src/site/data-fetcher.js
CHANGED
|
@@ -20,7 +20,7 @@ import { readFile } from 'node:fs/promises'
|
|
|
20
20
|
import { join } from 'node:path'
|
|
21
21
|
import { existsSync } from 'node:fs'
|
|
22
22
|
import yaml from 'js-yaml'
|
|
23
|
-
import { matchWhere } from '@uniweb/core'
|
|
23
|
+
import { matchWhere, collectionDataUrl } from '@uniweb/core'
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
26
|
* Infer schema name from path or URL
|
|
@@ -290,7 +290,7 @@ export function parseFetchConfig(fetch) {
|
|
|
290
290
|
if (fetch.collection) {
|
|
291
291
|
if (fetch.filter !== undefined) warnFilterDeprecated()
|
|
292
292
|
return {
|
|
293
|
-
path:
|
|
293
|
+
path: collectionDataUrl(fetch.collection),
|
|
294
294
|
url: undefined,
|
|
295
295
|
schema: fetch.schema || fetch.collection,
|
|
296
296
|
prerender: fetch.prerender ?? true,
|
package/src/site/plugin.js
CHANGED
|
@@ -33,13 +33,15 @@
|
|
|
33
33
|
import { resolve, join } from 'node:path'
|
|
34
34
|
import { watch, existsSync } from 'node:fs'
|
|
35
35
|
import { readFile, readdir } from 'node:fs/promises'
|
|
36
|
-
import { resolveDefaultLocale } from '@uniweb/core'
|
|
36
|
+
import { resolveDefaultLocale, DATA_DIR } from '@uniweb/core'
|
|
37
37
|
import {
|
|
38
38
|
renderSiteIndex,
|
|
39
39
|
renderPageMarkdown,
|
|
40
40
|
resolveAgentsConfig,
|
|
41
41
|
selectIndexablePages,
|
|
42
|
+
selectIndexBranches,
|
|
42
43
|
pageMarkdownFilename,
|
|
44
|
+
branchIndexFilename,
|
|
43
45
|
applyRouteTranslation,
|
|
44
46
|
INDEX_FILENAME
|
|
45
47
|
} from '@uniweb/projections'
|
|
@@ -275,19 +277,68 @@ function escapeXml(str) {
|
|
|
275
277
|
.replace(/'/g, ''')
|
|
276
278
|
}
|
|
277
279
|
|
|
280
|
+
/** The three defined Content Signals, in the order they are emitted. */
|
|
281
|
+
const CONTENT_SIGNAL_KEYS = ['search', 'ai-input', 'ai-train']
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Format the `Content-Signal:` directive from `seo.robots.contentSignals`.
|
|
285
|
+
*
|
|
286
|
+
* Content Signals express what a site permits its content to be *used for*,
|
|
287
|
+
* which is a different axis from `Disallow:` — that governs fetching, this
|
|
288
|
+
* governs use after fetching. The three defined signals:
|
|
289
|
+
*
|
|
290
|
+
* - `search` — appear in search results
|
|
291
|
+
* - `ai-input` — be retrieved at inference time (RAG, grounding)
|
|
292
|
+
* - `ai-train` — be used to train a model
|
|
293
|
+
*
|
|
294
|
+
* **Emitted only when declared.** There is no default: a preference the site
|
|
295
|
+
* owner did not state is not ours to assert, in either direction. An absent
|
|
296
|
+
* signal means "unstated", which is not the same as `no`.
|
|
297
|
+
*
|
|
298
|
+
* Unknown keys are ignored rather than passed through — the vocabulary is a
|
|
299
|
+
* closed set, and forwarding an invented signal would produce a directive no
|
|
300
|
+
* crawler honors while reading as though it were doing something.
|
|
301
|
+
*
|
|
302
|
+
* @param {Object|null} signals - e.g. `{ search: true, 'ai-input': true, 'ai-train': false }`
|
|
303
|
+
* @returns {string} The directive line, or `''` when nothing is declared
|
|
304
|
+
*/
|
|
305
|
+
export function formatContentSignals(signals) {
|
|
306
|
+
if (!signals || typeof signals !== 'object') return ''
|
|
307
|
+
|
|
308
|
+
const parts = []
|
|
309
|
+
for (const key of CONTENT_SIGNAL_KEYS) {
|
|
310
|
+
if (!(key in signals)) continue
|
|
311
|
+
const value = signals[key]
|
|
312
|
+
const yes = value === true || value === 'yes'
|
|
313
|
+
const no = value === false || value === 'no'
|
|
314
|
+
if (!yes && !no) continue
|
|
315
|
+
parts.push(`${key}=${yes ? 'yes' : 'no'}`)
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
return parts.length ? `Content-Signal: ${parts.join(', ')}` : ''
|
|
319
|
+
}
|
|
320
|
+
|
|
278
321
|
/**
|
|
279
322
|
* Generate robots.txt content
|
|
280
323
|
*/
|
|
281
|
-
function generateRobotsTxt(baseUrl, options = {}) {
|
|
324
|
+
export function generateRobotsTxt(baseUrl, options = {}) {
|
|
282
325
|
const {
|
|
283
326
|
disallow = [],
|
|
284
327
|
allow = [],
|
|
285
328
|
crawlDelay = null,
|
|
286
|
-
additionalSitemaps = []
|
|
329
|
+
additionalSitemaps = [],
|
|
330
|
+
contentSignals = null
|
|
287
331
|
} = options
|
|
288
332
|
|
|
289
333
|
let content = 'User-agent: *\n'
|
|
290
334
|
|
|
335
|
+
const signals = formatContentSignals(contentSignals)
|
|
336
|
+
if (signals) {
|
|
337
|
+
// Inside the User-agent group, before the rules — the directive applies to
|
|
338
|
+
// the group it sits in.
|
|
339
|
+
content += `${signals}\n`
|
|
340
|
+
}
|
|
341
|
+
|
|
291
342
|
for (const path of allow) {
|
|
292
343
|
content += `Allow: ${path}\n`
|
|
293
344
|
}
|
|
@@ -622,6 +673,31 @@ export function siteContentPlugin(options = {}) {
|
|
|
622
673
|
source: renderSiteIndex(content, { ...options, exclude: agents.exclude })
|
|
623
674
|
})
|
|
624
675
|
console.log(`[site-content] Generated ${localeDir}${INDEX_FILENAME}`)
|
|
676
|
+
|
|
677
|
+
// Branch indexes are ADDITIVE — the root index above still enumerates
|
|
678
|
+
// every page, because the two-hop criterion depends on it. These are a
|
|
679
|
+
// scoped entry point for an agent already inside a branch.
|
|
680
|
+
if (agents.branchIndexes) {
|
|
681
|
+
const branches = selectIndexBranches(content.pages, {
|
|
682
|
+
exclude: agents.exclude,
|
|
683
|
+
minPages: agents.branchMinPages
|
|
684
|
+
})
|
|
685
|
+
for (const branch of branches) {
|
|
686
|
+
this.emitFile({
|
|
687
|
+
type: 'asset',
|
|
688
|
+
fileName: `${localeDir}${branchIndexFilename(branch.route)}`,
|
|
689
|
+
source: renderSiteIndex(content, {
|
|
690
|
+
...options,
|
|
691
|
+
exclude: agents.exclude,
|
|
692
|
+
branch: branch.route
|
|
693
|
+
})
|
|
694
|
+
})
|
|
695
|
+
}
|
|
696
|
+
if (branches.length) {
|
|
697
|
+
const names = branches.map(b => `${b.route} (${b.count})`).join(', ')
|
|
698
|
+
console.log(`[site-content] Generated ${branches.length} branch index(es): ${names}`)
|
|
699
|
+
}
|
|
700
|
+
}
|
|
625
701
|
}
|
|
626
702
|
|
|
627
703
|
if (agents.markdown) {
|
|
@@ -1006,18 +1082,39 @@ export function siteContentPlugin(options = {}) {
|
|
|
1006
1082
|
const agents = resolveAgentsConfig(siteContent.config)
|
|
1007
1083
|
const url = req.url.split('?')[0]
|
|
1008
1084
|
|
|
1009
|
-
|
|
1085
|
+
// `/llms.txt`, `/fr/llms.txt`, and the branch form `/docs/llms.txt`.
|
|
1086
|
+
// The optional middle group is the branch route; an empty one is the
|
|
1087
|
+
// site index, so a single pattern serves both rather than two that
|
|
1088
|
+
// could drift.
|
|
1089
|
+
const indexMatch = url.match(
|
|
1090
|
+
new RegExp(`^(?:\\/(${LOCALE_RE}))?((?:\\/[^/]+)*)\\/${INDEX_FILENAME}$`)
|
|
1091
|
+
)
|
|
1010
1092
|
if (indexMatch && agents.index) {
|
|
1011
1093
|
const localized = (indexMatch[1] ? await getTranslatedContent(indexMatch[1]) : null) || siteContent
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1094
|
+
const branch = indexMatch[2] || null
|
|
1095
|
+
|
|
1096
|
+
// Serve a branch index only where the build would emit one, or dev
|
|
1097
|
+
// and the built output disagree about which URLs exist.
|
|
1098
|
+
const served =
|
|
1099
|
+
!branch ||
|
|
1100
|
+
(agents.branchIndexes &&
|
|
1101
|
+
selectIndexBranches(localized.pages, {
|
|
1102
|
+
exclude: agents.exclude,
|
|
1103
|
+
minPages: agents.branchMinPages
|
|
1104
|
+
}).some(b => b.route === branch))
|
|
1105
|
+
|
|
1106
|
+
if (served) {
|
|
1107
|
+
res.setHeader('Content-Type', 'text/plain; charset=utf-8')
|
|
1108
|
+
res.end(
|
|
1109
|
+
renderSiteIndex(localized, {
|
|
1110
|
+
...projectionOptions(localized),
|
|
1111
|
+
locale: indexMatch[1] || projectionOptions(localized).locale,
|
|
1112
|
+
exclude: agents.exclude,
|
|
1113
|
+
branch
|
|
1114
|
+
})
|
|
1115
|
+
)
|
|
1116
|
+
return
|
|
1117
|
+
}
|
|
1021
1118
|
}
|
|
1022
1119
|
|
|
1023
1120
|
const markdownMatch = url.match(new RegExp(`^(?:\\/(${LOCALE_RE}))?\\/(.+)\\.md$`))
|
|
@@ -1038,12 +1135,12 @@ export function siteContentPlugin(options = {}) {
|
|
|
1038
1135
|
}
|
|
1039
1136
|
|
|
1040
1137
|
// Handle localized collection data (e.g., /fr/data/articles.json)
|
|
1041
|
-
const localeDataMatch = req.url.match(new RegExp(`^\\/(${LOCALE_RE})\\/
|
|
1138
|
+
const localeDataMatch = req.url.match(new RegExp(`^\\/(${LOCALE_RE})\\/${DATA_DIR}\\/(.+\\.json)$`))
|
|
1042
1139
|
if (localeDataMatch) {
|
|
1043
1140
|
const locale = localeDataMatch[1]
|
|
1044
1141
|
const filename = localeDataMatch[2]
|
|
1045
1142
|
const collectionName = filename.replace('.json', '')
|
|
1046
|
-
const sourcePath = join(resolvedSitePath, 'public',
|
|
1143
|
+
const sourcePath = join(resolvedSitePath, 'public', DATA_DIR, filename)
|
|
1047
1144
|
|
|
1048
1145
|
if (existsSync(sourcePath)) {
|
|
1049
1146
|
try {
|
package/src/uwx/site.js
CHANGED
|
@@ -54,7 +54,7 @@ import {
|
|
|
54
54
|
processMarkdownFile,
|
|
55
55
|
} from '../site/content-collector.js'
|
|
56
56
|
import { normalizeHideIn } from '../site/nav-visibility.js'
|
|
57
|
-
import { resolveDefaultLocale, validateLanguageConfig } from '@uniweb/core'
|
|
57
|
+
import { resolveDefaultLocale, validateLanguageConfig, collectionDataUrl } from '@uniweb/core'
|
|
58
58
|
import { emitEntitySyncPackage } from './entity-document.js'
|
|
59
59
|
import { loadLocaleTranslations, localizeScalar, localizeScalarList, localizeContentDoc, localesDir, isLocalizedContent } from './locale-sync.js'
|
|
60
60
|
import { unwrapLocalized } from './backfill.js'
|
|
@@ -182,7 +182,7 @@ function buildPageData(config, ctx) {
|
|
|
182
182
|
// `schema` (the collection name) is BOTH the content.data key and part of the
|
|
183
183
|
// dataStore cache key (deriveCacheKey hashes {path,url,schema,…}; `collection`
|
|
184
184
|
// is ignored). Mirrors the static build's parseFetchConfig resolution.
|
|
185
|
-
fetch = { path:
|
|
185
|
+
fetch = { path: collectionDataUrl(collection), schema: collection, ...rest }
|
|
186
186
|
}
|
|
187
187
|
setIf(data, 'fetch', fetch)
|
|
188
188
|
if (isDynamic) {
|
package/src/validate-data.js
CHANGED
|
@@ -29,8 +29,9 @@ import { readFile } from 'node:fs/promises'
|
|
|
29
29
|
import { existsSync } from 'node:fs'
|
|
30
30
|
import { join, resolve, basename } from 'node:path'
|
|
31
31
|
import yaml from 'js-yaml'
|
|
32
|
+
import { collectionNameFromUrl } from '@uniweb/core'
|
|
32
33
|
|
|
33
|
-
import { SCALAR_KINDS, FORMAT_TYPES } from './resolve-data-schema.js'
|
|
34
|
+
import { SCALAR_KINDS, FORMAT_TYPES, validateAndNormalizeSchema } from './resolve-data-schema.js'
|
|
34
35
|
import { buildSchema } from './schema.js'
|
|
35
36
|
import { resolveFoundationSrcPath } from './utils/foundation-source-root.js'
|
|
36
37
|
import { collectSiteContent } from './site/content-collector.js'
|
|
@@ -352,6 +353,13 @@ export async function validateDataInputs({ siteRoot, foundationPath }) {
|
|
|
352
353
|
})
|
|
353
354
|
}
|
|
354
355
|
|
|
356
|
+
// Pass 3 — concept blocks, which join to a schema by CONVENTION rather than
|
|
357
|
+
// by a foundation binding. Additive and silent unless a schema resolves.
|
|
358
|
+
const concepts = await validateConceptBlocks(site)
|
|
359
|
+
violations.push(...concepts.violations)
|
|
360
|
+
for (const ref of concepts.schemas) schemasSeen.add(ref)
|
|
361
|
+
recordCount += concepts.checked
|
|
362
|
+
|
|
355
363
|
return {
|
|
356
364
|
violations,
|
|
357
365
|
deferred,
|
|
@@ -365,6 +373,161 @@ export async function validateDataInputs({ siteRoot, foundationPath }) {
|
|
|
365
373
|
}
|
|
366
374
|
}
|
|
367
375
|
|
|
376
|
+
/**
|
|
377
|
+
* Check each ```md:<tag> concept block against `@std/<tag>`, when that schema
|
|
378
|
+
* exists.
|
|
379
|
+
*
|
|
380
|
+
* THREE PROPERTIES MAKE THIS SAFE, and all three have to hold:
|
|
381
|
+
*
|
|
382
|
+
* 1. It adds NO REGISTRY. The resolution is mechanical — `md:faq` → `@std/faq`,
|
|
383
|
+
* the same `@std` → `@uniweb/schemas` mapping every other ref uses. What the
|
|
384
|
+
* framework gains is a naming convention; no code branches on the value of a
|
|
385
|
+
* tag, and nothing here knows which concepts exist. A hardcoded list of
|
|
386
|
+
* concept names is the thing this whole design exists to avoid, and it would
|
|
387
|
+
* arrive through this door if the check needed to know what `faq` means.
|
|
388
|
+
*
|
|
389
|
+
* 2. It never touches SHAPE. A concept block's shape comes from its fence,
|
|
390
|
+
* unconditionally. This runs after the parse and changes nothing: a block
|
|
391
|
+
* with no resolvable schema still parses, still delivers items, still
|
|
392
|
+
* renders. The schema is a check, never a gate.
|
|
393
|
+
*
|
|
394
|
+
* 3. It never fails at RENDER. Findings only — this whole module is a pre-live
|
|
395
|
+
* dev/CI gate and the runtime stays tolerant.
|
|
396
|
+
*
|
|
397
|
+
* ⛔ A standard schema for a concept MUST be authored in the ITEM vocabulary —
|
|
398
|
+
* `title`, `paragraphs`, and the rest of the parsed shape — because that is what
|
|
399
|
+
* a concept block always produces. An `@std/faq` written as `{ question, answer }`
|
|
400
|
+
* could only be checked with a per-concept field mapping, which is the forbidden
|
|
401
|
+
* registry arriving by the back door. Author the schema to match the parse, or
|
|
402
|
+
* do not ship the schema.
|
|
403
|
+
*
|
|
404
|
+
* ⛔ AND FOR A PROSE CONCEPT, NO FACET CAN FIRE AT ALL — so do not write an
|
|
405
|
+
* `@std` schema for one. Measured 2026-07-30:
|
|
406
|
+
*
|
|
407
|
+
* - `required` is inert. The item vocabulary is TOTAL — `flattenGroup` fills
|
|
408
|
+
* every field it declares, so a titleless item has `title: ''` rather than
|
|
409
|
+
* no title, and `required` fires only on absent or null. "The author
|
|
410
|
+
* actually wrote a question" is not expressible.
|
|
411
|
+
* - `type` cannot fail either. Inside a concept block `title` is always a
|
|
412
|
+
* string (never an array — `alwaysItems` suppresses the same-level merge
|
|
413
|
+
* that would make one) and `paragraphs` is always an array of strings.
|
|
414
|
+
* - which leaves `enum` / `format`, and neither has a natural application to
|
|
415
|
+
* a question or an answer. The test suite had to invent `format: 'url'` on
|
|
416
|
+
* a question to make anything fire — that is the tell, not a fixture quirk.
|
|
417
|
+
*
|
|
418
|
+
* The mechanism still earns its place, but it is waiting for a different shape:
|
|
419
|
+
* a concept that carries a tagged DATA BLOCK. Verified that one reaches the item
|
|
420
|
+
* — ```` ```md:steps ```` holding a ```` ```yaml:meta ```` gives
|
|
421
|
+
* `items[0].data.meta` — and there `required` fires when an author omits the
|
|
422
|
+
* block, `enum` constrains a status, `format` constrains a duration. That is the
|
|
423
|
+
* trigger to write a schema. Until then the frontend holds the concept names and
|
|
424
|
+
* their shapes, which is where they belong: its extension encodes the shape
|
|
425
|
+
* executably, and a `standard/faq.js` in `@uniweb/schemas` whose only consumer is
|
|
426
|
+
* that app would be this framework stating which concepts exist — the registry
|
|
427
|
+
* this design forbids, spelled as a filename instead of a switch.
|
|
428
|
+
*
|
|
429
|
+
* Note on resolution: this deliberately does NOT go through `resolveSchemaRef`,
|
|
430
|
+
* which resolves a package from a FOUNDATION's node_modules and throws when a
|
|
431
|
+
* ref is unknown. Neither fits — a concept block needs no foundation (so this
|
|
432
|
+
* works on a link-mode site whose foundation is a registry ref with nothing
|
|
433
|
+
* local), and an unresolved tag must be silent rather than an error. So the
|
|
434
|
+
* package is resolved from this build's own graph, where it is an
|
|
435
|
+
* optionalDependency, exactly as `i18n/collections.js` resolves it.
|
|
436
|
+
*
|
|
437
|
+
* @param {Object} site - collected site content (`{ pages }`)
|
|
438
|
+
* @returns {Promise<{ violations: Array, schemas: Set<string>, checked: number }>}
|
|
439
|
+
*/
|
|
440
|
+
export async function validateConceptBlocks(site) {
|
|
441
|
+
const empty = { violations: [], schemas: new Set(), checked: 0 }
|
|
442
|
+
|
|
443
|
+
const parse = await loadSemanticParser()
|
|
444
|
+
if (!parse) return empty // no parser available — nothing to derive items from
|
|
445
|
+
|
|
446
|
+
const standards = await loadStandardSchemas()
|
|
447
|
+
if (!standards) return empty // @uniweb/schemas absent — nothing to check against
|
|
448
|
+
|
|
449
|
+
const violations = []
|
|
450
|
+
const schemasSeen = new Set()
|
|
451
|
+
let checked = 0
|
|
452
|
+
|
|
453
|
+
for (const page of site.pages || []) {
|
|
454
|
+
walkSections(page.sections || [], (section) => {
|
|
455
|
+
const doc = section.content
|
|
456
|
+
if (doc?.type !== 'doc') return
|
|
457
|
+
|
|
458
|
+
for (const node of conceptBlockNodes(doc)) {
|
|
459
|
+
const tag = node.attrs?.tag
|
|
460
|
+
if (!tag) continue
|
|
461
|
+
|
|
462
|
+
const raw = standards(tag)
|
|
463
|
+
if (!raw) continue // no `@std/<tag>` — say nothing, by design
|
|
464
|
+
|
|
465
|
+
let schema
|
|
466
|
+
try {
|
|
467
|
+
schema = validateAndNormalizeSchema(raw, `@std/${tag}`)
|
|
468
|
+
} catch {
|
|
469
|
+
continue // a malformed standard schema is that package's problem
|
|
470
|
+
}
|
|
471
|
+
if (!isStaticallyCheckable(schema)) continue
|
|
472
|
+
|
|
473
|
+
schemasSeen.add(`@std/${tag}`)
|
|
474
|
+
const { items } = parse({ type: 'doc', content: node.content || [] }, { alwaysItems: true })
|
|
475
|
+
|
|
476
|
+
items.forEach((item, idx) => {
|
|
477
|
+
checked++
|
|
478
|
+
for (const finding of validateItem(schema, item)) {
|
|
479
|
+
violations.push({
|
|
480
|
+
file: `${page.route || '/'} › ${section.type || 'section'} › md:${tag}`,
|
|
481
|
+
schema: `@std/${tag}`,
|
|
482
|
+
item: `item ${idx + 1}`,
|
|
483
|
+
users: [{ route: page.route, section: section.type, key: tag }],
|
|
484
|
+
...finding,
|
|
485
|
+
})
|
|
486
|
+
}
|
|
487
|
+
})
|
|
488
|
+
}
|
|
489
|
+
})
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
return { violations, schemas: schemasSeen, checked }
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** Every concept block in a doc, including any nested inside a container. */
|
|
496
|
+
function conceptBlockNodes(doc) {
|
|
497
|
+
const out = []
|
|
498
|
+
const walk = (nodes) => {
|
|
499
|
+
for (const node of nodes || []) {
|
|
500
|
+
if (!node) continue
|
|
501
|
+
if (node.type === 'concept_block') out.push(node)
|
|
502
|
+
else if (Array.isArray(node.content)) walk(node.content)
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
walk(doc?.content)
|
|
506
|
+
return out
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/** `parseContent`, or null when the parser is not installed. */
|
|
510
|
+
async function loadSemanticParser() {
|
|
511
|
+
try {
|
|
512
|
+
const mod = await import('@uniweb/semantic-parser')
|
|
513
|
+
return typeof mod.parseContent === 'function' ? mod.parseContent : null
|
|
514
|
+
} catch {
|
|
515
|
+
return null
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/** A `(name) => schema | undefined` lookup over `@std`, or null when absent. */
|
|
520
|
+
async function loadStandardSchemas() {
|
|
521
|
+
try {
|
|
522
|
+
const mod = await import('@uniweb/schemas')
|
|
523
|
+
if (typeof mod.getSchema === 'function') return (name) => mod.getSchema(name)
|
|
524
|
+
const table = mod.schemas ?? mod.default
|
|
525
|
+
return table ? (name) => table[name] : null
|
|
526
|
+
} catch {
|
|
527
|
+
return null
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
368
531
|
/**
|
|
369
532
|
* The data inputs available to a section, deduped by key. A section receives
|
|
370
533
|
* its own fetch plus any inherited page-level and site-level fetch (default-on
|
|
@@ -402,8 +565,9 @@ function walkSections(sections, visit) {
|
|
|
402
565
|
* (hand-authored data) is read from disk. Either way no prior build is needed.
|
|
403
566
|
*/
|
|
404
567
|
async function resolveRecords(path, { collections, siteRoot }) {
|
|
405
|
-
//
|
|
406
|
-
|
|
568
|
+
// A compiled-collection URL → a declared collection? Use the compiled
|
|
569
|
+
// records. Anything else falls through to the file read below.
|
|
570
|
+
const name = collectionNameFromUrl(path)
|
|
407
571
|
let records
|
|
408
572
|
if (Object.prototype.hasOwnProperty.call(collections, name)) {
|
|
409
573
|
records = collections[name]
|