@uniweb/build 1.13.3 → 1.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/i18n/audit.js +8 -1
- package/src/i18n/index.js +20 -2
- package/src/i18n/locales.js +2 -1
- package/src/i18n/outdated.js +146 -0
- package/src/i18n/records.js +2 -1
- package/src/i18n/sync.js +21 -0
- package/src/i18n/word-overlap.js +34 -0
- package/src/uwx/backfill.js +4 -4
- package/src/uwx/index.js +3 -0
- package/src/uwx/locale-sync.js +308 -28
- package/src/uwx/records-project.js +4 -2
- package/src/uwx/records.js +13 -8
- package/src/uwx/site-diff.js +17 -0
- package/src/uwx/site-project.js +4 -2
- package/src/uwx/site.js +47 -4
- package/src/uwx/sync-package.js +11 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@uniweb/build",
|
|
3
|
-
"version": "1.13.
|
|
3
|
+
"version": "1.13.4",
|
|
4
4
|
"description": "Build tooling for the Uniweb Component Web Platform",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -58,10 +58,10 @@
|
|
|
58
58
|
"yaml": "^2.5.0",
|
|
59
59
|
"@uniweb/content-reader": "~1.5.7",
|
|
60
60
|
"@uniweb/content-writer": "~1.3.1",
|
|
61
|
-
"@uniweb/projections": "~1.2.8",
|
|
62
61
|
"@uniweb/schemas": "~1.6.7",
|
|
63
|
-
"@uniweb/
|
|
64
|
-
"@uniweb/theming": "~1.2.1"
|
|
62
|
+
"@uniweb/projections": "~1.2.8",
|
|
63
|
+
"@uniweb/theming": "~1.2.1",
|
|
64
|
+
"@uniweb/semantic-parser": "~1.13.0"
|
|
65
65
|
},
|
|
66
66
|
"optionalDependencies": {
|
|
67
67
|
"@uniweb/runtime": "~1.3.3"
|
package/src/i18n/audit.js
CHANGED
|
@@ -48,6 +48,7 @@ export async function auditLocale(localesPath, locale) {
|
|
|
48
48
|
const missing = []
|
|
49
49
|
const stale = []
|
|
50
50
|
const losesMarkup = []
|
|
51
|
+
const outdated = []
|
|
51
52
|
|
|
52
53
|
// Check manifest entries
|
|
53
54
|
for (const hash of manifestHashes) {
|
|
@@ -55,6 +56,11 @@ export async function auditLocale(localesPath, locale) {
|
|
|
55
56
|
const unit = manifest.units[hash]
|
|
56
57
|
const source = unit.source
|
|
57
58
|
const translation = getTranslationText(translations[hash])
|
|
59
|
+
// Made from an older source text (`locale-sync.js::settleEntry`). It still shows.
|
|
60
|
+
if (translations[hash]?.outdated) {
|
|
61
|
+
outdated.push({ hash, source, translation, was: [...new Set(Object.values(translations[hash].outdated))] })
|
|
62
|
+
continue
|
|
63
|
+
}
|
|
58
64
|
valid.push({ hash, source, translation })
|
|
59
65
|
|
|
60
66
|
// ⭐ The source element carries inline markdown, and what the merge makes of
|
|
@@ -98,7 +104,8 @@ export async function auditLocale(localesPath, locale) {
|
|
|
98
104
|
valid,
|
|
99
105
|
missing,
|
|
100
106
|
stale,
|
|
101
|
-
losesMarkup
|
|
107
|
+
losesMarkup,
|
|
108
|
+
outdated
|
|
102
109
|
}
|
|
103
110
|
}
|
|
104
111
|
|
package/src/i18n/index.js
CHANGED
|
@@ -15,7 +15,8 @@ import { computeHash, normalizeText } from './hash.js'
|
|
|
15
15
|
import { availableLocales, resolveLocaleList, localesToBuild } from './locales.js'
|
|
16
16
|
import { extractTranslatableContent } from './extract.js'
|
|
17
17
|
import { dataBlockModels } from './data-models.js'
|
|
18
|
-
import { syncManifests, formatSyncReport } from './sync.js'
|
|
18
|
+
import { syncManifests, formatSyncReport, formatKeptTranslations } from './sync.js'
|
|
19
|
+
import { keepEditedTranslations, rowsWantTranslation, manifestRows } from './outdated.js'
|
|
19
20
|
import { mergeTranslations, generateAllLocales } from './merge.js'
|
|
20
21
|
import { auditLocale, cleanLocale, formatAuditReport } from './audit.js'
|
|
21
22
|
import {
|
|
@@ -64,6 +65,9 @@ export {
|
|
|
64
65
|
extractTranslatableContent,
|
|
65
66
|
syncManifests,
|
|
66
67
|
formatSyncReport,
|
|
68
|
+
formatKeptTranslations,
|
|
69
|
+
keepEditedTranslations,
|
|
70
|
+
rowsWantTranslation,
|
|
67
71
|
mergeTranslations,
|
|
68
72
|
generateAllLocales,
|
|
69
73
|
|
|
@@ -182,6 +186,12 @@ export async function extractManifest(siteRoot, siteContent, options = {}) {
|
|
|
182
186
|
// Generate sync report
|
|
183
187
|
const report = syncManifests(previousManifest, manifest)
|
|
184
188
|
|
|
189
|
+
// What the manifest is about to forget says what an edited text's translation was made from.
|
|
190
|
+
report.kept = keepEditedTranslations(siteRoot, manifestRows(previousManifest), manifestRows(manifest), {
|
|
191
|
+
sourceLocale: manifest.defaultLocale,
|
|
192
|
+
dryRun
|
|
193
|
+
})
|
|
194
|
+
|
|
185
195
|
// Write new manifest (skip in dry-run mode)
|
|
186
196
|
if (!dryRun) {
|
|
187
197
|
await writeFile(manifestPath, JSON.stringify(manifest, null, 2))
|
|
@@ -282,9 +292,13 @@ export async function getTranslationStatus(siteRoot, options = {}) {
|
|
|
282
292
|
|
|
283
293
|
let translated = 0
|
|
284
294
|
let missing = 0
|
|
295
|
+
let outdated = 0
|
|
285
296
|
|
|
286
297
|
for (const hash of Object.keys(manifest.units)) {
|
|
287
|
-
|
|
298
|
+
// An entry flagged `outdated` still renders, and is not this text's translation.
|
|
299
|
+
if (translations[hash]?.outdated) {
|
|
300
|
+
outdated++
|
|
301
|
+
} else if (translatedHashes.has(hash)) {
|
|
288
302
|
translated++
|
|
289
303
|
} else {
|
|
290
304
|
missing++
|
|
@@ -295,6 +309,7 @@ export async function getTranslationStatus(siteRoot, options = {}) {
|
|
|
295
309
|
exists: true,
|
|
296
310
|
translated,
|
|
297
311
|
missing,
|
|
312
|
+
...(outdated ? { outdated } : {}),
|
|
298
313
|
coverage: totalUnits > 0 ? Math.round((translated / totalUnits) * 100) : 100
|
|
299
314
|
}
|
|
300
315
|
}
|
|
@@ -642,6 +657,9 @@ export function formatTranslationStatus(status) {
|
|
|
642
657
|
if (info.missing > 0) {
|
|
643
658
|
lines.push(` ${info.missing} strings missing`)
|
|
644
659
|
}
|
|
660
|
+
if (info.outdated > 0) {
|
|
661
|
+
lines.push(` ${info.outdated} outdated: the old translation still shows, and its source has changed`)
|
|
662
|
+
}
|
|
645
663
|
}
|
|
646
664
|
}
|
|
647
665
|
|
package/src/i18n/locales.js
CHANGED
|
@@ -13,7 +13,8 @@ import { existsSync, readdirSync } from 'node:fs'
|
|
|
13
13
|
import { normalizeLanguageList, resolveDefaultLocale } from '@uniweb/core/locale-config'
|
|
14
14
|
|
|
15
15
|
// Files in the locales directory that are not a locale's translations.
|
|
16
|
-
export const
|
|
16
|
+
export const KEPT_FILE = '_kept.json'
|
|
17
|
+
export const RESERVED_LOCALE_FILES = new Set(['manifest.json', '_memory.json', KEPT_FILE])
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* The locales a directory holds translations for — one per `<locale>.json` — sorted.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ⭐ A SOURCE EDIT KEEPS ITS TRANSLATION UNTIL SOMEONE REVIEWS IT, in a file as in the app: a slightly
|
|
3
|
+
* outdated sentence beats a sudden one in the source language, and a typo fixed in the source should
|
|
4
|
+
* not change a translated page. So a row with no translation, in a place where a similar text has gone,
|
|
5
|
+
* takes that text's translation, and its entry says so (`{ default, outdated: { place: older text } }`).
|
|
6
|
+
* It is what a pull writes for an edit made in the app (`uwx/locale-sync.js::settleEntry`), and a push
|
|
7
|
+
* sends it back as the app keeps it.
|
|
8
|
+
*
|
|
9
|
+
* ⛔ Until 2026-10-31 the row fell back to the source at the next build or push, whatever the edit.
|
|
10
|
+
*
|
|
11
|
+
* Each lane reads the old text from its own record: `uniweb i18n sync` from the manifest it is about to
|
|
12
|
+
* replace, a push from the store it is about to change. Only a block element's text in a page's section,
|
|
13
|
+
* as the app's rows are: a picture, a data block's string, a page's title and a record's body fall back
|
|
14
|
+
* as they did.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { computeHash } from './hash.js'
|
|
18
|
+
import { toPlainText } from './merge.js'
|
|
19
|
+
import { isSmallEdit, wordOverlap } from './word-overlap.js'
|
|
20
|
+
import { discoverLocales, loadLocaleTranslations, translationFor, addTranslationEntries } from '../uwx/locale-sync.js'
|
|
21
|
+
|
|
22
|
+
// A language's words for a text here. Not an empty one, nor the text's own: `uniweb i18n init` starts a
|
|
23
|
+
// language's file with each text's source, and that is nothing to keep.
|
|
24
|
+
function translationHere(table, text, context) {
|
|
25
|
+
const hash = computeHash(text)
|
|
26
|
+
const said = translationFor(table[hash], context)
|
|
27
|
+
if (typeof said !== 'string' || said.trim() === '' || computeHash(toPlainText(said)) === hash) return undefined
|
|
28
|
+
return said
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Whether a language has nothing written for a row here.
|
|
32
|
+
function wants(table, row, context) {
|
|
33
|
+
const said = translationFor(table[computeHash(row)], context)
|
|
34
|
+
return typeof said !== 'string' || said.trim() === ''
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The entries a language takes for what changed. A row with a translation here keeps it, and a text
|
|
39
|
+
* that has gone goes to one row, the most like it: of a sentence split in two, one half keeps it.
|
|
40
|
+
*
|
|
41
|
+
* @param {{ [place: string]: string[] }} previous - each place's rows as they were
|
|
42
|
+
* @param {{ [place: string]: string[] }} current - each place's rows now, in order
|
|
43
|
+
* @param {object} table - the language's translation file
|
|
44
|
+
* @returns {{ [hash: string]: object }} what to add to the file
|
|
45
|
+
*/
|
|
46
|
+
export function carriedTranslations(previous, current, table) {
|
|
47
|
+
const added = {}
|
|
48
|
+
for (const [place, rows] of Object.entries(current)) {
|
|
49
|
+
const context = { key: place }
|
|
50
|
+
const now = new Set(rows.map(computeHash))
|
|
51
|
+
const gone = new Map()
|
|
52
|
+
for (const text of previous[place] || []) {
|
|
53
|
+
const hash = computeHash(text)
|
|
54
|
+
const said = translationHere(table, text, context)
|
|
55
|
+
if (!now.has(hash) && said !== undefined) gone.set(hash, { text, said })
|
|
56
|
+
}
|
|
57
|
+
if (gone.size === 0) continue
|
|
58
|
+
const wanting = new Map()
|
|
59
|
+
for (const row of rows) {
|
|
60
|
+
if (wants(table, row, context)) wanting.set(computeHash(row), row)
|
|
61
|
+
}
|
|
62
|
+
const pairs = []
|
|
63
|
+
for (const [hash, row] of wanting) {
|
|
64
|
+
for (const [from, { text }] of gone) {
|
|
65
|
+
if (isSmallEdit(text, row)) pairs.push({ hash, from, overlap: wordOverlap(text, row) })
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
pairs.sort((x, y) => y.overlap - x.overlap)
|
|
69
|
+
const taken = new Set()
|
|
70
|
+
for (const { hash, from } of pairs) {
|
|
71
|
+
if (taken.has(hash) || !gone.has(from)) continue
|
|
72
|
+
const { text, said } = gone.get(from)
|
|
73
|
+
gone.delete(from)
|
|
74
|
+
taken.add(hash)
|
|
75
|
+
const entry = (added[hash] ||= table[hash] && typeof table[hash] === 'object' ? { ...table[hash] } : {})
|
|
76
|
+
if (entry.default === undefined) entry.default = said
|
|
77
|
+
else if (entry.default !== said) entry.overrides = { ...entry.overrides, [place]: said }
|
|
78
|
+
entry.outdated = { ...entry.outdated, [place]: text }
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return added
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* What each place held at the last extraction: a manifest's units that are a block element's text.
|
|
86
|
+
*
|
|
87
|
+
* @param {object|null} manifest
|
|
88
|
+
* @returns {{ [place: string]: string[] }}
|
|
89
|
+
*/
|
|
90
|
+
export function manifestRows(manifest) {
|
|
91
|
+
const rows = {}
|
|
92
|
+
for (const unit of Object.values(manifest?.units || {})) {
|
|
93
|
+
if (!/^(title|subtitle|heading|paragraph|list)(\.|$)/.test(unit.field || '')) continue
|
|
94
|
+
for (const context of unit.contexts || []) {
|
|
95
|
+
if (context.section === '_meta' || context.section === '_site') continue
|
|
96
|
+
;(rows[`${context.page}:${context.section}`] ||= []).push(unit.source)
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return rows
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Each language's translation file but the source's, if it has one.
|
|
103
|
+
function languageTables(siteRoot, sourceLocale) {
|
|
104
|
+
return loadLocaleTranslations(siteRoot, discoverLocales(siteRoot).filter((locale) => locale !== sourceLocale))
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Whether a language has no words for one of these rows. Only such a row can keep a translation, so a
|
|
109
|
+
* push that finds none has no reason to read the store.
|
|
110
|
+
*
|
|
111
|
+
* @param {string} siteRoot
|
|
112
|
+
* @param {{ [place: string]: string[] }} current
|
|
113
|
+
* @param {{ sourceLocale?: string }} [o]
|
|
114
|
+
* @returns {boolean}
|
|
115
|
+
*/
|
|
116
|
+
export function rowsWantTranslation(siteRoot, current, { sourceLocale = null } = {}) {
|
|
117
|
+
const tables = Object.values(languageTables(siteRoot, sourceLocale))
|
|
118
|
+
return Object.entries(current).some(([place, rows]) =>
|
|
119
|
+
rows.some((row) => tables.some((table) => wants(table, row, { key: place }))),
|
|
120
|
+
)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Give each language's file what `carriedTranslations` finds, and say what it found.
|
|
125
|
+
*
|
|
126
|
+
* @param {string} siteRoot
|
|
127
|
+
* @param {{ [place: string]: string[] }} previous
|
|
128
|
+
* @param {{ [place: string]: string[] }} current
|
|
129
|
+
* @param {object} [o]
|
|
130
|
+
* @param {string} [o.sourceLocale] - its file, if it has one, is no translation
|
|
131
|
+
* @param {boolean} [o.dryRun]
|
|
132
|
+
* @returns {{ [locale: string]: Array<{ was: string, now: string }> }} a language with nothing kept is absent
|
|
133
|
+
*/
|
|
134
|
+
export function keepEditedTranslations(siteRoot, previous, current, { sourceLocale = null, dryRun = false } = {}) {
|
|
135
|
+
const sourceOf = new Map(Object.values(current).flat().map((text) => [computeHash(text), text]))
|
|
136
|
+
const kept = {}
|
|
137
|
+
for (const [locale, table] of Object.entries(languageTables(siteRoot, sourceLocale))) {
|
|
138
|
+
const added = carriedTranslations(previous, current, table)
|
|
139
|
+
if (Object.keys(added).length === 0) continue
|
|
140
|
+
if (!dryRun) addTranslationEntries(siteRoot, locale, added)
|
|
141
|
+
kept[locale] = Object.entries(added).flatMap(([hash, entry]) =>
|
|
142
|
+
[...new Set(Object.values(entry.outdated))].map((was) => ({ was, now: sourceOf.get(hash) })),
|
|
143
|
+
)
|
|
144
|
+
}
|
|
145
|
+
return kept
|
|
146
|
+
}
|
package/src/i18n/records.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { readFile, writeFile, readdir, mkdir, rm } from 'fs/promises'
|
|
14
|
+
import { RESERVED_LOCALE_FILES } from './locales.js'
|
|
14
15
|
import { existsSync } from 'node:fs'
|
|
15
16
|
import { join, resolve, sep } from 'node:path'
|
|
16
17
|
import { pathToFileURL } from 'node:url'
|
|
@@ -1338,7 +1339,7 @@ export async function getRecordLocales(localesPath) {
|
|
|
1338
1339
|
try {
|
|
1339
1340
|
const files = await readdir(recordLocalesDir)
|
|
1340
1341
|
return files
|
|
1341
|
-
.filter(f => f.endsWith('.json') && f
|
|
1342
|
+
.filter(f => f.endsWith('.json') && !RESERVED_LOCALE_FILES.has(f))
|
|
1342
1343
|
.map(f => f.replace('.json', ''))
|
|
1343
1344
|
.sort()
|
|
1344
1345
|
} catch {
|
package/src/i18n/sync.js
CHANGED
|
@@ -208,6 +208,8 @@ export function formatSyncReport(report) {
|
|
|
208
208
|
}
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
+
lines.push(...formatKeptTranslations(report.kept).map((line) => ` ${line}`))
|
|
212
|
+
|
|
211
213
|
if (report.added.length > 0) {
|
|
212
214
|
lines.push(` + ${report.added.length} new strings`)
|
|
213
215
|
for (const item of report.added.slice(0, 5)) {
|
|
@@ -235,6 +237,25 @@ export function formatSyncReport(report) {
|
|
|
235
237
|
return lines.join('\n')
|
|
236
238
|
}
|
|
237
239
|
|
|
240
|
+
/**
|
|
241
|
+
* What `keepEditedTranslations` kept, as lines to print.
|
|
242
|
+
*
|
|
243
|
+
* @param {{ [locale: string]: Array<{ was: string, now: string }> }} [kept]
|
|
244
|
+
* @returns {string[]} none when nothing was kept
|
|
245
|
+
*/
|
|
246
|
+
export function formatKeptTranslations(kept) {
|
|
247
|
+
const pairs = new Map()
|
|
248
|
+
for (const { was, now } of Object.values(kept || {}).flat()) pairs.set(`${was}\n${now}`, { was, now })
|
|
249
|
+
if (pairs.size === 0) return []
|
|
250
|
+
const lines = [
|
|
251
|
+
`↺ ${pairs.size} edited string${pairs.size === 1 ? ' keeps its' : 's keep their'} old translation until someone reviews it (${Object.keys(kept).join(', ')})`,
|
|
252
|
+
]
|
|
253
|
+
for (const { was, now } of [...pairs.values()].slice(0, 5)) lines.push(` - "${truncate(was, 30)}" → "${truncate(now, 30)}"`)
|
|
254
|
+
if (pairs.size > 5) lines.push(` ... and ${pairs.size - 5} more`)
|
|
255
|
+
lines.push(' Each is flagged `outdated` in the language files: replace it with the new translation, or with the same words as a plain string if they still hold.')
|
|
256
|
+
return lines
|
|
257
|
+
}
|
|
258
|
+
|
|
238
259
|
/**
|
|
239
260
|
* Format a context object for display
|
|
240
261
|
*/
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What counts as a small edit: the app's rule for when a translation stays with its row, flagged, after
|
|
3
|
+
* the row's source changes. Each side holds its own copy, so a pull and a push flag the rows the editor
|
|
4
|
+
* flags. The editor's was measured from the outside on 2026-10-31, with pairs pushed to it
|
|
5
|
+
* (`tests/i18n-edit-keeps-translation.test.js`), since it re-tests a row it reads: a row kept under a text
|
|
6
|
+
* it does not count as similar shows the old translation, and no flag.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const wordsOf = (text) => String(text).toLowerCase().match(/[\p{L}\p{N}]+/gu) || []
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* How much of two texts is the same: twice the words they share, each as often as both have it, over the
|
|
13
|
+
* words of both. Case and punctuation are no words, and a hyphen or an apostrophe splits one.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} a
|
|
16
|
+
* @param {string} b
|
|
17
|
+
* @returns {number} from 0 to 1
|
|
18
|
+
*/
|
|
19
|
+
export function wordOverlap(a, b) {
|
|
20
|
+
const [first, second] = [wordsOf(a), wordsOf(b)]
|
|
21
|
+
if (first.length + second.length === 0) return 0
|
|
22
|
+
const left = new Map()
|
|
23
|
+
for (const word of first) left.set(word, (left.get(word) || 0) + 1)
|
|
24
|
+
let shared = 0
|
|
25
|
+
for (const word of second) {
|
|
26
|
+
if (!left.get(word)) continue
|
|
27
|
+
left.set(word, left.get(word) - 1)
|
|
28
|
+
shared++
|
|
29
|
+
}
|
|
30
|
+
return (2 * shared) / (first.length + second.length)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Whether a translation made from one text stays with another. */
|
|
34
|
+
export const isSmallEdit = (a, b) => wordOverlap(a, b) >= 0.5
|
package/src/uwx/backfill.js
CHANGED
|
@@ -43,7 +43,7 @@ import { parseFrontmatter } from './entity-source.js'
|
|
|
43
43
|
import { setYamlKeyFirst } from './yaml-edit.js'
|
|
44
44
|
import { isProseMirrorField, isOpenMapSection, openMapFromRows } from './data-schema.js'
|
|
45
45
|
import { recordFileLayout, contentBodyTarget } from './record-layout.js'
|
|
46
|
-
import { unwrapLocalizedContent } from './locale-sync.js'
|
|
46
|
+
import { recordFieldPlace, unwrapLocalizedContent } from './locale-sync.js'
|
|
47
47
|
import { parseBibtex, exportBibtex } from '@citestyle/bibtex'
|
|
48
48
|
|
|
49
49
|
// Probed in this order to locate a single-record source file by slug.
|
|
@@ -351,7 +351,7 @@ function decodeRecord(fields, value, dec, sectionName) {
|
|
|
351
351
|
continue
|
|
352
352
|
}
|
|
353
353
|
const isBody = Boolean(dec.bodyAt) && sectionName === dec.bodyAt.section && key === dec.bodyAt.key
|
|
354
|
-
const decoded = decodeLeaf(raw, field, dec, isBody)
|
|
354
|
+
const decoded = decodeLeaf(raw, field, dec, isBody, key)
|
|
355
355
|
if (isBody) dec.body = typeof decoded === 'string' ? decoded : String(decoded ?? '')
|
|
356
356
|
// A translatable value with no text in any language is no value: the field is not written.
|
|
357
357
|
else if (decoded !== undefined) out[key] = decoded
|
|
@@ -386,11 +386,11 @@ function decodeList(def, value, dec) {
|
|
|
386
386
|
// list of localized values is unwrapped element by element. ⛔ Until 2026-09-26 a body was
|
|
387
387
|
// taken to be ProseMirror, so a markdown one's translations were captured nowhere and a
|
|
388
388
|
// clone rendered it in the source language.
|
|
389
|
-
function decodeLeaf(raw, field, dec, isBody) {
|
|
389
|
+
function decodeLeaf(raw, field, dec, isBody, key) {
|
|
390
390
|
if (field?.type === 'entity_ref') return decodeReference(raw, field, dec)
|
|
391
391
|
if (isProseMirrorField(field)) {
|
|
392
392
|
const sourceDoc = field.localized
|
|
393
|
-
? unwrapLocalizedContent(raw, dec.sourceLocale, dec.collector, isBody ? dec.freeformRelPath : undefined, null, dec.context)
|
|
393
|
+
? unwrapLocalizedContent(raw, dec.sourceLocale, dec.collector, isBody ? dec.freeformRelPath : undefined, null, dec.context, undefined, dec.context ? recordFieldPlace(dec.context, key) : null)
|
|
394
394
|
: raw
|
|
395
395
|
return sourceDoc ? proseMirrorToMarkdown(sourceDoc) : ''
|
|
396
396
|
}
|
package/src/uwx/index.js
CHANGED
|
@@ -41,8 +41,10 @@ export {
|
|
|
41
41
|
foundationSchemaToEntity,
|
|
42
42
|
emitFoundationSchemaPackage,
|
|
43
43
|
} from './foundation-schema.js'
|
|
44
|
+
export { documentSourceLocale } from './source-locale.js'
|
|
44
45
|
export {
|
|
45
46
|
siteProjectToDocument,
|
|
47
|
+
documentRows,
|
|
46
48
|
emitSiteSyncPackage,
|
|
47
49
|
writeSiteEntityUuid,
|
|
48
50
|
extensionDeclaration,
|
|
@@ -138,6 +140,7 @@ export {
|
|
|
138
140
|
diffSiteUnits,
|
|
139
141
|
describeSiteDiff,
|
|
140
142
|
computeUnitHashes,
|
|
143
|
+
sectionsChangedSince,
|
|
141
144
|
collectSiteUnits,
|
|
142
145
|
walkSiteUnits,
|
|
143
146
|
collectUnitUuids,
|
package/src/uwx/locale-sync.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
// surface, used by the projector (pull → files) and, later, the producer
|
|
17
17
|
// (files → push).
|
|
18
18
|
|
|
19
|
-
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'node:fs'
|
|
19
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs'
|
|
20
20
|
import { join, dirname } from 'node:path'
|
|
21
21
|
import yaml from 'js-yaml'
|
|
22
22
|
import { YAML_OPTIONS } from '../utils/yaml-schema.js'
|
|
@@ -29,6 +29,8 @@ import { ASSET_SLOTS } from '@uniweb/semantic-parser'
|
|
|
29
29
|
import { computeSourceHash, manifestText } from '../i18n/freeform-manifest.js'
|
|
30
30
|
import { sameMarkdownDocument, canonicalJson } from './same-content.js'
|
|
31
31
|
import { parseFrontmatter } from '../utils/frontmatter.js'
|
|
32
|
+
import { KEPT_FILE, RESERVED_LOCALE_FILES } from '../i18n/locales.js'
|
|
33
|
+
import { isSmallEdit, wordOverlap } from '../i18n/word-overlap.js'
|
|
32
34
|
|
|
33
35
|
const FREEFORM_MANIFEST = '.manifest.json'
|
|
34
36
|
|
|
@@ -92,7 +94,7 @@ export function discoverLocales(siteRoot, subdir = '') {
|
|
|
92
94
|
if (!existsSync(dir)) return []
|
|
93
95
|
try {
|
|
94
96
|
return readdirSync(dir)
|
|
95
|
-
.filter((f) => f.endsWith('.json') && f
|
|
97
|
+
.filter((f) => f.endsWith('.json') && !RESERVED_LOCALE_FILES.has(f) && !f.startsWith('.'))
|
|
96
98
|
.map((f) => f.slice(0, -'.json'.length))
|
|
97
99
|
} catch {
|
|
98
100
|
return []
|
|
@@ -130,6 +132,142 @@ export function loadLocaleTranslations(siteRoot, locales, subdir = '') {
|
|
|
130
132
|
return out
|
|
131
133
|
}
|
|
132
134
|
|
|
135
|
+
/**
|
|
136
|
+
* ⭐ WHAT A PULL KEEPS FOR A PUSH TO SEND BACK, `locales/_kept.json`: for a section, its `@` (what the
|
|
137
|
+
* app holds beside its translations, opaque here) and its documents in languages the site does not
|
|
138
|
+
* list (a language removed in the app keeps them, so adding it back loses nothing). A file has no
|
|
139
|
+
* other place for either, and THE ROUND-TRIP LAW holds for both. Absent on almost every site.
|
|
140
|
+
*
|
|
141
|
+
* ⛔ Until 2026-10-31 a pull dropped both, and the next push of the section removed them from the store.
|
|
142
|
+
*
|
|
143
|
+
* Keyed as a translation's place is (`contextKey`), since a section's identity on a backend is no use
|
|
144
|
+
* to another backend's copy.
|
|
145
|
+
*
|
|
146
|
+
* @param {string} siteRoot
|
|
147
|
+
* @param {string} [subdir] - `records` for record translations
|
|
148
|
+
* @returns {{ [place: string]: { '@'?: object, languages?: object } }}
|
|
149
|
+
*/
|
|
150
|
+
export function readKept(siteRoot, subdir = '') {
|
|
151
|
+
try {
|
|
152
|
+
const parsed = JSON.parse(readFileSync(join(localesDir(siteRoot, subdir), KEPT_FILE), 'utf8'))
|
|
153
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {}
|
|
154
|
+
} catch {
|
|
155
|
+
return {}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Where a record's localized field is kept: a record has more than one such field. */
|
|
160
|
+
export function recordFieldPlace(context, field) {
|
|
161
|
+
return `${contextKey(context)}#${field}`
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* A localized `content` with what the site keeps for it: never over a language the push has now.
|
|
166
|
+
*
|
|
167
|
+
* @param {object} localized
|
|
168
|
+
* @param {object} [held] - what `readKept` gives for it
|
|
169
|
+
* @param {object} [files] - what its translation files say (`mapWithFlags`)
|
|
170
|
+
*/
|
|
171
|
+
export function restoreKept(localized, held, files = null) {
|
|
172
|
+
if (!isLocalizedContent(localized)) return localized
|
|
173
|
+
const at = files ? mapWithFlags(held?.['@'], localized, files) : held?.['@']
|
|
174
|
+
if (!held && !at) return localized
|
|
175
|
+
const out = { ...localized }
|
|
176
|
+
for (const [locale, doc] of Object.entries(held?.languages || {})) {
|
|
177
|
+
if (!(locale in out)) out[locale] = doc
|
|
178
|
+
}
|
|
179
|
+
if (at) out['@'] = at
|
|
180
|
+
return out
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* ⭐ WHICH ROWS ARE OUTDATED IS THE TRANSLATION FILES' TO SAY, and the app reads it from `@.translations`:
|
|
185
|
+
* a row is outdated when its translation is kept there under a similar text (`olderSourceOf`). So a flagged
|
|
186
|
+
* row goes with its entry there, and a row its file does not flag goes without one. An entry already there
|
|
187
|
+
* stays as it is: a push's link addresses are not the store's. Nor is one touched that flags nothing, such
|
|
188
|
+
* as a deleted row's whose words a row happens to read: what it is for is a text coming back.
|
|
189
|
+
*
|
|
190
|
+
* @param {object|undefined} at - the `@` a pull kept
|
|
191
|
+
* @param {object} localized - the section's `content`: the source doc and each translated language's
|
|
192
|
+
* @param {object} files
|
|
193
|
+
* @param {string} files.sourceLocale
|
|
194
|
+
* @param {object} files.translations - `{ locale: { hash: entry } }`
|
|
195
|
+
* @param {object} files.context - the section's translation context
|
|
196
|
+
* @param {Set<string>} [files.freeform] - languages whose whole document is written by hand, so no row is the file's
|
|
197
|
+
* @returns {object|undefined}
|
|
198
|
+
*/
|
|
199
|
+
function mapWithFlags(at, localized, { sourceLocale, translations, context, freeform = null }) {
|
|
200
|
+
const place = contextKey(context)
|
|
201
|
+
const rows = blockElements(localized[sourceLocale])
|
|
202
|
+
let map = at?.translations
|
|
203
|
+
const writable = () => (map === at?.translations ? (map = JSON.parse(JSON.stringify(map || {}))) : map)
|
|
204
|
+
for (const [locale, table] of Object.entries(translations || {})) {
|
|
205
|
+
if (locale === sourceLocale || freeform?.has(locale) || !isProseMirrorDoc(localized[locale])) continue
|
|
206
|
+
const read = blockElements(localized[locale])
|
|
207
|
+
if (read.length !== rows.length) continue
|
|
208
|
+
const keys = rows.map((row) => elementText(row))
|
|
209
|
+
const olderAt = (key) => (key ? table[computeHash(key)]?.outdated?.[place] : undefined)
|
|
210
|
+
const claimed = new Set(keys.map(olderAt))
|
|
211
|
+
keys.forEach((key, i) => {
|
|
212
|
+
const older = olderAt(key)
|
|
213
|
+
if (typeof older === 'string' && older !== key && !map?.[older]?.[locale]) (writable()[older] ||= {})[locale] = read[i].content
|
|
214
|
+
})
|
|
215
|
+
keys.forEach((key, i) => {
|
|
216
|
+
if (!key || olderAt(key) || translationFor(table[computeHash(key)], context) === undefined) return
|
|
217
|
+
const says = elementText(read[i])
|
|
218
|
+
for (const [older, byLocale] of Object.entries(map || {})) {
|
|
219
|
+
if (older === key || claimed.has(older) || !Array.isArray(byLocale?.[locale]) || !isSmallEdit(older, key)) continue
|
|
220
|
+
if (elementText({ content: byLocale[locale] }) !== says) continue
|
|
221
|
+
delete writable()[older][locale]
|
|
222
|
+
if (Object.keys(map[older]).length === 0) delete map[older]
|
|
223
|
+
}
|
|
224
|
+
})
|
|
225
|
+
}
|
|
226
|
+
if (map === at?.translations) return at
|
|
227
|
+
const { translations: _map, ...rest } = at || {}
|
|
228
|
+
const settled = Object.keys(map).length ? { ...at, translations: map } : rest
|
|
229
|
+
return Object.keys(settled).length ? settled : undefined
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Write what a pull read to `locales/_kept.json`, removing the file when nothing is kept. An entry
|
|
234
|
+
* keeps its place and the pull adds nothing it did not change, as `writeLocaleTranslations` does.
|
|
235
|
+
*
|
|
236
|
+
* @param {string} siteRoot
|
|
237
|
+
* @param {object} kept - the collector's `kept`
|
|
238
|
+
* @param {string} [subdir]
|
|
239
|
+
* @param {Set<string>|null} [read] - the places this pull read, and only these change. None: every
|
|
240
|
+
* place is the document's to say, for a lane whose document is the whole site.
|
|
241
|
+
* @returns {'updated'|'unchanged'}
|
|
242
|
+
*/
|
|
243
|
+
export function writeKept(siteRoot, kept, subdir = '', read = null) {
|
|
244
|
+
const filePath = join(localesDir(siteRoot, subdir), KEPT_FILE)
|
|
245
|
+
let text = null
|
|
246
|
+
try {
|
|
247
|
+
text = readFileSync(filePath, 'utf8')
|
|
248
|
+
} catch {
|
|
249
|
+
// absent
|
|
250
|
+
}
|
|
251
|
+
const existing = text ? readKept(siteRoot, subdir) : {}
|
|
252
|
+
const order = text ? topLevelKeyOrder(text).filter((key) => key in existing) : []
|
|
253
|
+
const entries = []
|
|
254
|
+
for (const key of order) {
|
|
255
|
+
if (key in kept) entries.push([key, canonicalJson(kept[key]) === canonicalJson(existing[key]) ? existing[key] : kept[key]])
|
|
256
|
+
else if (read && !read.has(key)) entries.push([key, existing[key]])
|
|
257
|
+
}
|
|
258
|
+
for (const key of Object.keys(kept).filter((k) => !order.includes(k)).sort()) entries.push([key, kept[key]])
|
|
259
|
+
if (entries.length === 0) {
|
|
260
|
+
if (text === null) return 'unchanged'
|
|
261
|
+
rmSync(filePath)
|
|
262
|
+
return 'updated'
|
|
263
|
+
}
|
|
264
|
+
const next = jsonObjectText(entries)
|
|
265
|
+
if (next === text) return 'unchanged'
|
|
266
|
+
mkdirSync(dirname(filePath), { recursive: true })
|
|
267
|
+
writeFileSync(filePath, next)
|
|
268
|
+
return 'updated'
|
|
269
|
+
}
|
|
270
|
+
|
|
133
271
|
/**
|
|
134
272
|
* Wrap a section's source-locale content DOC into its localized `content` form: the
|
|
135
273
|
* source locale stays the doc, and EACH TARGET LOCALE BECOMES A SELF-CONTAINED DOC
|
|
@@ -303,12 +441,25 @@ export function isLocalizedContent(value) {
|
|
|
303
441
|
* (source text hashed exactly as `i18n extract` keys it). Non-string or missing
|
|
304
442
|
* values are skipped (only hashable scalars belong in the hash manifest).
|
|
305
443
|
*
|
|
306
|
-
* With `languages` (`documentLanguages`), a value in any other language is left out
|
|
307
|
-
* language is named in `outside` for the caller to report
|
|
444
|
+
* With `languages` (`documentLanguages`), a value in any other language is left out of the
|
|
445
|
+
* translation files, and the language is named in `outside` for the caller to report. A whole document
|
|
446
|
+
* in one is kept (`keepLanguage`, `readKept`), so a push sends it back.
|
|
308
447
|
*/
|
|
309
448
|
export function createTranslationCollector(sourceLocale, { siteRoot = null, languages = null } = {}) {
|
|
310
449
|
const byLocale = {} // { locale: { hash: target } }
|
|
311
450
|
const outside = new Set() // the languages read that the site does not have
|
|
451
|
+
const kept = {} // { place: { '@'?, languages? } }, what `readKept` gives back
|
|
452
|
+
const keptRead = new Set() // the places read, whether or not they keep anything
|
|
453
|
+
const marks = { cleared: {}, outdated: {} } // what `addStructuralMap` was told of each row beyond its value
|
|
454
|
+
|
|
455
|
+
function keepMeta(place, at) {
|
|
456
|
+
keptRead.add(place)
|
|
457
|
+
if (at && typeof at === 'object' && Object.keys(at).length > 0) (kept[place] ||= {})['@'] = at
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function keepLanguage(place, locale, doc) {
|
|
461
|
+
if (isProseMirrorDoc(doc)) ((kept[place] ||= {}).languages ||= {})[locale] = doc
|
|
462
|
+
}
|
|
312
463
|
|
|
313
464
|
function has(locale) {
|
|
314
465
|
if (!languages || languages.includes(locale)) return true
|
|
@@ -359,8 +510,16 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null, lang
|
|
|
359
510
|
// context-specific overrides (`writeLocaleTranslations`). ⛔ Until 2026-09-26 the last section
|
|
360
511
|
// read won, and a pull wrote that one string over the author's `{ default, overrides }`.
|
|
361
512
|
// `checks` — what says each element's translation (`deriveStructuralMap`); a value as read, else.
|
|
362
|
-
|
|
513
|
+
// `notes` — its rows that hold their source, and those that hold an older source's translation.
|
|
514
|
+
function addStructuralMap(locale, map, context = null, checks = null, notes = null) {
|
|
363
515
|
if (locale === sourceLocale || !map || typeof map !== 'object' || Array.isArray(map) || !has(locale)) return
|
|
516
|
+
const place = context ? contextKey(context) : ''
|
|
517
|
+
for (const [src, row] of Object.entries(notes?.cleared || {})) {
|
|
518
|
+
;((marks.cleared[locale] ||= {})[computeHash(src)] ||= {})[place] = row
|
|
519
|
+
}
|
|
520
|
+
for (const [src, older] of Object.entries(notes?.outdated || {})) {
|
|
521
|
+
;((marks.outdated[locale] ||= {})[computeHash(src)] ||= {})[place] = older
|
|
522
|
+
}
|
|
364
523
|
const table = (byLocale[locale] ||= {})
|
|
365
524
|
for (const [src, target] of Object.entries(map)) {
|
|
366
525
|
if (typeof src !== 'string' || typeof target !== 'string') continue
|
|
@@ -401,7 +560,7 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null, lang
|
|
|
401
560
|
return null
|
|
402
561
|
}
|
|
403
562
|
|
|
404
|
-
return { add, addStructuralMap, noteFreeform, keptFreeform, has, byLocale, says, freeformPending, outside }
|
|
563
|
+
return { add, addStructuralMap, noteFreeform, keptFreeform, keepMeta, keepLanguage, has, byLocale, says, marks, freeformPending, outside, kept, keptRead }
|
|
405
564
|
}
|
|
406
565
|
|
|
407
566
|
/**
|
|
@@ -416,13 +575,22 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null, lang
|
|
|
416
575
|
* inline markdown (serializeInlineContent), so emphasis, inline icons, and
|
|
417
576
|
* per-locale link hrefs survive. Keyed by the source element's cleaned text, so
|
|
418
577
|
* `computeHash(key)` matches the extractor's unit hash — closing the round trip.
|
|
578
|
+
*
|
|
579
|
+
* ⭐ `notes` hears what a value alone can't say. `cleared`: a row holding its source, so that source's
|
|
580
|
+
* entry goes, and a translation an author cleared stays cleared. `outdated`: a row holding what the
|
|
581
|
+
* app keeps as an older source text's translation (`olderSource`). It is still what the row reads, so
|
|
582
|
+
* it stays in the map, and the caller records it as outdated rather than as this source's.
|
|
583
|
+
*
|
|
584
|
+
* ⭐ TWO ROWS OF ONE SOURCE TEXT THAT READ DIFFERENTLY cannot be said by one entry, so the target is
|
|
585
|
+
* free-form. ⛔ Until 2026-10-31 the last one won: `(zh_A, zh_B)` became `(zh_B, zh_B)` on the next push.
|
|
419
586
|
*/
|
|
420
|
-
function deriveStructuralMap(sourceDoc, targetDoc, modelFor = () => null, checks = {}) {
|
|
587
|
+
function deriveStructuralMap(sourceDoc, targetDoc, modelFor = () => null, checks = {}, notes = { cleared: {}, outdated: {} }, olderSource = () => null) {
|
|
421
588
|
if (!isProseMirrorDoc(sourceDoc) || !isProseMirrorDoc(targetDoc)) return null
|
|
422
589
|
const src = blockElements(sourceDoc)
|
|
423
590
|
const tgt = blockElements(targetDoc)
|
|
424
591
|
if (src.length !== tgt.length) return null // structurally divergent → free-form
|
|
425
592
|
const map = {}
|
|
593
|
+
const read = new Map()
|
|
426
594
|
try {
|
|
427
595
|
for (let i = 0; i < src.length; i++) {
|
|
428
596
|
if (src[i].type !== tgt[i].type) return null // divergent → free-form
|
|
@@ -430,9 +598,18 @@ function deriveStructuralMap(sourceDoc, targetDoc, modelFor = () => null, checks
|
|
|
430
598
|
if (!key) continue
|
|
431
599
|
const srcMd = serializeInlineContent(src[i].content || [])
|
|
432
600
|
const tgtMd = serializeInlineContent(tgt[i].content || [])
|
|
433
|
-
|
|
434
|
-
|
|
601
|
+
const says = srcMd === tgtMd ? null : labelsAlone(src[i].content || [], tgt[i].content || [], tgtMd) ?? tgtMd
|
|
602
|
+
if (read.has(key) && read.get(key) !== says) return null // one entry cannot say both → free-form
|
|
603
|
+
read.set(key, says)
|
|
604
|
+
if (says === null) {
|
|
605
|
+
// unchanged → untranslated → omit, and no entry says otherwise
|
|
606
|
+
notes.cleared[key] = { markdown: srcMd, says: saysElement(src[i].content || [], tgtMd) }
|
|
607
|
+
continue
|
|
608
|
+
}
|
|
609
|
+
map[key] = says
|
|
435
610
|
checks[key] = saysElement(src[i].content || [], tgtMd)
|
|
611
|
+
const older = olderSource(tgt[i], key)
|
|
612
|
+
if (older) notes.outdated[key] = older
|
|
436
613
|
}
|
|
437
614
|
} catch {
|
|
438
615
|
// a mark we can't serialize → don't risk a lossy map; store as free-form
|
|
@@ -608,19 +785,28 @@ function addDataBlockStrings(map, sourceDoc, targetDoc, modelFor = () => null, c
|
|
|
608
785
|
* map still on the wire (legacy / transition) passes through unchanged. The
|
|
609
786
|
* reserved `@` (and `$`-prefixed) key is opaque metadata — NEVER a locale. A bare
|
|
610
787
|
* doc (no locale wrap) is returned unchanged.
|
|
788
|
+
*
|
|
789
|
+
* `place` names what is kept for this field (`readKept`): the translation's place, unless a record's
|
|
790
|
+
* field needs its own.
|
|
611
791
|
*/
|
|
612
|
-
export function unwrapLocalizedContent(content, sourceLocale, collector, freeformRelPath, freeformCandidates = null, context = null, modelFor = () => null) {
|
|
792
|
+
export function unwrapLocalizedContent(content, sourceLocale, collector, freeformRelPath, freeformCandidates = null, context = null, modelFor = () => null, place = context ? contextKey(context) : null) {
|
|
613
793
|
if (!isLocalizedContent(content)) return content
|
|
614
794
|
const source = content[sourceLocale]
|
|
795
|
+
if (place) collector?.keepMeta?.(place, content['@'])
|
|
615
796
|
for (const [locale, value] of Object.entries(content)) {
|
|
616
797
|
if (locale === sourceLocale) continue
|
|
617
798
|
if (locale === '@' || locale.startsWith('$')) continue // reserved metadata, not a locale
|
|
799
|
+
if (collector?.has && !collector.has(locale)) {
|
|
800
|
+
if (place) collector.keepLanguage(place, locale, value)
|
|
801
|
+
continue
|
|
802
|
+
}
|
|
618
803
|
if (isProseMirrorDoc(value)) {
|
|
619
804
|
// A translation the site keeps as a free-form file stays one, even when it lines up.
|
|
620
805
|
const kept = collector?.keptFreeform?.(locale, freeformCandidates || freeformRelPath) || null
|
|
621
806
|
const checks = {}
|
|
622
|
-
const
|
|
623
|
-
|
|
807
|
+
const notes = { cleared: {}, outdated: {} }
|
|
808
|
+
const map = kept ? null : deriveStructuralMap(source, value, modelFor, checks, notes, olderSourceOf(content['@'], locale))
|
|
809
|
+
if (map) collector?.addStructuralMap?.(locale, map, context, checks, notes)
|
|
624
810
|
else collector?.noteFreeform?.(locale, value, source, kept || freeformRelPath)
|
|
625
811
|
} else {
|
|
626
812
|
collector?.addStructuralMap?.(locale, value, context)
|
|
@@ -629,6 +815,26 @@ export function unwrapLocalizedContent(content, sourceLocale, collector, freefor
|
|
|
629
815
|
return source
|
|
630
816
|
}
|
|
631
817
|
|
|
818
|
+
// ⭐ Which older source text a row's translation was made from, by what the app keeps in `@.translations`
|
|
819
|
+
// (`{ <source text>: { <lang>: <inline content> } }`): a row is outdated when its translation is one of
|
|
820
|
+
// those under a similar text, the most like it if several (`isSmallEdit`, the editor's own test). Compared
|
|
821
|
+
// as text, the key's own terms: a push's link addresses are not the store's.
|
|
822
|
+
function olderSourceOf(at, locale) {
|
|
823
|
+
const held = at?.translations
|
|
824
|
+
if (!held || typeof held !== 'object') return () => null
|
|
825
|
+
const byText = new Map()
|
|
826
|
+
for (const [older, byLocale] of Object.entries(held)) {
|
|
827
|
+
const inline = byLocale?.[locale]
|
|
828
|
+
if (!Array.isArray(inline)) continue
|
|
829
|
+
const text = elementText({ content: inline })
|
|
830
|
+
byText.set(text, [...(byText.get(text) || []), older])
|
|
831
|
+
}
|
|
832
|
+
return (element, key) => {
|
|
833
|
+
const similar = (byText.get(elementText(element)) || []).filter((older) => older !== key && isSmallEdit(older, key))
|
|
834
|
+
return similar.sort((x, y) => wordOverlap(y, key) - wordOverlap(x, key))[0] ?? null
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
632
838
|
// A translation read in several sections: what each section's context says, and what a place with
|
|
633
839
|
// no context (a page title) says. Reconciled with the author's entry when the file is written.
|
|
634
840
|
class SeenInContexts {
|
|
@@ -795,6 +1001,75 @@ function jsonObjectText(entries) {
|
|
|
795
1001
|
return `{\n${lines.join(',\n')}\n}\n`
|
|
796
1002
|
}
|
|
797
1003
|
|
|
1004
|
+
/**
|
|
1005
|
+
* ⭐ THE ENTRY A TRANSLATION FILE HOLDS for what a pull read of one source text, in one language.
|
|
1006
|
+
*
|
|
1007
|
+
* - What the places read, as the author keeps it (`reconcileTranslation`).
|
|
1008
|
+
* - `cleared`: places whose row holds its source. With nothing else read the entry goes, so a
|
|
1009
|
+
* cleared translation stays cleared; beside a place that reads a translation, an override says so.
|
|
1010
|
+
* ⛔ Until 2026-10-31 the entry stayed, and the next push brought the old translation back.
|
|
1011
|
+
* - `outdated`: places whose row holds an older source text's translation, `{ place: older text }`. It
|
|
1012
|
+
* is still what they read, so the entry gives it to a build and a push, and `uniweb i18n status`
|
|
1013
|
+
* lists it. ⛔ Until 2026-10-31 it read as this source's translation, and nothing could say otherwise.
|
|
1014
|
+
*
|
|
1015
|
+
* @returns {string|object|undefined} undefined: no entry
|
|
1016
|
+
*/
|
|
1017
|
+
function settleEntry(existing, seen, check, cleared, outdated) {
|
|
1018
|
+
let own = existing
|
|
1019
|
+
if (own && typeof own === 'object' && 'outdated' in own) {
|
|
1020
|
+
const { outdated: _flag, ...rest } = own
|
|
1021
|
+
own = rest
|
|
1022
|
+
}
|
|
1023
|
+
const rows = Object.entries(cleared || {})
|
|
1024
|
+
const places = rows.map(([place]) => place).filter(Boolean)
|
|
1025
|
+
const reads = seen instanceof SeenInContexts ? seen : seen === undefined ? new SeenInContexts() : new SeenInContexts(seen)
|
|
1026
|
+
// Whether the author's entry, pushed to these rows, gives them what they hold: their source. That is
|
|
1027
|
+
// no translation cleared, in whatever spelling it is written. `Pricing` is the label of `[Pricing](/pricing)`.
|
|
1028
|
+
const ownSaysThem = (value) => typeof value === 'string' && rows.every(([, row]) => row.says(value))
|
|
1029
|
+
let entry
|
|
1030
|
+
if (reads.plain !== undefined || Object.keys(reads.values).length > 0) {
|
|
1031
|
+
entry = reconcileTranslation(own, reads, check)
|
|
1032
|
+
const readsElsewhere = typeof entry === 'string' ? entry : entry?.default
|
|
1033
|
+
if (places.length && !ownSaysThem(readsElsewhere)) {
|
|
1034
|
+
const overrides = { ...(entry && typeof entry === 'object' ? entry.overrides : {}) }
|
|
1035
|
+
for (const [place, row] of rows) if (place) overrides[place] = row.markdown
|
|
1036
|
+
entry = { default: readsElsewhere, overrides }
|
|
1037
|
+
}
|
|
1038
|
+
} else if (ownSaysThem(own)) {
|
|
1039
|
+
entry = own
|
|
1040
|
+
} else if (own && typeof own === 'object') {
|
|
1041
|
+
// Its default and any overrides here go. An override for a place this pull did not see stays.
|
|
1042
|
+
const rest = Object.fromEntries(Object.entries(own.overrides || {}).filter(([place]) => !places.includes(place)))
|
|
1043
|
+
const defaultStays = typeof own.default === 'string' && ownSaysThem(own.default)
|
|
1044
|
+
entry = defaultStays || Object.keys(rest).length ? { ...(defaultStays ? { default: own.default } : {}), ...(Object.keys(rest).length ? { overrides: rest } : {}) } : undefined
|
|
1045
|
+
}
|
|
1046
|
+
if (entry === undefined) return undefined
|
|
1047
|
+
if (outdated && Object.keys(outdated).length) {
|
|
1048
|
+
entry = { ...(typeof entry === 'string' ? { default: entry } : entry), outdated }
|
|
1049
|
+
}
|
|
1050
|
+
if (entry && typeof entry === 'object' && Object.keys(entry).length === 1 && typeof entry.default === 'string') return entry.default
|
|
1051
|
+
return entry
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
/**
|
|
1055
|
+
* Add entries to a language's translation file: one it holds changes in its place, and new ones follow,
|
|
1056
|
+
* sorted, as `writeLocaleTranslations` writes them.
|
|
1057
|
+
*
|
|
1058
|
+
* @param {string} siteRoot
|
|
1059
|
+
* @param {string} locale
|
|
1060
|
+
* @param {{ [hash: string]: object|string }} entries
|
|
1061
|
+
* @param {string} [subdir] - `records` for record translations
|
|
1062
|
+
*/
|
|
1063
|
+
export function addTranslationEntries(siteRoot, locale, entries, subdir = '') {
|
|
1064
|
+
const filePath = localeFilePath(siteRoot, locale, subdir)
|
|
1065
|
+
const text = readFileSync(filePath, 'utf8')
|
|
1066
|
+
const parsed = JSON.parse(text)
|
|
1067
|
+
const merged = [...new Set([...topLevelKeyOrder(text).filter((key) => key in parsed), ...Object.keys(parsed)])]
|
|
1068
|
+
.map((key) => [key, key in entries ? entries[key] : parsed[key]])
|
|
1069
|
+
for (const key of Object.keys(entries).filter((k) => !(k in parsed)).sort()) merged.push([key, entries[key]])
|
|
1070
|
+
writeFileSync(filePath, jsonObjectText(merged))
|
|
1071
|
+
}
|
|
1072
|
+
|
|
798
1073
|
/**
|
|
799
1074
|
* Merge `{ locale: { hash: target } }` into each `locales/{locale}.json`,
|
|
800
1075
|
* preserving existing entries (translations for strings not in this projection are
|
|
@@ -807,12 +1082,17 @@ function jsonObjectText(entries) {
|
|
|
807
1082
|
* @param {string} [subdir] - `records` for record translations
|
|
808
1083
|
* @param {object} [says] - the collector's `says`: what says each value read, in its place
|
|
809
1084
|
* (`reconcileTranslation`). Without it, a value says only itself.
|
|
1085
|
+
* @param {object} [marks] - the collector's `marks`: the rows that hold their source, and those
|
|
1086
|
+
* that hold an older source's translation (`settleEntry`)
|
|
810
1087
|
* @returns {{ [locale: string]: 'updated' | 'unchanged' }}
|
|
811
1088
|
*/
|
|
812
|
-
export function writeLocaleTranslations(siteRoot, byLocale, subdir = '', says = null) {
|
|
1089
|
+
export function writeLocaleTranslations(siteRoot, byLocale, subdir = '', says = null, marks = null) {
|
|
813
1090
|
const report = {}
|
|
814
|
-
for (
|
|
815
|
-
|
|
1091
|
+
for (const locale of new Set([...Object.keys(byLocale || {}), ...Object.keys(marks?.cleared || {})])) {
|
|
1092
|
+
const read = byLocale?.[locale] || {}
|
|
1093
|
+
const cleared = marks?.cleared?.[locale] || {}
|
|
1094
|
+
const outdated = marks?.outdated?.[locale] || {}
|
|
1095
|
+
if (Object.keys(read).length === 0 && Object.keys(cleared).length === 0) continue
|
|
816
1096
|
const filePath = localeFilePath(siteRoot, locale, subdir)
|
|
817
1097
|
|
|
818
1098
|
let existing = {}
|
|
@@ -837,22 +1117,22 @@ export function writeLocaleTranslations(siteRoot, byLocale, subdir = '', says =
|
|
|
837
1117
|
// A translation read in several sections becomes the string or the `{ default, overrides }` the
|
|
838
1118
|
// author keeps (`reconcileTranslation`) — and one read in one place too, which keeps an entry
|
|
839
1119
|
// that says it in another spelling, and an override for a place the pull did not see.
|
|
840
|
-
entries =
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
])
|
|
845
|
-
|
|
846
|
-
const fresh = Object.keys(entries).filter((key) => !(key in existing)).sort()
|
|
847
|
-
|
|
848
|
-
|
|
1120
|
+
const entries = {}
|
|
1121
|
+
for (const key of new Set([...Object.keys(read), ...Object.keys(cleared)])) {
|
|
1122
|
+
// Nothing to say of a row that holds its source where the file has no entry for it, and no place reads one.
|
|
1123
|
+
if (!(key in read) && !(key in existing)) continue
|
|
1124
|
+
entries[key] = settleEntry(existing[key], read[key], says?.[locale]?.[key], cleared[key], outdated[key])
|
|
1125
|
+
}
|
|
1126
|
+
const fresh = Object.keys(entries).filter((key) => !(key in existing) && entries[key] !== undefined).sort()
|
|
1127
|
+
const same = (key) => (entries[key] === undefined ? !(key in existing) : canonicalJson(existing[key]) === canonicalJson(entries[key]))
|
|
1128
|
+
if (fresh.length === 0 && Object.keys(entries).every(same)) {
|
|
1129
|
+
if (existsSync(filePath)) report[locale] = 'unchanged'
|
|
849
1130
|
continue
|
|
850
1131
|
}
|
|
851
1132
|
// An entry the pull did not change keeps its value as written — key order included.
|
|
852
|
-
const merged = [...new Set([...order, ...Object.keys(existing)])]
|
|
853
|
-
key
|
|
854
|
-
key in entries && canonicalJson(entries[key]) !== canonicalJson(existing[key]) ? entries[key] : existing[key]
|
|
855
|
-
])
|
|
1133
|
+
const merged = [...new Set([...order, ...Object.keys(existing)])]
|
|
1134
|
+
.filter((key) => !(key in entries) || entries[key] !== undefined)
|
|
1135
|
+
.map((key) => [key, key in entries && canonicalJson(entries[key]) !== canonicalJson(existing[key]) ? entries[key] : existing[key]])
|
|
856
1136
|
for (const key of fresh) merged.push([key, entries[key]])
|
|
857
1137
|
const next = jsonObjectText(merged)
|
|
858
1138
|
|
|
@@ -55,7 +55,7 @@ import { labelForFile } from '../site/entry-label.js'
|
|
|
55
55
|
import { LINK_MODEL, writeLinkRecords } from './link-records.js'
|
|
56
56
|
import { FILE_MODEL, filesToPull } from './file-records.js'
|
|
57
57
|
import { parseBibtex } from '@citestyle/bibtex'
|
|
58
|
-
import { createTranslationCollector, writeLocaleTranslations, writeFreeformTranslations } from './locale-sync.js'
|
|
58
|
+
import { createTranslationCollector, writeLocaleTranslations, writeFreeformTranslations, writeKept } from './locale-sync.js'
|
|
59
59
|
import { siteSourceLocale } from './source-locale.js'
|
|
60
60
|
import { buildFreeformRecordPath } from '../i18n/freeform.js'
|
|
61
61
|
|
|
@@ -900,8 +900,10 @@ export function recordsToProject({ folderDoc, recordDocs = [], siteRoot, opts =
|
|
|
900
900
|
|
|
901
901
|
// Flush localized record-field translations to locales/records/{locale}.json,
|
|
902
902
|
// and any prosemirror free-form body overrides to locales/freeform/{locale}/.
|
|
903
|
-
const locales = writeLocaleTranslations(siteRoot, collector.byLocale, 'records', collector.says)
|
|
903
|
+
const locales = writeLocaleTranslations(siteRoot, collector.byLocale, 'records', collector.says, collector.marks)
|
|
904
904
|
const freeform = writeFreeformTranslations(siteRoot, collector.freeformPending)
|
|
905
|
+
// A record this pull did not read keeps what is kept for it.
|
|
906
|
+
writeKept(siteRoot, collector.kept, 'records', collector.keptRead)
|
|
905
907
|
|
|
906
908
|
if (opts.backend && Object.keys(learned).length) {
|
|
907
909
|
updateBackendMap(siteRoot, opts.backend, 'records', learned)
|
package/src/uwx/records.js
CHANGED
|
@@ -66,7 +66,7 @@ import { isUuid } from './uuid.js'
|
|
|
66
66
|
import { markdownToProseMirror } from '@uniweb/content-reader'
|
|
67
67
|
import { LOCALIZED_FIELD_ASSUMPTION } from './localize.js'
|
|
68
68
|
import { siteSourceLocale } from './source-locale.js'
|
|
69
|
-
import { localizeScalar, localizeScalarList, localizeContentDoc, loadLocaleTranslations, discoverLocales, discoverFreeformLocales, localesDir, isLocalizedContent } from './locale-sync.js'
|
|
69
|
+
import { readKept, restoreKept, recordFieldPlace, localizeScalar, localizeScalarList, localizeContentDoc, loadLocaleTranslations, discoverLocales, discoverFreeformLocales, localesDir, isLocalizedContent } from './locale-sync.js'
|
|
70
70
|
import { loadFreeformRecord } from '../i18n/freeform.js'
|
|
71
71
|
import { isDraftRecord } from '../site/record-draft.js'
|
|
72
72
|
import { LINK_MODEL, linkRecordRefusal } from './link-records.js'
|
|
@@ -172,7 +172,7 @@ export function entityContentHash(document) {
|
|
|
172
172
|
// ⭐ `context` is the record the value is in — `{ key: '<pool>/<handle>' }`, the identity its
|
|
173
173
|
// translations are kept under (`i18n/records.js`) — so an entry that reads differently in some
|
|
174
174
|
// records (`{ default, overrides }`) is sent as each one renders it.
|
|
175
|
-
function encodeFieldValue(value, field, sourceLocale, translations, context = null) {
|
|
175
|
+
function encodeFieldValue(value, field, sourceLocale, translations, context = null, heldFor = null) {
|
|
176
176
|
if (value == null) return value
|
|
177
177
|
if (isProseMirrorField(field)) {
|
|
178
178
|
// markdown source → ProseMirror doc. When localized, wrap per-locale exactly
|
|
@@ -185,7 +185,8 @@ function encodeFieldValue(value, field, sourceLocale, translations, context = nu
|
|
|
185
185
|
// localized field MUST ride as a `{ lang: value }` map on the wire — the
|
|
186
186
|
// schema-driven projector drops a localized field whose value isn't a map — so
|
|
187
187
|
// wrap the source doc, consistent with localizeScalar (which always wraps).
|
|
188
|
-
|
|
188
|
+
// And what a pull kept for this field goes back with it (`readKept`).
|
|
189
|
+
return restoreKept(isLocalizedContent(localized) ? localized : { [sourceLocale]: localized }, heldFor, { sourceLocale, translations, context })
|
|
189
190
|
}
|
|
190
191
|
if (field.localized) {
|
|
191
192
|
// A list of localized values rides as a list of `{ lang: value }` maps, one per
|
|
@@ -249,6 +250,7 @@ export function recordsToEntities({
|
|
|
249
250
|
sourceLocale = LOCALIZED_FIELD_ASSUMPTION.defaultSourceLocale,
|
|
250
251
|
translations,
|
|
251
252
|
refs = null,
|
|
253
|
+
kept = null,
|
|
252
254
|
}) {
|
|
253
255
|
if (!declaration || !declaration.name) {
|
|
254
256
|
throw new Error('uwx/records: a declaration with a name is required')
|
|
@@ -299,14 +301,14 @@ export function recordsToEntities({
|
|
|
299
301
|
const hasBody = typeof record.$body === 'string' && record.$body.trim() !== ''
|
|
300
302
|
|
|
301
303
|
const context = { key: `${label}/${slug}` } // as the build keys the record's translations
|
|
302
|
-
const enc = encoder(sourceLocale, translations, refs, context)
|
|
304
|
+
const enc = encoder(sourceLocale, translations, refs, context, kept)
|
|
303
305
|
const sections = layout.flat ? readFlat(layout, record, enc) : readBySection(layout, record, enc)
|
|
304
306
|
// The markdown body fills the content body field, in whatever section declares it,
|
|
305
307
|
// unless the file already sets that field.
|
|
306
308
|
if (hasBody && bodyTarget) {
|
|
307
309
|
const data = (sections[bodyTarget.section] ??= {})
|
|
308
310
|
if (data[bodyTarget.key] == null) {
|
|
309
|
-
data[bodyTarget.key] = encodeFieldValue(record.$body, bodyTarget.field, sourceLocale, translations, context)
|
|
311
|
+
data[bodyTarget.key] = encodeFieldValue(record.$body, bodyTarget.field, sourceLocale, translations, context, kept?.[recordFieldPlace(context, bodyTarget.key)])
|
|
310
312
|
}
|
|
311
313
|
}
|
|
312
314
|
// The brief is the card, and is always sent — an empty one included, so a required
|
|
@@ -389,8 +391,8 @@ export function recordsToEntities({
|
|
|
389
391
|
// What one record's read collects beside the values: the keys with no place in the
|
|
390
392
|
// schema, the keys written flat in a schema written by section, and values of the
|
|
391
393
|
// wrong shape for their section.
|
|
392
|
-
function encoder(sourceLocale, translations, refs = null, context = null) {
|
|
393
|
-
return { sourceLocale, translations, refs, context, undeclared: [], misplaced: [], shape: [], unresolved: [], namesNew: 0 }
|
|
394
|
+
function encoder(sourceLocale, translations, refs = null, context = null, kept = null) {
|
|
395
|
+
return { sourceLocale, translations, refs, context, kept, undeclared: [], misplaced: [], shape: [], unresolved: [], namesNew: 0 }
|
|
394
396
|
}
|
|
395
397
|
|
|
396
398
|
// A flat record: its keys, minus the record's own, are the one section's fields.
|
|
@@ -489,7 +491,7 @@ function encodeRecord(fields, value, enc, prefix, reserved) {
|
|
|
489
491
|
? encodeSection(field, v, enc, `${prefix}${key}`)
|
|
490
492
|
: field?.type === 'entity_ref'
|
|
491
493
|
? encodeReference(v, field, enc, `${prefix}${key}`)
|
|
492
|
-
: encodeFieldValue(v, field, enc.sourceLocale, enc.translations, enc.context)
|
|
494
|
+
: encodeFieldValue(v, field, enc.sourceLocale, enc.translations, enc.context, enc.kept?.[recordFieldPlace(enc.context, key)])
|
|
493
495
|
if (encoded !== undefined) out[key] = encoded
|
|
494
496
|
}
|
|
495
497
|
for (const key of Object.keys(value)) {
|
|
@@ -1041,6 +1043,8 @@ export async function buildRecordEntities(siteRoot, opts = {}) {
|
|
|
1041
1043
|
].filter((l) => l !== sourceLocale)
|
|
1042
1044
|
const translations =
|
|
1043
1045
|
targetLocales.length > 0 ? loadLocaleTranslations(siteRoot, targetLocales, 'records') : null
|
|
1046
|
+
// What a pull kept for this push to send back (`readKept`).
|
|
1047
|
+
const kept = readKept(siteRoot, 'records')
|
|
1044
1048
|
|
|
1045
1049
|
// ⭐ THE FILE'S `$uuid` IS OURS; THE WIRE'S IS THE BACKEND'S (2026-09-20).
|
|
1046
1050
|
//
|
|
@@ -1209,6 +1213,7 @@ export async function buildRecordEntities(siteRoot, opts = {}) {
|
|
|
1209
1213
|
sourceLocale,
|
|
1210
1214
|
translations,
|
|
1211
1215
|
refs,
|
|
1216
|
+
kept,
|
|
1212
1217
|
})
|
|
1213
1218
|
// Free-form per-locale body overrides (a full localized doc beats the structural
|
|
1214
1219
|
// map) — only meaningful for a multi-locale site with a prosemirror content field.
|
package/src/uwx/site-diff.js
CHANGED
|
@@ -132,6 +132,23 @@ export function walkSiteUnits(doc, cb, sourceLocale = documentSourceLocale(doc))
|
|
|
132
132
|
}
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
+
/**
|
|
136
|
+
* The sections of a document that differ from a base in its own representation: this copy's edits since
|
|
137
|
+
* the last sync, read against `unitBases.local`. A section the base does not name counts as changed.
|
|
138
|
+
*
|
|
139
|
+
* @param {object} doc
|
|
140
|
+
* @param {Object<string,string>} [base] - per-unit hashes (`computeUnitHashes`)
|
|
141
|
+
* @param {string} [sourceLocale]
|
|
142
|
+
* @returns {Set<object>} the section records of `doc`
|
|
143
|
+
*/
|
|
144
|
+
export function sectionsChangedSince(doc, base = {}, sourceLocale = documentSourceLocale(doc)) {
|
|
145
|
+
const changed = new Set()
|
|
146
|
+
walkSiteUnits(doc, (path, record, kind) => {
|
|
147
|
+
if (kind === 'section' && entityContentHash(ownContent(record, '$children')) !== base[path]) changed.add(record)
|
|
148
|
+
}, sourceLocale)
|
|
149
|
+
return changed
|
|
150
|
+
}
|
|
151
|
+
|
|
135
152
|
/** Per-unit content hashes: `{ <path>: <sha256> }`. */
|
|
136
153
|
export function computeUnitHashes(doc, sourceLocale) {
|
|
137
154
|
const out = {}
|
package/src/uwx/site-project.js
CHANGED
|
@@ -52,7 +52,7 @@ import { buildRouteOf } from './site.js'
|
|
|
52
52
|
import { DEFAULT_SECTION_TYPE } from '@uniweb/core/section-type'
|
|
53
53
|
import { translationContext, SITE_META_CONTEXT } from '../i18n/extract.js'
|
|
54
54
|
import { authorableDeclaration, DECLARATION_KEYS } from '../site/fetch-shapes.js'
|
|
55
|
-
import { createTranslationCollector, writeLocaleTranslations, writeFreeformTranslations, unwrapLocalizedContent, isProseMirrorDoc } from './locale-sync.js'
|
|
55
|
+
import { createTranslationCollector, writeLocaleTranslations, writeFreeformTranslations, writeKept, unwrapLocalizedContent, isProseMirrorDoc } from './locale-sync.js'
|
|
56
56
|
import { unwritableNodes } from '@uniweb/content-writer'
|
|
57
57
|
import { buildFreeformPath, freeformPathsFor } from '../i18n/freeform.js'
|
|
58
58
|
import { unwrapLocalized, unwrapLocalizedList } from './backfill.js'
|
|
@@ -1448,9 +1448,11 @@ export function siteContentDocumentToProject({ document, siteRoot, backend = nul
|
|
|
1448
1448
|
}
|
|
1449
1449
|
projectLayout(document?.layout_sections, layoutBaseDir, report, prune, ctx)
|
|
1450
1450
|
|
|
1451
|
-
report.locales = writeLocaleTranslations(siteRoot, collector.byLocale, '', collector.says)
|
|
1451
|
+
report.locales = writeLocaleTranslations(siteRoot, collector.byLocale, '', collector.says, collector.marks)
|
|
1452
1452
|
// Target-locale FREE-FORM bodies → locales/freeform/{locale}/<relpath> + manifest.
|
|
1453
1453
|
report.freeform = writeFreeformTranslations(siteRoot, collector.freeformPending)
|
|
1454
|
+
// What the files hold no place for, kept for the push to send back: this document is the whole site.
|
|
1455
|
+
report.kept = writeKept(siteRoot, collector.kept)
|
|
1454
1456
|
report.outsideLanguages = [...collector.outside].sort()
|
|
1455
1457
|
writePullIndex(siteRoot, ctx.newIndex)
|
|
1456
1458
|
return report
|
package/src/uwx/site.js
CHANGED
|
@@ -81,11 +81,12 @@ import { resolveDefaultLocale, validateLanguageConfig, queryDataUrl } from '@uni
|
|
|
81
81
|
import { emitEntitySyncPackage } from './entity-document.js'
|
|
82
82
|
import { locateSectionMedia, siteTree } from './located-media.js'
|
|
83
83
|
import { mapPageRefs, pageRefIndex, pageRefPages, storedPageRef } from './page-refs.js'
|
|
84
|
-
import { loadLocaleTranslations, localizeScalar, localizeScalarList, localizeContentDoc, localesDir, isLocalizedContent, plainTextTargets } from './locale-sync.js'
|
|
84
|
+
import { readKept, restoreKept, contextKey, loadLocaleTranslations, localizeScalar, localizeScalarList, localizeContentDoc, localesDir, isLocalizedContent, isProseMirrorDoc, plainTextTargets } from './locale-sync.js'
|
|
85
85
|
import { unwrapLocalized } from './backfill.js'
|
|
86
86
|
import { loadFreeformTranslation } from '../i18n/freeform.js'
|
|
87
87
|
import { resolveLocaleList } from '../i18n/locales.js'
|
|
88
|
-
import { translationContext, SITE_META_CONTEXT } from '../i18n/extract.js'
|
|
88
|
+
import { translationContext, SITE_META_CONTEXT, blockElements, elementText } from '../i18n/extract.js'
|
|
89
|
+
import { documentSourceLocale } from './source-locale.js'
|
|
89
90
|
import { splitOpenGraph, openGraphValue, seoBesideOpenGraph } from './open-graph.js'
|
|
90
91
|
import { isAuthoredPreview } from './authored-preview.js'
|
|
91
92
|
import { updateBackendState, readBackendState } from './sync-store.js'
|
|
@@ -207,6 +208,8 @@ export function buildRouteOf(record, parentRoute, sourceLocale) {
|
|
|
207
208
|
// hits the filesystem.
|
|
208
209
|
async function localizeContentTree(pages, layoutSections, sourceLocale, targetLocales, translations, siteRoot) {
|
|
209
210
|
const freeformBase = localesDir(siteRoot)
|
|
211
|
+
// What a pull kept for this push to send back (`readKept`).
|
|
212
|
+
const kept = readKept(siteRoot)
|
|
210
213
|
// What each section type declares its data blocks are, so a block is pushed as the build translates it.
|
|
211
214
|
const dataModel = await dataBlockModels(siteRoot)
|
|
212
215
|
|
|
@@ -216,7 +219,9 @@ async function localizeContentTree(pages, layoutSections, sourceLocale, targetLo
|
|
|
216
219
|
// Resolved in the section's context, as the build resolves it (`translationContext`): a
|
|
217
220
|
// context-specific override applies here and nowhere else. ⛔ Until 2026-09-26 every section was
|
|
218
221
|
// resolved with none, so an override never reached a backend.
|
|
219
|
-
|
|
222
|
+
const context = translationContext({ stableId }, page?.route)
|
|
223
|
+
let localized = localizeContentDoc(record.content, sourceLocale, targetLocales, translations, context, (tag) => dataModel(record.type, tag))
|
|
224
|
+
const freeform = new Set()
|
|
220
225
|
if (page) {
|
|
221
226
|
const section = { stableId }
|
|
222
227
|
for (const locale of targetLocales) {
|
|
@@ -227,9 +232,10 @@ async function localizeContentTree(pages, layoutSections, sourceLocale, targetLo
|
|
|
227
232
|
// override (isLocalizedContent excludes a PM doc, which is also an object).
|
|
228
233
|
if (!isLocalizedContent(localized)) localized = { [sourceLocale]: record.content }
|
|
229
234
|
localized[locale] = body // free-form full body overrides the structural map
|
|
235
|
+
freeform.add(locale)
|
|
230
236
|
}
|
|
231
237
|
}
|
|
232
|
-
record.content = localized
|
|
238
|
+
record.content = restoreKept(localized, kept[contextKey(context)], { sourceLocale, translations, context, freeform })
|
|
233
239
|
}
|
|
234
240
|
|
|
235
241
|
const visitSections = async (sections, page) => {
|
|
@@ -256,6 +262,43 @@ async function localizeContentTree(pages, layoutSections, sourceLocale, targetLo
|
|
|
256
262
|
for (const s of layoutSections || []) await localizeSection(s, { route: layoutAreaRoute(s.layout_name, s.area || s.stable_id) })
|
|
257
263
|
}
|
|
258
264
|
|
|
265
|
+
/**
|
|
266
|
+
* Each section's rows in a site-content document, ours or a backend's, by the place its translations
|
|
267
|
+
* apply (`translationContext`): the text of its block elements in the source language, as extraction
|
|
268
|
+
* keys them. The walk `localizeContentTree` makes.
|
|
269
|
+
*
|
|
270
|
+
* @param {object} doc
|
|
271
|
+
* @param {string} [sourceLocale]
|
|
272
|
+
* @param {(record: object) => boolean} [only] - the sections to read
|
|
273
|
+
* @returns {{ [place: string]: string[] }}
|
|
274
|
+
*/
|
|
275
|
+
export function documentRows(doc, sourceLocale = documentSourceLocale(doc), only = () => true) {
|
|
276
|
+
const rows = {}
|
|
277
|
+
const add = (record, route) => {
|
|
278
|
+
if (!only(record)) return
|
|
279
|
+
const source = isLocalizedContent(record.content) ? record.content[sourceLocale] : record.content
|
|
280
|
+
if (!isProseMirrorDoc(source)) return
|
|
281
|
+
const texts = blockElements(source).map((element) => elementText(element)).filter(Boolean)
|
|
282
|
+
if (texts.length) rows[contextKey(translationContext({ stableId: record.stable_id || record.$id }, route))] = texts
|
|
283
|
+
}
|
|
284
|
+
const visitSections = (sections, route) => {
|
|
285
|
+
for (const s of sections || []) {
|
|
286
|
+
add(s, route)
|
|
287
|
+
visitSections(s.$children, route)
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
const visitPages = (pages, parentRoute) => {
|
|
291
|
+
for (const p of pages || []) {
|
|
292
|
+
const route = buildRouteOf(p, parentRoute, sourceLocale)
|
|
293
|
+
visitSections(p.page_sections, route)
|
|
294
|
+
visitPages(p.$children, route)
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
visitPages(doc?.pages, '/')
|
|
298
|
+
for (const s of doc?.layout_sections || []) add(s, layoutAreaRoute(s.layout_name, s.area || s.stable_id))
|
|
299
|
+
return rows
|
|
300
|
+
}
|
|
301
|
+
|
|
259
302
|
// ⭐ A page's title, label, description and keywords are resolved in the page's context, as the
|
|
260
303
|
// build translates them (`i18n/merge.js::translatePageMeta`, `<route>:_meta`): an override for one
|
|
261
304
|
// page applies there and nowhere else. ⛔ Until 2026-09-26 they were resolved with no page, so an
|
package/src/uwx/sync-package.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
// changed records. An untouched site with records pushes nothing on either
|
|
20
20
|
// lane (the idempotent no-op).
|
|
21
21
|
|
|
22
|
+
import { basename } from 'node:path'
|
|
22
23
|
import { buildRecordEntities, entityContentHash } from './records.js'
|
|
23
24
|
import { recordItemLists, hasListItems, stampRecordItems } from './record-items.js'
|
|
24
25
|
import { ASSET_SLOTS } from '@uniweb/semantic-parser'
|
|
@@ -459,14 +460,11 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
|
|
|
459
460
|
let itemIdentity = null
|
|
460
461
|
if (siteDoc && opts.itemUuids && typeof opts.itemUuids === 'object') {
|
|
461
462
|
itemIdentity = stampUnitUuids(siteDoc, opts.itemUuids, sourceLocale)
|
|
462
|
-
for (const path of itemIdentity.collisions) {
|
|
463
|
-
warnings.push(
|
|
464
|
-
`Two sections resolve to the same file (${path}) — their stable ids collide ` +
|
|
465
|
-
`(e.g. \`1-name.md\` and \`name.md\`). Only the first keeps its identity, and a pull ` +
|
|
466
|
-
`would collapse them into one file. Rename one.`
|
|
467
|
-
)
|
|
468
|
-
}
|
|
469
463
|
}
|
|
464
|
+
// ⭐ TWO SECTIONS OF ONE PAGE WITH ONE ID ARE REFUSED, on a first push too. ⛔ Until 2026-10-31 it was
|
|
465
|
+
// a warning and the push went on. Measured with `1-hero.md` beside `hero.md`: the store held both, a
|
|
466
|
+
// pull deleted one of the files, and from then on a push was refused as stale with nothing to pull.
|
|
467
|
+
const sameId = siteDoc ? (itemIdentity ?? stampUnitUuids(siteDoc, {}, sourceLocale)).collisions : []
|
|
470
468
|
|
|
471
469
|
// Local-media over push (Slice 5). `assetRewrite` ({ '/images/x.png': serveUrl })
|
|
472
470
|
// is supplied by the deploy's SECOND emit, after it has uploaded the files the
|
|
@@ -589,6 +587,12 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
|
|
|
589
587
|
// by `$ref`, and the backend refused a reference to a record the package does not carry
|
|
590
588
|
// (measured 2026-09-26); sent, it would be a second record. A pull writes it back.
|
|
591
589
|
const refusals = [...(col.refusals || [])]
|
|
590
|
+
for (const path of sameId) {
|
|
591
|
+
refusals.push(
|
|
592
|
+
`${path}: two sections of this page have the id \`${basename(path, '.md')}\`, as \`1-name.md\` and \`name.md\` do. ` +
|
|
593
|
+
'A section is named by its file, without the number. Rename one.'
|
|
594
|
+
)
|
|
595
|
+
}
|
|
592
596
|
for (const r of recordChanged) {
|
|
593
597
|
if (r.changed || r.entity.document.$uuid) continue
|
|
594
598
|
refusals.push(
|