@uniweb/build 1.13.2 → 1.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/i18n/audit.js +8 -1
- package/src/i18n/index.js +20 -2
- package/src/i18n/locales.js +2 -1
- package/src/i18n/outdated.js +146 -0
- package/src/i18n/records.js +2 -1
- package/src/i18n/sync.js +21 -0
- package/src/i18n/word-overlap.js +34 -0
- package/src/site/plugin.js +5 -3
- package/src/uwx/backfill.js +4 -4
- package/src/uwx/index.js +3 -0
- package/src/uwx/locale-sync.js +320 -29
- package/src/uwx/records-project.js +4 -2
- package/src/uwx/records.js +13 -8
- package/src/uwx/site-diff.js +34 -12
- package/src/uwx/site-project.js +66 -49
- package/src/uwx/site.js +47 -4
- package/src/uwx/source-locale.js +21 -0
- package/src/uwx/sync-package.js +11 -7
package/src/uwx/locale-sync.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
// surface, used by the projector (pull → files) and, later, the producer
|
|
17
17
|
// (files → push).
|
|
18
18
|
|
|
19
|
-
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'node:fs'
|
|
19
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs'
|
|
20
20
|
import { join, dirname } from 'node:path'
|
|
21
21
|
import yaml from 'js-yaml'
|
|
22
22
|
import { YAML_OPTIONS } from '../utils/yaml-schema.js'
|
|
@@ -29,6 +29,8 @@ import { ASSET_SLOTS } from '@uniweb/semantic-parser'
|
|
|
29
29
|
import { computeSourceHash, manifestText } from '../i18n/freeform-manifest.js'
|
|
30
30
|
import { sameMarkdownDocument, canonicalJson } from './same-content.js'
|
|
31
31
|
import { parseFrontmatter } from '../utils/frontmatter.js'
|
|
32
|
+
import { KEPT_FILE, RESERVED_LOCALE_FILES } from '../i18n/locales.js'
|
|
33
|
+
import { isSmallEdit, wordOverlap } from '../i18n/word-overlap.js'
|
|
32
34
|
|
|
33
35
|
const FREEFORM_MANIFEST = '.manifest.json'
|
|
34
36
|
|
|
@@ -92,7 +94,7 @@ export function discoverLocales(siteRoot, subdir = '') {
|
|
|
92
94
|
if (!existsSync(dir)) return []
|
|
93
95
|
try {
|
|
94
96
|
return readdirSync(dir)
|
|
95
|
-
.filter((f) => f.endsWith('.json') && f
|
|
97
|
+
.filter((f) => f.endsWith('.json') && !RESERVED_LOCALE_FILES.has(f) && !f.startsWith('.'))
|
|
96
98
|
.map((f) => f.slice(0, -'.json'.length))
|
|
97
99
|
} catch {
|
|
98
100
|
return []
|
|
@@ -130,6 +132,142 @@ export function loadLocaleTranslations(siteRoot, locales, subdir = '') {
|
|
|
130
132
|
return out
|
|
131
133
|
}
|
|
132
134
|
|
|
135
|
+
/**
|
|
136
|
+
* ⭐ WHAT A PULL KEEPS FOR A PUSH TO SEND BACK, `locales/_kept.json`: for a section, its `@` (what the
|
|
137
|
+
* app holds beside its translations, opaque here) and its documents in languages the site does not
|
|
138
|
+
* list (a language removed in the app keeps them, so adding it back loses nothing). A file has no
|
|
139
|
+
* other place for either, and THE ROUND-TRIP LAW holds for both. Absent on almost every site.
|
|
140
|
+
*
|
|
141
|
+
* ⛔ Until 2026-10-31 a pull dropped both, and the next push of the section removed them from the store.
|
|
142
|
+
*
|
|
143
|
+
* Keyed as a translation's place is (`contextKey`), since a section's identity on a backend is no use
|
|
144
|
+
* to another backend's copy.
|
|
145
|
+
*
|
|
146
|
+
* @param {string} siteRoot
|
|
147
|
+
* @param {string} [subdir] - `records` for record translations
|
|
148
|
+
* @returns {{ [place: string]: { '@'?: object, languages?: object } }}
|
|
149
|
+
*/
|
|
150
|
+
export function readKept(siteRoot, subdir = '') {
|
|
151
|
+
try {
|
|
152
|
+
const parsed = JSON.parse(readFileSync(join(localesDir(siteRoot, subdir), KEPT_FILE), 'utf8'))
|
|
153
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {}
|
|
154
|
+
} catch {
|
|
155
|
+
return {}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Where a record's localized field is kept: a record has more than one such field. */
|
|
160
|
+
export function recordFieldPlace(context, field) {
|
|
161
|
+
return `${contextKey(context)}#${field}`
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* A localized `content` with what the site keeps for it: never over a language the push has now.
|
|
166
|
+
*
|
|
167
|
+
* @param {object} localized
|
|
168
|
+
* @param {object} [held] - what `readKept` gives for it
|
|
169
|
+
* @param {object} [files] - what its translation files say (`mapWithFlags`)
|
|
170
|
+
*/
|
|
171
|
+
export function restoreKept(localized, held, files = null) {
|
|
172
|
+
if (!isLocalizedContent(localized)) return localized
|
|
173
|
+
const at = files ? mapWithFlags(held?.['@'], localized, files) : held?.['@']
|
|
174
|
+
if (!held && !at) return localized
|
|
175
|
+
const out = { ...localized }
|
|
176
|
+
for (const [locale, doc] of Object.entries(held?.languages || {})) {
|
|
177
|
+
if (!(locale in out)) out[locale] = doc
|
|
178
|
+
}
|
|
179
|
+
if (at) out['@'] = at
|
|
180
|
+
return out
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* ⭐ WHICH ROWS ARE OUTDATED IS THE TRANSLATION FILES' TO SAY, and the app reads it from `@.translations`:
|
|
185
|
+
* a row is outdated when its translation is kept there under a similar text (`olderSourceOf`). So a flagged
|
|
186
|
+
* row goes with its entry there, and a row its file does not flag goes without one. An entry already there
|
|
187
|
+
* stays as it is: a push's link addresses are not the store's. Nor is one touched that flags nothing, such
|
|
188
|
+
* as a deleted row's whose words a row happens to read: what it is for is a text coming back.
|
|
189
|
+
*
|
|
190
|
+
* @param {object|undefined} at - the `@` a pull kept
|
|
191
|
+
* @param {object} localized - the section's `content`: the source doc and each translated language's
|
|
192
|
+
* @param {object} files
|
|
193
|
+
* @param {string} files.sourceLocale
|
|
194
|
+
* @param {object} files.translations - `{ locale: { hash: entry } }`
|
|
195
|
+
* @param {object} files.context - the section's translation context
|
|
196
|
+
* @param {Set<string>} [files.freeform] - languages whose whole document is written by hand, so no row is the file's
|
|
197
|
+
* @returns {object|undefined}
|
|
198
|
+
*/
|
|
199
|
+
function mapWithFlags(at, localized, { sourceLocale, translations, context, freeform = null }) {
|
|
200
|
+
const place = contextKey(context)
|
|
201
|
+
const rows = blockElements(localized[sourceLocale])
|
|
202
|
+
let map = at?.translations
|
|
203
|
+
const writable = () => (map === at?.translations ? (map = JSON.parse(JSON.stringify(map || {}))) : map)
|
|
204
|
+
for (const [locale, table] of Object.entries(translations || {})) {
|
|
205
|
+
if (locale === sourceLocale || freeform?.has(locale) || !isProseMirrorDoc(localized[locale])) continue
|
|
206
|
+
const read = blockElements(localized[locale])
|
|
207
|
+
if (read.length !== rows.length) continue
|
|
208
|
+
const keys = rows.map((row) => elementText(row))
|
|
209
|
+
const olderAt = (key) => (key ? table[computeHash(key)]?.outdated?.[place] : undefined)
|
|
210
|
+
const claimed = new Set(keys.map(olderAt))
|
|
211
|
+
keys.forEach((key, i) => {
|
|
212
|
+
const older = olderAt(key)
|
|
213
|
+
if (typeof older === 'string' && older !== key && !map?.[older]?.[locale]) (writable()[older] ||= {})[locale] = read[i].content
|
|
214
|
+
})
|
|
215
|
+
keys.forEach((key, i) => {
|
|
216
|
+
if (!key || olderAt(key) || translationFor(table[computeHash(key)], context) === undefined) return
|
|
217
|
+
const says = elementText(read[i])
|
|
218
|
+
for (const [older, byLocale] of Object.entries(map || {})) {
|
|
219
|
+
if (older === key || claimed.has(older) || !Array.isArray(byLocale?.[locale]) || !isSmallEdit(older, key)) continue
|
|
220
|
+
if (elementText({ content: byLocale[locale] }) !== says) continue
|
|
221
|
+
delete writable()[older][locale]
|
|
222
|
+
if (Object.keys(map[older]).length === 0) delete map[older]
|
|
223
|
+
}
|
|
224
|
+
})
|
|
225
|
+
}
|
|
226
|
+
if (map === at?.translations) return at
|
|
227
|
+
const { translations: _map, ...rest } = at || {}
|
|
228
|
+
const settled = Object.keys(map).length ? { ...at, translations: map } : rest
|
|
229
|
+
return Object.keys(settled).length ? settled : undefined
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Write what a pull read to `locales/_kept.json`, removing the file when nothing is kept. An entry
|
|
234
|
+
* keeps its place and the pull adds nothing it did not change, as `writeLocaleTranslations` does.
|
|
235
|
+
*
|
|
236
|
+
* @param {string} siteRoot
|
|
237
|
+
* @param {object} kept - the collector's `kept`
|
|
238
|
+
* @param {string} [subdir]
|
|
239
|
+
* @param {Set<string>|null} [read] - the places this pull read, and only these change. None: every
|
|
240
|
+
* place is the document's to say, for a lane whose document is the whole site.
|
|
241
|
+
* @returns {'updated'|'unchanged'}
|
|
242
|
+
*/
|
|
243
|
+
export function writeKept(siteRoot, kept, subdir = '', read = null) {
|
|
244
|
+
const filePath = join(localesDir(siteRoot, subdir), KEPT_FILE)
|
|
245
|
+
let text = null
|
|
246
|
+
try {
|
|
247
|
+
text = readFileSync(filePath, 'utf8')
|
|
248
|
+
} catch {
|
|
249
|
+
// absent
|
|
250
|
+
}
|
|
251
|
+
const existing = text ? readKept(siteRoot, subdir) : {}
|
|
252
|
+
const order = text ? topLevelKeyOrder(text).filter((key) => key in existing) : []
|
|
253
|
+
const entries = []
|
|
254
|
+
for (const key of order) {
|
|
255
|
+
if (key in kept) entries.push([key, canonicalJson(kept[key]) === canonicalJson(existing[key]) ? existing[key] : kept[key]])
|
|
256
|
+
else if (read && !read.has(key)) entries.push([key, existing[key]])
|
|
257
|
+
}
|
|
258
|
+
for (const key of Object.keys(kept).filter((k) => !order.includes(k)).sort()) entries.push([key, kept[key]])
|
|
259
|
+
if (entries.length === 0) {
|
|
260
|
+
if (text === null) return 'unchanged'
|
|
261
|
+
rmSync(filePath)
|
|
262
|
+
return 'updated'
|
|
263
|
+
}
|
|
264
|
+
const next = jsonObjectText(entries)
|
|
265
|
+
if (next === text) return 'unchanged'
|
|
266
|
+
mkdirSync(dirname(filePath), { recursive: true })
|
|
267
|
+
writeFileSync(filePath, next)
|
|
268
|
+
return 'updated'
|
|
269
|
+
}
|
|
270
|
+
|
|
133
271
|
/**
|
|
134
272
|
* Wrap a section's source-locale content DOC into its localized `content` form: the
|
|
135
273
|
* source locale stays the doc, and EACH TARGET LOCALE BECOMES A SELF-CONTAINED DOC
|
|
@@ -302,9 +440,32 @@ export function isLocalizedContent(value) {
|
|
|
302
440
|
* `add(localizedValue)` records, for each non-source locale, `hash(source) → target`
|
|
303
441
|
* (source text hashed exactly as `i18n extract` keys it). Non-string or missing
|
|
304
442
|
* values are skipped (only hashable scalars belong in the hash manifest).
|
|
443
|
+
*
|
|
444
|
+
* With `languages` (`documentLanguages`), a value in any other language is left out of the
|
|
445
|
+
* translation files, and the language is named in `outside` for the caller to report. A whole document
|
|
446
|
+
* in one is kept (`keepLanguage`, `readKept`), so a push sends it back.
|
|
305
447
|
*/
|
|
306
|
-
export function createTranslationCollector(sourceLocale, { siteRoot = null } = {}) {
|
|
448
|
+
export function createTranslationCollector(sourceLocale, { siteRoot = null, languages = null } = {}) {
|
|
307
449
|
const byLocale = {} // { locale: { hash: target } }
|
|
450
|
+
const outside = new Set() // the languages read that the site does not have
|
|
451
|
+
const kept = {} // { place: { '@'?, languages? } }, what `readKept` gives back
|
|
452
|
+
const keptRead = new Set() // the places read, whether or not they keep anything
|
|
453
|
+
const marks = { cleared: {}, outdated: {} } // what `addStructuralMap` was told of each row beyond its value
|
|
454
|
+
|
|
455
|
+
function keepMeta(place, at) {
|
|
456
|
+
keptRead.add(place)
|
|
457
|
+
if (at && typeof at === 'object' && Object.keys(at).length > 0) (kept[place] ||= {})['@'] = at
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function keepLanguage(place, locale, doc) {
|
|
461
|
+
if (isProseMirrorDoc(doc)) ((kept[place] ||= {}).languages ||= {})[locale] = doc
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function has(locale) {
|
|
465
|
+
if (!languages || languages.includes(locale)) return true
|
|
466
|
+
outside.add(locale)
|
|
467
|
+
return false
|
|
468
|
+
}
|
|
308
469
|
// ⭐ What says each value read, in the place it was read — the push's own rule there — so the
|
|
309
470
|
// writer keeps an author's entry that says the same thing in another spelling (`reconcileTranslation`).
|
|
310
471
|
const says = {} // { locale: { hash: { plain?: (value) => boolean, values: { [contextKey]: (value) => boolean } } } }
|
|
@@ -328,7 +489,7 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null } = {
|
|
|
328
489
|
const hash = computeHash(source)
|
|
329
490
|
for (const [locale, value] of Object.entries(localizedValue)) {
|
|
330
491
|
if (locale === sourceLocale) continue
|
|
331
|
-
if (typeof value !== 'string') continue
|
|
492
|
+
if (typeof value !== 'string' || !has(locale)) continue
|
|
332
493
|
const table = (byLocale[locale] ||= {})
|
|
333
494
|
noteSays(locale, hash, context, plainText ? saysAsText(value) : saysAsIs(value))
|
|
334
495
|
if (!context) {
|
|
@@ -349,8 +510,16 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null } = {
|
|
|
349
510
|
// context-specific overrides (`writeLocaleTranslations`). ⛔ Until 2026-09-26 the last section
|
|
350
511
|
// read won, and a pull wrote that one string over the author's `{ default, overrides }`.
|
|
351
512
|
// `checks` — what says each element's translation (`deriveStructuralMap`); a value as read, else.
|
|
352
|
-
|
|
353
|
-
|
|
513
|
+
// `notes` — its rows that hold their source, and those that hold an older source's translation.
|
|
514
|
+
function addStructuralMap(locale, map, context = null, checks = null, notes = null) {
|
|
515
|
+
if (locale === sourceLocale || !map || typeof map !== 'object' || Array.isArray(map) || !has(locale)) return
|
|
516
|
+
const place = context ? contextKey(context) : ''
|
|
517
|
+
for (const [src, row] of Object.entries(notes?.cleared || {})) {
|
|
518
|
+
;((marks.cleared[locale] ||= {})[computeHash(src)] ||= {})[place] = row
|
|
519
|
+
}
|
|
520
|
+
for (const [src, older] of Object.entries(notes?.outdated || {})) {
|
|
521
|
+
;((marks.outdated[locale] ||= {})[computeHash(src)] ||= {})[place] = older
|
|
522
|
+
}
|
|
354
523
|
const table = (byLocale[locale] ||= {})
|
|
355
524
|
for (const [src, target] of Object.entries(map)) {
|
|
356
525
|
if (typeof src !== 'string' || typeof target !== 'string') continue
|
|
@@ -373,6 +542,7 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null } = {
|
|
|
373
542
|
// — null when the caller can't place it (e.g. layout sections have no freeform
|
|
374
543
|
// home), so it's surfaced rather than dropped.
|
|
375
544
|
function noteFreeform(locale, target, source, relpath) {
|
|
545
|
+
if (!has(locale)) return
|
|
376
546
|
freeformPending.push({ locale, target, source, relpath: relpath || null })
|
|
377
547
|
}
|
|
378
548
|
|
|
@@ -390,7 +560,7 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null } = {
|
|
|
390
560
|
return null
|
|
391
561
|
}
|
|
392
562
|
|
|
393
|
-
return { add, addStructuralMap, noteFreeform, keptFreeform, byLocale, says, freeformPending }
|
|
563
|
+
return { add, addStructuralMap, noteFreeform, keptFreeform, keepMeta, keepLanguage, has, byLocale, says, marks, freeformPending, outside, kept, keptRead }
|
|
394
564
|
}
|
|
395
565
|
|
|
396
566
|
/**
|
|
@@ -405,13 +575,22 @@ export function createTranslationCollector(sourceLocale, { siteRoot = null } = {
|
|
|
405
575
|
* inline markdown (serializeInlineContent), so emphasis, inline icons, and
|
|
406
576
|
* per-locale link hrefs survive. Keyed by the source element's cleaned text, so
|
|
407
577
|
* `computeHash(key)` matches the extractor's unit hash — closing the round trip.
|
|
578
|
+
*
|
|
579
|
+
* ⭐ `notes` hears what a value alone can't say. `cleared`: a row holding its source, so that source's
|
|
580
|
+
* entry goes, and a translation an author cleared stays cleared. `outdated`: a row holding what the
|
|
581
|
+
* app keeps as an older source text's translation (`olderSource`). It is still what the row reads, so
|
|
582
|
+
* it stays in the map, and the caller records it as outdated rather than as this source's.
|
|
583
|
+
*
|
|
584
|
+
* ⭐ TWO ROWS OF ONE SOURCE TEXT THAT READ DIFFERENTLY cannot be said by one entry, so the target is
|
|
585
|
+
* free-form. ⛔ Until 2026-10-31 the last one won: `(zh_A, zh_B)` became `(zh_B, zh_B)` on the next push.
|
|
408
586
|
*/
|
|
409
|
-
function deriveStructuralMap(sourceDoc, targetDoc, modelFor = () => null, checks = {}) {
|
|
587
|
+
function deriveStructuralMap(sourceDoc, targetDoc, modelFor = () => null, checks = {}, notes = { cleared: {}, outdated: {} }, olderSource = () => null) {
|
|
410
588
|
if (!isProseMirrorDoc(sourceDoc) || !isProseMirrorDoc(targetDoc)) return null
|
|
411
589
|
const src = blockElements(sourceDoc)
|
|
412
590
|
const tgt = blockElements(targetDoc)
|
|
413
591
|
if (src.length !== tgt.length) return null // structurally divergent → free-form
|
|
414
592
|
const map = {}
|
|
593
|
+
const read = new Map()
|
|
415
594
|
try {
|
|
416
595
|
for (let i = 0; i < src.length; i++) {
|
|
417
596
|
if (src[i].type !== tgt[i].type) return null // divergent → free-form
|
|
@@ -419,9 +598,18 @@ function deriveStructuralMap(sourceDoc, targetDoc, modelFor = () => null, checks
|
|
|
419
598
|
if (!key) continue
|
|
420
599
|
const srcMd = serializeInlineContent(src[i].content || [])
|
|
421
600
|
const tgtMd = serializeInlineContent(tgt[i].content || [])
|
|
422
|
-
|
|
423
|
-
|
|
601
|
+
const says = srcMd === tgtMd ? null : labelsAlone(src[i].content || [], tgt[i].content || [], tgtMd) ?? tgtMd
|
|
602
|
+
if (read.has(key) && read.get(key) !== says) return null // one entry cannot say both → free-form
|
|
603
|
+
read.set(key, says)
|
|
604
|
+
if (says === null) {
|
|
605
|
+
// unchanged → untranslated → omit, and no entry says otherwise
|
|
606
|
+
notes.cleared[key] = { markdown: srcMd, says: saysElement(src[i].content || [], tgtMd) }
|
|
607
|
+
continue
|
|
608
|
+
}
|
|
609
|
+
map[key] = says
|
|
424
610
|
checks[key] = saysElement(src[i].content || [], tgtMd)
|
|
611
|
+
const older = olderSource(tgt[i], key)
|
|
612
|
+
if (older) notes.outdated[key] = older
|
|
425
613
|
}
|
|
426
614
|
} catch {
|
|
427
615
|
// a mark we can't serialize → don't risk a lossy map; store as free-form
|
|
@@ -597,19 +785,28 @@ function addDataBlockStrings(map, sourceDoc, targetDoc, modelFor = () => null, c
|
|
|
597
785
|
* map still on the wire (legacy / transition) passes through unchanged. The
|
|
598
786
|
* reserved `@` (and `$`-prefixed) key is opaque metadata — NEVER a locale. A bare
|
|
599
787
|
* doc (no locale wrap) is returned unchanged.
|
|
788
|
+
*
|
|
789
|
+
* `place` names what is kept for this field (`readKept`): the translation's place, unless a record's
|
|
790
|
+
* field needs its own.
|
|
600
791
|
*/
|
|
601
|
-
export function unwrapLocalizedContent(content, sourceLocale, collector, freeformRelPath, freeformCandidates = null, context = null, modelFor = () => null) {
|
|
792
|
+
export function unwrapLocalizedContent(content, sourceLocale, collector, freeformRelPath, freeformCandidates = null, context = null, modelFor = () => null, place = context ? contextKey(context) : null) {
|
|
602
793
|
if (!isLocalizedContent(content)) return content
|
|
603
794
|
const source = content[sourceLocale]
|
|
795
|
+
if (place) collector?.keepMeta?.(place, content['@'])
|
|
604
796
|
for (const [locale, value] of Object.entries(content)) {
|
|
605
797
|
if (locale === sourceLocale) continue
|
|
606
798
|
if (locale === '@' || locale.startsWith('$')) continue // reserved metadata, not a locale
|
|
799
|
+
if (collector?.has && !collector.has(locale)) {
|
|
800
|
+
if (place) collector.keepLanguage(place, locale, value)
|
|
801
|
+
continue
|
|
802
|
+
}
|
|
607
803
|
if (isProseMirrorDoc(value)) {
|
|
608
804
|
// A translation the site keeps as a free-form file stays one, even when it lines up.
|
|
609
805
|
const kept = collector?.keptFreeform?.(locale, freeformCandidates || freeformRelPath) || null
|
|
610
806
|
const checks = {}
|
|
611
|
-
const
|
|
612
|
-
|
|
807
|
+
const notes = { cleared: {}, outdated: {} }
|
|
808
|
+
const map = kept ? null : deriveStructuralMap(source, value, modelFor, checks, notes, olderSourceOf(content['@'], locale))
|
|
809
|
+
if (map) collector?.addStructuralMap?.(locale, map, context, checks, notes)
|
|
613
810
|
else collector?.noteFreeform?.(locale, value, source, kept || freeformRelPath)
|
|
614
811
|
} else {
|
|
615
812
|
collector?.addStructuralMap?.(locale, value, context)
|
|
@@ -618,6 +815,26 @@ export function unwrapLocalizedContent(content, sourceLocale, collector, freefor
|
|
|
618
815
|
return source
|
|
619
816
|
}
|
|
620
817
|
|
|
818
|
+
// ⭐ Which older source text a row's translation was made from, by what the app keeps in `@.translations`
|
|
819
|
+
// (`{ <source text>: { <lang>: <inline content> } }`): a row is outdated when its translation is one of
|
|
820
|
+
// those under a similar text, the most like it if several (`isSmallEdit`, the editor's own test). Compared
|
|
821
|
+
// as text, the key's own terms: a push's link addresses are not the store's.
|
|
822
|
+
function olderSourceOf(at, locale) {
|
|
823
|
+
const held = at?.translations
|
|
824
|
+
if (!held || typeof held !== 'object') return () => null
|
|
825
|
+
const byText = new Map()
|
|
826
|
+
for (const [older, byLocale] of Object.entries(held)) {
|
|
827
|
+
const inline = byLocale?.[locale]
|
|
828
|
+
if (!Array.isArray(inline)) continue
|
|
829
|
+
const text = elementText({ content: inline })
|
|
830
|
+
byText.set(text, [...(byText.get(text) || []), older])
|
|
831
|
+
}
|
|
832
|
+
return (element, key) => {
|
|
833
|
+
const similar = (byText.get(elementText(element)) || []).filter((older) => older !== key && isSmallEdit(older, key))
|
|
834
|
+
return similar.sort((x, y) => wordOverlap(y, key) - wordOverlap(x, key))[0] ?? null
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
621
838
|
// A translation read in several sections: what each section's context says, and what a place with
|
|
622
839
|
// no context (a page title) says. Reconciled with the author's entry when the file is written.
|
|
623
840
|
class SeenInContexts {
|
|
@@ -784,6 +1001,75 @@ function jsonObjectText(entries) {
|
|
|
784
1001
|
return `{\n${lines.join(',\n')}\n}\n`
|
|
785
1002
|
}
|
|
786
1003
|
|
|
1004
|
+
/**
|
|
1005
|
+
* ⭐ THE ENTRY A TRANSLATION FILE HOLDS for what a pull read of one source text, in one language.
|
|
1006
|
+
*
|
|
1007
|
+
* - What the places read, as the author keeps it (`reconcileTranslation`).
|
|
1008
|
+
* - `cleared`: places whose row holds its source. With nothing else read the entry goes, so a
|
|
1009
|
+
* cleared translation stays cleared; beside a place that reads a translation, an override says so.
|
|
1010
|
+
* ⛔ Until 2026-10-31 the entry stayed, and the next push brought the old translation back.
|
|
1011
|
+
* - `outdated`: places whose row holds an older source text's translation, `{ place: older text }`. It
|
|
1012
|
+
* is still what they read, so the entry gives it to a build and a push, and `uniweb i18n status`
|
|
1013
|
+
* lists it. ⛔ Until 2026-10-31 it read as this source's translation, and nothing could say otherwise.
|
|
1014
|
+
*
|
|
1015
|
+
* @returns {string|object|undefined} undefined: no entry
|
|
1016
|
+
*/
|
|
1017
|
+
function settleEntry(existing, seen, check, cleared, outdated) {
|
|
1018
|
+
let own = existing
|
|
1019
|
+
if (own && typeof own === 'object' && 'outdated' in own) {
|
|
1020
|
+
const { outdated: _flag, ...rest } = own
|
|
1021
|
+
own = rest
|
|
1022
|
+
}
|
|
1023
|
+
const rows = Object.entries(cleared || {})
|
|
1024
|
+
const places = rows.map(([place]) => place).filter(Boolean)
|
|
1025
|
+
const reads = seen instanceof SeenInContexts ? seen : seen === undefined ? new SeenInContexts() : new SeenInContexts(seen)
|
|
1026
|
+
// Whether the author's entry, pushed to these rows, gives them what they hold: their source. That is
|
|
1027
|
+
// no translation cleared, in whatever spelling it is written. `Pricing` is the label of `[Pricing](/pricing)`.
|
|
1028
|
+
const ownSaysThem = (value) => typeof value === 'string' && rows.every(([, row]) => row.says(value))
|
|
1029
|
+
let entry
|
|
1030
|
+
if (reads.plain !== undefined || Object.keys(reads.values).length > 0) {
|
|
1031
|
+
entry = reconcileTranslation(own, reads, check)
|
|
1032
|
+
const readsElsewhere = typeof entry === 'string' ? entry : entry?.default
|
|
1033
|
+
if (places.length && !ownSaysThem(readsElsewhere)) {
|
|
1034
|
+
const overrides = { ...(entry && typeof entry === 'object' ? entry.overrides : {}) }
|
|
1035
|
+
for (const [place, row] of rows) if (place) overrides[place] = row.markdown
|
|
1036
|
+
entry = { default: readsElsewhere, overrides }
|
|
1037
|
+
}
|
|
1038
|
+
} else if (ownSaysThem(own)) {
|
|
1039
|
+
entry = own
|
|
1040
|
+
} else if (own && typeof own === 'object') {
|
|
1041
|
+
// Its default and any overrides here go. An override for a place this pull did not see stays.
|
|
1042
|
+
const rest = Object.fromEntries(Object.entries(own.overrides || {}).filter(([place]) => !places.includes(place)))
|
|
1043
|
+
const defaultStays = typeof own.default === 'string' && ownSaysThem(own.default)
|
|
1044
|
+
entry = defaultStays || Object.keys(rest).length ? { ...(defaultStays ? { default: own.default } : {}), ...(Object.keys(rest).length ? { overrides: rest } : {}) } : undefined
|
|
1045
|
+
}
|
|
1046
|
+
if (entry === undefined) return undefined
|
|
1047
|
+
if (outdated && Object.keys(outdated).length) {
|
|
1048
|
+
entry = { ...(typeof entry === 'string' ? { default: entry } : entry), outdated }
|
|
1049
|
+
}
|
|
1050
|
+
if (entry && typeof entry === 'object' && Object.keys(entry).length === 1 && typeof entry.default === 'string') return entry.default
|
|
1051
|
+
return entry
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
/**
|
|
1055
|
+
* Add entries to a language's translation file: one it holds changes in its place, and new ones follow,
|
|
1056
|
+
* sorted, as `writeLocaleTranslations` writes them.
|
|
1057
|
+
*
|
|
1058
|
+
* @param {string} siteRoot
|
|
1059
|
+
* @param {string} locale
|
|
1060
|
+
* @param {{ [hash: string]: object|string }} entries
|
|
1061
|
+
* @param {string} [subdir] - `records` for record translations
|
|
1062
|
+
*/
|
|
1063
|
+
export function addTranslationEntries(siteRoot, locale, entries, subdir = '') {
|
|
1064
|
+
const filePath = localeFilePath(siteRoot, locale, subdir)
|
|
1065
|
+
const text = readFileSync(filePath, 'utf8')
|
|
1066
|
+
const parsed = JSON.parse(text)
|
|
1067
|
+
const merged = [...new Set([...topLevelKeyOrder(text).filter((key) => key in parsed), ...Object.keys(parsed)])]
|
|
1068
|
+
.map((key) => [key, key in entries ? entries[key] : parsed[key]])
|
|
1069
|
+
for (const key of Object.keys(entries).filter((k) => !(k in parsed)).sort()) merged.push([key, entries[key]])
|
|
1070
|
+
writeFileSync(filePath, jsonObjectText(merged))
|
|
1071
|
+
}
|
|
1072
|
+
|
|
787
1073
|
/**
|
|
788
1074
|
* Merge `{ locale: { hash: target } }` into each `locales/{locale}.json`,
|
|
789
1075
|
* preserving existing entries (translations for strings not in this projection are
|
|
@@ -796,12 +1082,17 @@ function jsonObjectText(entries) {
|
|
|
796
1082
|
* @param {string} [subdir] - `records` for record translations
|
|
797
1083
|
* @param {object} [says] - the collector's `says`: what says each value read, in its place
|
|
798
1084
|
* (`reconcileTranslation`). Without it, a value says only itself.
|
|
1085
|
+
* @param {object} [marks] - the collector's `marks`: the rows that hold their source, and those
|
|
1086
|
+
* that hold an older source's translation (`settleEntry`)
|
|
799
1087
|
* @returns {{ [locale: string]: 'updated' | 'unchanged' }}
|
|
800
1088
|
*/
|
|
801
|
-
export function writeLocaleTranslations(siteRoot, byLocale, subdir = '', says = null) {
|
|
1089
|
+
export function writeLocaleTranslations(siteRoot, byLocale, subdir = '', says = null, marks = null) {
|
|
802
1090
|
const report = {}
|
|
803
|
-
for (
|
|
804
|
-
|
|
1091
|
+
for (const locale of new Set([...Object.keys(byLocale || {}), ...Object.keys(marks?.cleared || {})])) {
|
|
1092
|
+
const read = byLocale?.[locale] || {}
|
|
1093
|
+
const cleared = marks?.cleared?.[locale] || {}
|
|
1094
|
+
const outdated = marks?.outdated?.[locale] || {}
|
|
1095
|
+
if (Object.keys(read).length === 0 && Object.keys(cleared).length === 0) continue
|
|
805
1096
|
const filePath = localeFilePath(siteRoot, locale, subdir)
|
|
806
1097
|
|
|
807
1098
|
let existing = {}
|
|
@@ -826,22 +1117,22 @@ export function writeLocaleTranslations(siteRoot, byLocale, subdir = '', says =
|
|
|
826
1117
|
// A translation read in several sections becomes the string or the `{ default, overrides }` the
|
|
827
1118
|
// author keeps (`reconcileTranslation`) — and one read in one place too, which keeps an entry
|
|
828
1119
|
// that says it in another spelling, and an override for a place the pull did not see.
|
|
829
|
-
entries =
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
])
|
|
834
|
-
|
|
835
|
-
const fresh = Object.keys(entries).filter((key) => !(key in existing)).sort()
|
|
836
|
-
|
|
837
|
-
|
|
1120
|
+
const entries = {}
|
|
1121
|
+
for (const key of new Set([...Object.keys(read), ...Object.keys(cleared)])) {
|
|
1122
|
+
// Nothing to say of a row that holds its source where the file has no entry for it, and no place reads one.
|
|
1123
|
+
if (!(key in read) && !(key in existing)) continue
|
|
1124
|
+
entries[key] = settleEntry(existing[key], read[key], says?.[locale]?.[key], cleared[key], outdated[key])
|
|
1125
|
+
}
|
|
1126
|
+
const fresh = Object.keys(entries).filter((key) => !(key in existing) && entries[key] !== undefined).sort()
|
|
1127
|
+
const same = (key) => (entries[key] === undefined ? !(key in existing) : canonicalJson(existing[key]) === canonicalJson(entries[key]))
|
|
1128
|
+
if (fresh.length === 0 && Object.keys(entries).every(same)) {
|
|
1129
|
+
if (existsSync(filePath)) report[locale] = 'unchanged'
|
|
838
1130
|
continue
|
|
839
1131
|
}
|
|
840
1132
|
// An entry the pull did not change keeps its value as written — key order included.
|
|
841
|
-
const merged = [...new Set([...order, ...Object.keys(existing)])]
|
|
842
|
-
key
|
|
843
|
-
key in entries && canonicalJson(entries[key]) !== canonicalJson(existing[key]) ? entries[key] : existing[key]
|
|
844
|
-
])
|
|
1133
|
+
const merged = [...new Set([...order, ...Object.keys(existing)])]
|
|
1134
|
+
.filter((key) => !(key in entries) || entries[key] !== undefined)
|
|
1135
|
+
.map((key) => [key, key in entries && canonicalJson(entries[key]) !== canonicalJson(existing[key]) ? entries[key] : existing[key]])
|
|
845
1136
|
for (const key of fresh) merged.push([key, entries[key]])
|
|
846
1137
|
const next = jsonObjectText(merged)
|
|
847
1138
|
|
|
@@ -55,7 +55,7 @@ import { labelForFile } from '../site/entry-label.js'
|
|
|
55
55
|
import { LINK_MODEL, writeLinkRecords } from './link-records.js'
|
|
56
56
|
import { FILE_MODEL, filesToPull } from './file-records.js'
|
|
57
57
|
import { parseBibtex } from '@citestyle/bibtex'
|
|
58
|
-
import { createTranslationCollector, writeLocaleTranslations, writeFreeformTranslations } from './locale-sync.js'
|
|
58
|
+
import { createTranslationCollector, writeLocaleTranslations, writeFreeformTranslations, writeKept } from './locale-sync.js'
|
|
59
59
|
import { siteSourceLocale } from './source-locale.js'
|
|
60
60
|
import { buildFreeformRecordPath } from '../i18n/freeform.js'
|
|
61
61
|
|
|
@@ -900,8 +900,10 @@ export function recordsToProject({ folderDoc, recordDocs = [], siteRoot, opts =
|
|
|
900
900
|
|
|
901
901
|
// Flush localized record-field translations to locales/records/{locale}.json,
|
|
902
902
|
// and any prosemirror free-form body overrides to locales/freeform/{locale}/.
|
|
903
|
-
const locales = writeLocaleTranslations(siteRoot, collector.byLocale, 'records', collector.says)
|
|
903
|
+
const locales = writeLocaleTranslations(siteRoot, collector.byLocale, 'records', collector.says, collector.marks)
|
|
904
904
|
const freeform = writeFreeformTranslations(siteRoot, collector.freeformPending)
|
|
905
|
+
// A record this pull did not read keeps what is kept for it.
|
|
906
|
+
writeKept(siteRoot, collector.kept, 'records', collector.keptRead)
|
|
905
907
|
|
|
906
908
|
if (opts.backend && Object.keys(learned).length) {
|
|
907
909
|
updateBackendMap(siteRoot, opts.backend, 'records', learned)
|
package/src/uwx/records.js
CHANGED
|
@@ -66,7 +66,7 @@ import { isUuid } from './uuid.js'
|
|
|
66
66
|
import { markdownToProseMirror } from '@uniweb/content-reader'
|
|
67
67
|
import { LOCALIZED_FIELD_ASSUMPTION } from './localize.js'
|
|
68
68
|
import { siteSourceLocale } from './source-locale.js'
|
|
69
|
-
import { localizeScalar, localizeScalarList, localizeContentDoc, loadLocaleTranslations, discoverLocales, discoverFreeformLocales, localesDir, isLocalizedContent } from './locale-sync.js'
|
|
69
|
+
import { readKept, restoreKept, recordFieldPlace, localizeScalar, localizeScalarList, localizeContentDoc, loadLocaleTranslations, discoverLocales, discoverFreeformLocales, localesDir, isLocalizedContent } from './locale-sync.js'
|
|
70
70
|
import { loadFreeformRecord } from '../i18n/freeform.js'
|
|
71
71
|
import { isDraftRecord } from '../site/record-draft.js'
|
|
72
72
|
import { LINK_MODEL, linkRecordRefusal } from './link-records.js'
|
|
@@ -172,7 +172,7 @@ export function entityContentHash(document) {
|
|
|
172
172
|
// ⭐ `context` is the record the value is in — `{ key: '<pool>/<handle>' }`, the identity its
|
|
173
173
|
// translations are kept under (`i18n/records.js`) — so an entry that reads differently in some
|
|
174
174
|
// records (`{ default, overrides }`) is sent as each one renders it.
|
|
175
|
-
function encodeFieldValue(value, field, sourceLocale, translations, context = null) {
|
|
175
|
+
function encodeFieldValue(value, field, sourceLocale, translations, context = null, heldFor = null) {
|
|
176
176
|
if (value == null) return value
|
|
177
177
|
if (isProseMirrorField(field)) {
|
|
178
178
|
// markdown source → ProseMirror doc. When localized, wrap per-locale exactly
|
|
@@ -185,7 +185,8 @@ function encodeFieldValue(value, field, sourceLocale, translations, context = nu
|
|
|
185
185
|
// localized field MUST ride as a `{ lang: value }` map on the wire — the
|
|
186
186
|
// schema-driven projector drops a localized field whose value isn't a map — so
|
|
187
187
|
// wrap the source doc, consistent with localizeScalar (which always wraps).
|
|
188
|
-
|
|
188
|
+
// And what a pull kept for this field goes back with it (`readKept`).
|
|
189
|
+
return restoreKept(isLocalizedContent(localized) ? localized : { [sourceLocale]: localized }, heldFor, { sourceLocale, translations, context })
|
|
189
190
|
}
|
|
190
191
|
if (field.localized) {
|
|
191
192
|
// A list of localized values rides as a list of `{ lang: value }` maps, one per
|
|
@@ -249,6 +250,7 @@ export function recordsToEntities({
|
|
|
249
250
|
sourceLocale = LOCALIZED_FIELD_ASSUMPTION.defaultSourceLocale,
|
|
250
251
|
translations,
|
|
251
252
|
refs = null,
|
|
253
|
+
kept = null,
|
|
252
254
|
}) {
|
|
253
255
|
if (!declaration || !declaration.name) {
|
|
254
256
|
throw new Error('uwx/records: a declaration with a name is required')
|
|
@@ -299,14 +301,14 @@ export function recordsToEntities({
|
|
|
299
301
|
const hasBody = typeof record.$body === 'string' && record.$body.trim() !== ''
|
|
300
302
|
|
|
301
303
|
const context = { key: `${label}/${slug}` } // as the build keys the record's translations
|
|
302
|
-
const enc = encoder(sourceLocale, translations, refs, context)
|
|
304
|
+
const enc = encoder(sourceLocale, translations, refs, context, kept)
|
|
303
305
|
const sections = layout.flat ? readFlat(layout, record, enc) : readBySection(layout, record, enc)
|
|
304
306
|
// The markdown body fills the content body field, in whatever section declares it,
|
|
305
307
|
// unless the file already sets that field.
|
|
306
308
|
if (hasBody && bodyTarget) {
|
|
307
309
|
const data = (sections[bodyTarget.section] ??= {})
|
|
308
310
|
if (data[bodyTarget.key] == null) {
|
|
309
|
-
data[bodyTarget.key] = encodeFieldValue(record.$body, bodyTarget.field, sourceLocale, translations, context)
|
|
311
|
+
data[bodyTarget.key] = encodeFieldValue(record.$body, bodyTarget.field, sourceLocale, translations, context, kept?.[recordFieldPlace(context, bodyTarget.key)])
|
|
310
312
|
}
|
|
311
313
|
}
|
|
312
314
|
// The brief is the card, and is always sent — an empty one included, so a required
|
|
@@ -389,8 +391,8 @@ export function recordsToEntities({
|
|
|
389
391
|
// What one record's read collects beside the values: the keys with no place in the
|
|
390
392
|
// schema, the keys written flat in a schema written by section, and values of the
|
|
391
393
|
// wrong shape for their section.
|
|
392
|
-
function encoder(sourceLocale, translations, refs = null, context = null) {
|
|
393
|
-
return { sourceLocale, translations, refs, context, undeclared: [], misplaced: [], shape: [], unresolved: [], namesNew: 0 }
|
|
394
|
+
function encoder(sourceLocale, translations, refs = null, context = null, kept = null) {
|
|
395
|
+
return { sourceLocale, translations, refs, context, kept, undeclared: [], misplaced: [], shape: [], unresolved: [], namesNew: 0 }
|
|
394
396
|
}
|
|
395
397
|
|
|
396
398
|
// A flat record: its keys, minus the record's own, are the one section's fields.
|
|
@@ -489,7 +491,7 @@ function encodeRecord(fields, value, enc, prefix, reserved) {
|
|
|
489
491
|
? encodeSection(field, v, enc, `${prefix}${key}`)
|
|
490
492
|
: field?.type === 'entity_ref'
|
|
491
493
|
? encodeReference(v, field, enc, `${prefix}${key}`)
|
|
492
|
-
: encodeFieldValue(v, field, enc.sourceLocale, enc.translations, enc.context)
|
|
494
|
+
: encodeFieldValue(v, field, enc.sourceLocale, enc.translations, enc.context, enc.kept?.[recordFieldPlace(enc.context, key)])
|
|
493
495
|
if (encoded !== undefined) out[key] = encoded
|
|
494
496
|
}
|
|
495
497
|
for (const key of Object.keys(value)) {
|
|
@@ -1041,6 +1043,8 @@ export async function buildRecordEntities(siteRoot, opts = {}) {
|
|
|
1041
1043
|
].filter((l) => l !== sourceLocale)
|
|
1042
1044
|
const translations =
|
|
1043
1045
|
targetLocales.length > 0 ? loadLocaleTranslations(siteRoot, targetLocales, 'records') : null
|
|
1046
|
+
// What a pull kept for this push to send back (`readKept`).
|
|
1047
|
+
const kept = readKept(siteRoot, 'records')
|
|
1044
1048
|
|
|
1045
1049
|
// ⭐ THE FILE'S `$uuid` IS OURS; THE WIRE'S IS THE BACKEND'S (2026-09-20).
|
|
1046
1050
|
//
|
|
@@ -1209,6 +1213,7 @@ export async function buildRecordEntities(siteRoot, opts = {}) {
|
|
|
1209
1213
|
sourceLocale,
|
|
1210
1214
|
translations,
|
|
1211
1215
|
refs,
|
|
1216
|
+
kept,
|
|
1212
1217
|
})
|
|
1213
1218
|
// Free-form per-locale body overrides (a full localized doc beats the structural
|
|
1214
1219
|
// map) — only meaningful for a multi-locale site with a prosemirror content field.
|