champollion 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +83 -36
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1689 -87
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +196 -14
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +270 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1127 -344
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +726 -78
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +965 -253
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +964 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +571 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +252 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +68 -128
  70. package/lib/methods/llm.js +80 -31
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +11 -0
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +2 -1
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +880 -17
  117. package/lib/verify.js +1296 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/method-registry.json +2 -0
  126. package/shared/metric-registry.json +96 -18
  127. package/shared/schemas/champollion-plugin.schema.json +4 -0
  128. package/shared/schemas/corpora-card.schema.json +8 -2
  129. package/shared/schemas/method-index-record.schema.json +67 -0
  130. package/shared/schemas/method-registry.schema.json +4 -0
  131. package/shared/schemas/metric-registry.schema.json +55 -1
  132. package/shared/docent/corpus.json +0 -11739
@@ -0,0 +1,270 @@
1
+ /**
2
+ * content-refusals.js — Markdown blocks and front-matter fields the quality
3
+ * gate refused, remembered so a plain sync does not send them to the same
4
+ * model again. The key-value rule (lib/locale-state.js, Round 4) for the two
5
+ * content lanes (lib/content-sync.js, lib/docusaurus-sync.js).
6
+ *
7
+ * WHY: a block the gate refused (lib/validate.js contentGateFault) was
8
+ * written as the '[EN] ' last resort, never cached, and its file's lock was
9
+ * not advanced — so EVERY sync sent it to the same paid model again, which
10
+ * answered the same way and was refused again. A refused front-matter field
11
+ * failed its page, and every sync re-sent the page's fields. With a paid
12
+ * model and no fallback that repeated the charge on each run, unlike keys,
13
+ * which are held back.
14
+ *
15
+ * THE RECORD — in .champollion-content.lock (committed, beside the lane's
16
+ * own entries), one entry per (page × locale) that has refusals:
17
+ *
18
+ * "refused:<manifestKey>": {
19
+ * "block:<hash12>": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" },
20
+ * "field:title": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" },
21
+ * "page": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" }
22
+ * }
23
+ *
24
+ * <manifestKey> is the lane's own key for the translation ("posts/a.md:fr",
25
+ * "docusaurus:docs/intro.md:fr"). A block is named by the hash of its source
26
+ * text, so an edited paragraph is a new block (the hold lifts); `source` is
27
+ * the 12-hex SHA-256 the key-value record uses, `methods` the TM method keys
28
+ * (method|model|register|coaching) whose answers the gate refused.
29
+ *
30
+ * "page" is the whole body of a page translated in one prompt
31
+ * (contentSegmentation: "page"): refused when the answer damages a protected
32
+ * placeholder or hollows the page (the whole-body checks), its `source` the
33
+ * hash of the body text. Editing the body — or switching to block
34
+ * segmentation — lifts it. A held page is not written, as a held field's
35
+ * page is not: there is no '[EN] ' text for a page translated whole.
36
+ *
37
+ * PRECEDENCE — the key-value rule, unit by unit:
38
+ * 1. Named for a redo — `--redo files:<page>`, `--redo content`
39
+ * (= --force-content), `--retranslate`, or anything under `--fresh`:
40
+ * always sent.
41
+ * 2. Refused before by the pair's method, for the unit's current source
42
+ * text: held back — not sent to that method again. A fallback method
43
+ * that has not refused it still gets it. A held block keeps its
44
+ * '[EN] ' text until it is filled; a page whose front-matter field is
45
+ * held is not written (the field gate fails a page whole, as it always
46
+ * did) — and nothing of it is sent.
47
+ * 3. A model or method change lifts the hold (the refusal was that method
48
+ * key's), as does a new source text.
49
+ * The cache is always consulted first (free): holding back only stops a
50
+ * paid call. A unit filled any other way — a fallback, the cache, a
51
+ * paragraph written by hand — drops its record.
52
+ */
53
+
54
+ import { tmMethodKey, lookupTM } from './tm.js';
55
+ import { holdState, shortSourceHash } from './locale-state.js';
56
+ import { contentRedoCommand } from './verify.js';
57
+ import { translatableBlockSources } from './content-estimate.js';
58
+
59
+ /** The content-lock key of one page × locale's refusals. */
60
+ export function refusalRecordKey(manifestKey) {
61
+ return `refused:${manifestKey}`;
62
+ }
63
+
64
+ /** A block's unit id: the hash of its (restored) source text. */
65
+ export const blockUnit = (source) => `block:${shortSourceHash(source)}`;
66
+
67
+ /** A front-matter field's unit id. */
68
+ export const fieldUnit = (field) => `field:${field}`;
69
+
70
+ /** The unit id of a page body translated whole (contentSegmentation: "page"). */
71
+ export const PAGE_UNIT = 'page';
72
+
73
+ /** How a held page body is named in messages and summaries. */
74
+ export const PAGE_NAME = 'the whole page body';
75
+
76
+ /**
77
+ * One page × locale's refusal record (entries that do not parse are dropped).
78
+ *
79
+ * @param {object} manifest - The content lock
80
+ * @param {string} manifestKey
81
+ * @returns {Record<string, { source: string, methods: string[], on?: string }>}
82
+ */
83
+ export function readRefusals(manifest, manifestKey) {
84
+ const rec = manifest?.[refusalRecordKey(manifestKey)];
85
+ if (!rec || typeof rec !== 'object' || Array.isArray(rec)) return {};
86
+ const out = {};
87
+ for (const [id, r] of Object.entries(rec)) {
88
+ if (r && typeof r === 'object' && typeof r.source === 'string' && Array.isArray(r.methods)) out[id] = r;
89
+ }
90
+ return out;
91
+ }
92
+
93
+ /**
94
+ * What a run may do with each unit of one page: 'send' (ask the pair's
95
+ * method), 'fallback-only' (the pair's method refused it; its fallback has
96
+ * not), 'held' (not sent at all).
97
+ *
98
+ * @param {object} units - readRefusals()
99
+ * @param {object} pairConfig
100
+ * @param {{ redo?: boolean }} [opts] - redo: the page was named for a redo (always sent)
101
+ * @returns {{ block: (source: string) => 'send'|'fallback-only'|'held',
102
+ * field: (field: string, source: string) => 'send'|'fallback-only'|'held',
103
+ * page: (body: string) => 'send'|'fallback-only'|'held' }}
104
+ */
105
+ export function contentHolds(units, pairConfig, { redo = false } = {}) {
106
+ // The key-value rule (lib/locale-state.js holdState), unit by unit.
107
+ const stateOf = (id, source) => (redo ? 'send' : holdState(units[id], source, pairConfig));
108
+ return {
109
+ block: (source) => stateOf(blockUnit(source), source),
110
+ field: (field, source) => stateOf(fieldUnit(field), source),
111
+ page: (body) => stateOf(PAGE_UNIT, body),
112
+ };
113
+ }
114
+
115
+ /**
116
+ * One page's record after a run processed it: earlier refusals that still
117
+ * apply (same source text, not filled this run), plus this run's.
118
+ *
119
+ * @param {object} prior - readRefusals()
120
+ * @param {object} p
121
+ * @param {string[]} p.blockSources - The page's current translatable block sources ([] in page mode)
122
+ * @param {string|null} [p.pageSource] - Its body, in page mode (null in block mode)
123
+ * @param {Record<string, string>} p.fields - Its current translatable front-matter fields
124
+ * @param {Array<{ unit: string, source: string, methods: string[] }>} p.refused - Refused this run, left unfilled
125
+ * @param {Set<string>} p.filled - Units filled this run (any method, the cache, a person)
126
+ * @returns {object|null} The record, or null when nothing is left to hold
127
+ */
128
+ export function nextRefusals(prior, { blockSources = [], pageSource = null, fields = {}, refused = [], filled = new Set() }) {
129
+ const live = new Map();
130
+ for (const s of blockSources) live.set(blockUnit(s), shortSourceHash(s));
131
+ if (typeof pageSource === 'string' && pageSource.trim()) live.set(PAGE_UNIT, shortSourceHash(pageSource));
132
+ for (const [f, v] of Object.entries(fields)) if (typeof v === 'string') live.set(fieldUnit(f), shortSourceHash(v));
133
+ const out = {};
134
+ for (const [id, r] of Object.entries(prior)) {
135
+ if (filled.has(id) || live.get(id) !== r.source) continue;
136
+ out[id] = r;
137
+ }
138
+ const on = new Date().toISOString().slice(0, 10);
139
+ for (const { unit, source, methods } of refused) {
140
+ if (!methods || methods.length === 0 || filled.has(unit)) continue;
141
+ const src = shortSourceHash(source);
142
+ const known = out[unit] && out[unit].source === src ? out[unit].methods : [];
143
+ out[unit] = { source: src, methods: [...new Set([...known, ...methods])], on };
144
+ }
145
+ return Object.keys(out).length > 0 ? out : null;
146
+ }
147
+
148
+ /** Write (or remove) one page's record in the content lock object. */
149
+ export function storeRefusals(manifest, manifestKey, record) {
150
+ const key = refusalRecordKey(manifestKey);
151
+ if (record) manifest[key] = record;
152
+ else delete manifest[key];
153
+ }
154
+
155
+ /**
156
+ * The warning for one page's held units — both lanes say it the same way.
157
+ *
158
+ * @param {object} p
159
+ * @param {string} p.file - The page as sync names it (what `--redo files:` matches)
160
+ * @param {string} p.code
161
+ * @param {string} p.pairKey
162
+ * @param {object} p.pairConfig
163
+ * @param {string[]} p.names - "paragraph 2", 'front matter "title"'
164
+ * @param {boolean} [p.pageHeld] - A front-matter field (or the page body, in
165
+ * page mode) is held: the page is not written
166
+ * @param {boolean} [p.handWritten] - The lane keeps a paragraph written by hand (contentDir)
167
+ * @param {string} [p.fallbackPrefix]
168
+ * @returns {string}
169
+ */
170
+ export function describeHeld({ file, code, pairKey, pairConfig, names, pageHeld = false, handWritten = false, fallbackPrefix = '[EN] ' }) {
171
+ const shown = names.slice(0, 5).join(', ') + (names.length > 5 ? `, +${names.length - 5} more` : '');
172
+ const fb = pairConfig.fallback;
173
+ const who = fb ? `${pairConfig.method}'s and its fallback's (${fb.method})` : `${pairConfig.method}'s`;
174
+ const left = !pageHeld
175
+ ? `The '${fallbackPrefix.trim()}' text stays until they are filled.`
176
+ : names.includes(PAGE_NAME)
177
+ ? 'A page translated whole has no \'[EN]\' text to keep, so it is not written (and nothing of it is sent) until it is filled.'
178
+ : 'A front-matter field the gate refuses fails its page, so the page is not written (and nothing of it is sent) until it is filled.';
179
+ const other = fb
180
+ ? 'another "fallback" method on the pair'
181
+ : `a "fallback" method on the pair (it is asked for what ${pairConfig.method} refused)`;
182
+ const hand = handWritten && !pageHeld ? ', or write those paragraphs by hand (a hand-written paragraph is kept)' : '';
183
+ return `${file} → ${code}: ${names.length} block(s)/field(s) held back (${shown}) — the quality gate refused ${who} translation of their `
184
+ + `current text before; not sent, not billed. ${left} Ask again: \`${contentRedoCommand(file, { pair: pairKey })}\`; `
185
+ + `or fill them another way — ${other}${hand}.`;
186
+ }
187
+
188
+ /**
189
+ * Said with a refusal this run recorded: what the next sync does with it.
190
+ *
191
+ * @param {object} p
192
+ * @param {string} p.file
193
+ * @param {string} p.pairKey
194
+ * @param {object} p.pairConfig
195
+ * @param {number} p.count - Units refused (and left unfilled) this run
196
+ * @param {boolean} [p.fallbackAsked] - Some of them still go to the fallback
197
+ * (it has not refused them — skipped by --max-cost, or no answer)
198
+ * @returns {string}
199
+ */
200
+ export function describeNewHold({ file, pairKey, pairConfig, count, fallbackAsked = false, lead = 'Remembered:' }) {
201
+ const them = count === 1 ? 'it' : `these ${count}`;
202
+ const fb = pairConfig.fallback;
203
+ const where = fb && fallbackAsked
204
+ ? `${pairConfig.method} again (its fallback, ${fb.method}, is asked for what it has not refused)`
205
+ : `${pairConfig.method}${fb ? ` or its fallback (${fb.method})` : ''} again`;
206
+ return `${lead} the next sync does not send ${them} to ${where} — not billed again; `
207
+ + `\`${contentRedoCommand(file, { pair: pairKey })}\` asks again.`;
208
+ }
209
+
210
+ /**
211
+ * The end of a lane's "N block(s) … written as '[EN] '-prefixed source" line:
212
+ * what the next sync does with them — asks again for a block that got no
213
+ * usable answer, holds back one the gate refused.
214
+ *
215
+ * @param {object} p
216
+ * @param {string} p.file
217
+ * @param {string} p.pairKey
218
+ * @param {object} p.pairConfig
219
+ * @param {number} p.fellBack - Blocks written as '[EN] ' this run
220
+ * @param {number[]} p.newlyHeld - Of those (indices into the batch), the ones the gate refused
221
+ * @param {string[][]} [p.refusedBy] - translateBlocksWithFallback().refusedBy
222
+ * @returns {string}
223
+ */
224
+ export function describeFallenBack({ file, pairKey, pairConfig, fellBack, newlyHeld, refusedBy = [] }) {
225
+ if (newlyHeld.length === 0) return 'Not cached, lock not advanced: the next sync retries just those block(s).';
226
+ const retry = fellBack - newlyHeld.length;
227
+ const fbKey = pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null;
228
+ const fallbackAsked = !!fbKey && newlyHeld.some(i => !(refusedBy[i] || []).includes(fbKey));
229
+ const n = newlyHeld.length;
230
+ return 'Not cached, lock not advanced. '
231
+ + (retry > 0 ? `The next sync asks again for the ${retry} that got no usable answer. ` : '')
232
+ + describeNewHold({
233
+ file, pairKey, pairConfig, count: n, fallbackAsked,
234
+ lead: `The ${n === 1 ? 'one' : n} the quality gate refused ${n === 1 ? 'is' : 'are'} remembered:`,
235
+ });
236
+ }
237
+
238
+ /**
239
+ * What a real run would hold back for one page — a dry run says so instead
240
+ * of "would create". The cache is consulted as the sync consults it (pure
241
+ * lookups: a preview changes nothing): only what it does not hold is held.
242
+ *
243
+ * @param {object} p
244
+ * @param {object} p.tm
245
+ * @param {string} p.code
246
+ * @param {object} p.pairConfig
247
+ * @param {Record<string, string>} p.fields - Translatable front-matter fields
248
+ * @param {string} p.body
249
+ * @param {'block'|'page'} p.segMode
250
+ * @param {ReturnType<typeof contentHolds>} p.holds
251
+ * @returns {{ names: string[], pageHeld: boolean }}
252
+ */
253
+ export function previewHeld({ tm, code, pairConfig, fields, body, segMode, holds }) {
254
+ const keys = [tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])];
255
+ const cached = (text) => keys.some(k => lookupTM(tm, text, code, k) !== null);
256
+ const names = [];
257
+ for (const [field, text] of Object.entries(fields || {})) {
258
+ if (typeof text === 'string' && !cached(text) && holds.field(field, text) === 'held') names.push(`front matter "${field}"`);
259
+ }
260
+ if (names.length > 0) return { names, pageHeld: true };
261
+ if (segMode === 'page' && typeof body === 'string' && body.trim() && !cached(body) && holds.page(body) === 'held') {
262
+ return { names: [PAGE_NAME], pageHeld: true };
263
+ }
264
+ if (segMode === 'block' && typeof body === 'string' && body.trim() && !cached(body)) {
265
+ translatableBlockSources(body).forEach((source, i) => {
266
+ if (!cached(source) && holds.block(source) === 'held') names.push(`paragraph ${i + 1}`);
267
+ });
268
+ }
269
+ return { names, pageHeld: false };
270
+ }
@@ -0,0 +1,372 @@
1
+ /**
2
+ * content-review.js — keep the edits a person makes to translated Markdown.
3
+ *
4
+ * THE BUG THIS CLOSES (persona finding, 2026-10-03): a reviewer corrects a
5
+ * sentence in newsletters/2026-10.crk.md; the author then edits a DIFFERENT
6
+ * paragraph of newsletters/2026-10.md; the next sync re-translates the file
7
+ * and the Translation Memory serves the OLD machine translation for every
8
+ * unchanged paragraph, so the reviewer's correction was overwritten with
9
+ * nothing said but "source updated, re-translating". The content lock held
10
+ * only the SOURCE hash, so nothing could tell a reviewed file from the file
11
+ * sync wrote.
12
+ *
13
+ * THE RECORD. Whenever the content lane writes (or adopts) a translated file
14
+ * it also records what it left on disk, in the same .champollion-content.lock,
15
+ * under "written:<relPath>:<locale>":
16
+ *
17
+ * {
18
+ * "file": 16 hex of SHA-256 over the whole file as sync left it,
19
+ * "blocks": "s:t s:t:r …" one entry per translatable body block, in
20
+ * order. s = 8 hex of the SOURCE block it translates, t = 8 hex
21
+ * of the block as written, ":r" = a person's text (kept on
22
+ * every later sync). null when the written body does not split
23
+ * into as many blocks as the source (no safe positional map).
24
+ * "fields": { "<name>": "s:t[:r]" } translatable front-matter fields.
25
+ * "owned": true the whole file is a person's (hand-translated, or
26
+ * brought up to date by hand) and has no block map.
27
+ * "held": 16 hex of the file when a sync last left it as is because
28
+ * its edits could not be merged with a source change.
29
+ * }
30
+ *
31
+ * The block unit is exactly the TM's (segment.js via translatableBlockSources:
32
+ * protect → split → restore, translatable segments only), and the record is
33
+ * computed by re-splitting the file AS WRITTEN — the same function the next
34
+ * sync runs on the file on disk — so "differs from the record" can only mean
35
+ * that a person changed it.
36
+ *
37
+ * WHAT A SYNC DOES WITH EDITS (lib/content-sync.js):
38
+ * - source unchanged → the file is not touched (as before);
39
+ * - source changed → edited paragraphs whose source paragraph is unchanged
40
+ * are kept word for word, everything else is updated; an edited paragraph
41
+ * whose source paragraph itself changed is re-translated and the run
42
+ * prints the edited wording so it can be re-applied;
43
+ * - edits that cannot be matched paragraph by paragraph (paragraphs added,
44
+ * removed or merged; 'page' segmentation) → the file is left as is and
45
+ * the run says so every time until the file is brought up to date by hand
46
+ * (the next sync then takes it as current) or re-translated by name.
47
+ */
48
+
49
+ import crypto from 'node:crypto';
50
+ import { parseContentFile } from './content.js';
51
+ import { translatableBlockSources } from './content-estimate.js';
52
+
53
+ const WRITTEN_PREFIX = 'written:';
54
+
55
+ function sha256(text) {
56
+ return crypto.createHash('sha256').update(text, 'utf-8').digest('hex');
57
+ }
58
+
59
+ /** 8-hex fingerprint of one block or field value (collisions only matter within one file). */
60
+ function blockHash(text) {
61
+ return sha256(text).slice(0, 8);
62
+ }
63
+
64
+ /** 16-hex fingerprint of a whole translated file. */
65
+ function fileHash(text) {
66
+ return sha256(text).slice(0, 16);
67
+ }
68
+
69
+ /** The lock key of the written-record for a content manifest key ("relPath:locale"). */
70
+ function writtenRecordKey(manifestKey) {
71
+ return `${WRITTEN_PREFIX}${manifestKey}`;
72
+ }
73
+
74
+ function encodePair(s, t, owned) {
75
+ return `${s}:${t}${owned ? ':r' : ''}`;
76
+ }
77
+
78
+ function decodePair(enc) {
79
+ if (typeof enc !== 'string') return null;
80
+ const [s, t, flag, extra] = enc.split(':');
81
+ if (!/^[0-9a-f]{8}$/.test(s || '') || !/^[0-9a-f]{8}$/.test(t || '')) return null;
82
+ if (extra !== undefined || (flag !== undefined && flag !== 'r')) return null;
83
+ return { s, t, owned: flag === 'r' };
84
+ }
85
+
86
+ /**
87
+ * Parse a stored written-record. Returns null when there is none; throws
88
+ * (with a plain message) when one is present but malformed — the caller
89
+ * must say so rather than silently treat the file as unedited.
90
+ *
91
+ * @param {*} value - The lock value under writtenRecordKey(...)
92
+ * @returns {{ file: string, blocks: Array<{s:string,t:string,owned:boolean}>|null,
93
+ * fields: Object<string,{s:string,t:string,owned:boolean}>, owned: boolean,
94
+ * held: string|null }|null}
95
+ */
96
+ function parseWrittenRecord(value) {
97
+ if (value === undefined || value === null) return null;
98
+ if (typeof value !== 'object' || Array.isArray(value) || typeof value.file !== 'string') {
99
+ throw new Error('not a written-record object');
100
+ }
101
+ let blocks = null;
102
+ if (typeof value.blocks === 'string') {
103
+ blocks = value.blocks === '' ? [] : value.blocks.split(' ').map(decodePair);
104
+ if (blocks.some(b => b === null)) throw new Error('malformed "blocks"');
105
+ } else if (value.blocks !== null && value.blocks !== undefined) {
106
+ throw new Error('malformed "blocks"');
107
+ }
108
+ const fields = {};
109
+ if (value.fields !== undefined) {
110
+ if (typeof value.fields !== 'object' || value.fields === null || Array.isArray(value.fields)) {
111
+ throw new Error('malformed "fields"');
112
+ }
113
+ for (const [name, enc] of Object.entries(value.fields)) {
114
+ const pair = decodePair(enc);
115
+ if (!pair) throw new Error(`malformed field "${name}"`);
116
+ fields[name] = pair;
117
+ }
118
+ }
119
+ return {
120
+ file: value.file,
121
+ blocks,
122
+ fields,
123
+ owned: value.owned === true,
124
+ held: typeof value.held === 'string' ? value.held : null,
125
+ };
126
+ }
127
+
128
+ /**
129
+ * Build the record of a file sync is leaving on disk.
130
+ *
131
+ * @param {object} args
132
+ * @param {string} args.sourceBody - Source Markdown body (front matter stripped)
133
+ * @param {Object<string,string>} args.sourceFields - Translatable source front-matter values
134
+ * @param {string} args.written - The complete target file as written
135
+ * @param {Set<number>} [args.ownedBlocks] - Source block positions holding a person's text
136
+ * @param {Set<string>} [args.ownedFields] - Field names holding a person's text
137
+ * @returns {object} JSON-ready record (see header)
138
+ */
139
+ function buildWrittenRecord({ sourceBody, sourceFields, written, ownedBlocks = new Set(), ownedFields = new Set() }) {
140
+ const target = parseContentFile(written);
141
+ const src = translatableBlockSources(sourceBody);
142
+ const out = translatableBlockSources(target.body);
143
+ const record = { file: fileHash(written), blocks: null };
144
+ if (src.length === out.length) {
145
+ record.blocks = src.map((s, j) => encodePair(blockHash(s), blockHash(out[j]), ownedBlocks.has(j))).join(' ');
146
+ } else if (ownedBlocks.size > 0) {
147
+ // A person's paragraphs with no block map: the whole file is theirs.
148
+ record.owned = true;
149
+ }
150
+ const fields = {};
151
+ for (const name of Object.keys(sourceFields).sort()) {
152
+ const value = sourceFields[name];
153
+ const tv = target.frontMatter[name];
154
+ if (typeof value === 'string' && typeof tv === 'string') {
155
+ fields[name] = encodePair(blockHash(value), blockHash(tv), ownedFields.has(name));
156
+ }
157
+ }
158
+ if (Object.keys(fields).length > 0) record.fields = fields;
159
+ return record;
160
+ }
161
+
162
+ /**
163
+ * Record a file a person wrote (hand-translated, or brought up to date by
164
+ * hand after a sync held it): every paragraph and field is theirs.
165
+ */
166
+ function buildAdoptedRecord({ sourceBody, sourceFields, written }) {
167
+ const blockCount = translatableBlockSources(sourceBody).length;
168
+ const ownedBlocks = new Set(Array.from({ length: blockCount }, (_, j) => j));
169
+ const record = buildWrittenRecord({
170
+ sourceBody, sourceFields, written, ownedBlocks, ownedFields: new Set(Object.keys(sourceFields)),
171
+ });
172
+ // A hand-written file with no translatable blocks is still a person's file.
173
+ if (record.blocks === null) record.owned = true;
174
+ return record;
175
+ }
176
+
177
+ /**
178
+ * First record of a file this version never recorded (written by an older
179
+ * version, source unchanged since). A block or field is marked as a person's
180
+ * when the TM holds a machine translation for its source and the file does
181
+ * NOT carry it — that is an edit made before the record existed. With no TM
182
+ * entry there is no evidence either way, and the text is taken as written.
183
+ *
184
+ * @param {object} args
185
+ * @param {(sourceText: string) => string[]} args.machineFor - Cached machine
186
+ * translations of a source text (exact entries; no side effects)
187
+ */
188
+ function buildBootstrapRecord({ sourceBody, sourceFields, written, machineFor }) {
189
+ const target = parseContentFile(written);
190
+ const src = translatableBlockSources(sourceBody);
191
+ const out = translatableBlockSources(target.body);
192
+ const differs = (sourceText, targetText) => {
193
+ const known = machineFor(sourceText);
194
+ return known.length > 0 && !known.includes(targetText);
195
+ };
196
+ const ownedBlocks = new Set();
197
+ if (src.length === out.length) {
198
+ src.forEach((s, j) => { if (differs(s, out[j])) ownedBlocks.add(j); });
199
+ }
200
+ const ownedFields = new Set();
201
+ for (const [name, value] of Object.entries(sourceFields)) {
202
+ const tv = target.frontMatter[name];
203
+ if (typeof value === 'string' && typeof tv === 'string' && differs(value, tv)) ownedFields.add(name);
204
+ }
205
+ return buildWrittenRecord({ sourceBody, sourceFields, written, ownedBlocks, ownedFields });
206
+ }
207
+
208
+ /**
209
+ * What a person changed in a translated file since sync last recorded it.
210
+ *
211
+ * @param {string} targetRaw - The translated file as it is on disk now
212
+ * @param {ReturnType<typeof parseWrittenRecord>} record
213
+ * @returns {{ state: 'clean' } |
214
+ * { state: 'unmappable', reason: string } |
215
+ * { state: 'edited', blocks: Array<{src:string,text:string,owned:boolean}>,
216
+ * fields: Object<string,{src:string,text:string,owned:boolean}>,
217
+ * ownedBlocks: number, ownedFields: number }}
218
+ */
219
+ function readReviewerEdits(targetRaw, record) {
220
+ const edited = fileHash(targetRaw) !== record.file;
221
+ const anyOwned = record.owned
222
+ || (record.blocks || []).some(b => b.owned)
223
+ || Object.values(record.fields).some(f => f.owned);
224
+ if (!edited && !anyOwned) return { state: 'clean' };
225
+
226
+ const target = parseContentFile(targetRaw);
227
+ const fields = {};
228
+ for (const [name, f] of Object.entries(record.fields)) {
229
+ const current = target.frontMatter[name];
230
+ if (typeof current !== 'string') continue; // removed from the file: nothing to keep
231
+ fields[name] = { src: f.s, text: current, owned: f.owned || blockHash(current) !== f.t };
232
+ }
233
+ const ownedFields = Object.values(fields).filter(f => f.owned).length;
234
+
235
+ if (record.blocks === null) {
236
+ if (!edited && !record.owned) {
237
+ // Only field flags: the body is as written.
238
+ return ownedFields > 0 ? { state: 'edited', blocks: null, fields, ownedBlocks: 0, ownedFields } : { state: 'clean' };
239
+ }
240
+ return {
241
+ state: 'unmappable',
242
+ reason: record.owned
243
+ ? 'it is maintained by hand'
244
+ : 'its paragraphs did not line up one to one with the source when sync wrote it',
245
+ };
246
+ }
247
+
248
+ const current = translatableBlockSources(target.body);
249
+ if (current.length !== record.blocks.length) {
250
+ return {
251
+ state: 'unmappable',
252
+ reason: `it now has ${current.length} paragraph(s) where sync wrote ${record.blocks.length} — paragraphs were added, removed or merged`,
253
+ };
254
+ }
255
+ const blocks = record.blocks.map((b, i) => ({
256
+ src: b.s, text: current[i], owned: b.owned || blockHash(current[i]) !== b.t,
257
+ }));
258
+ const ownedBlocks = blocks.filter(b => b.owned).length;
259
+ // Edits only to separators, passthrough blocks (code) or untranslated
260
+ // front matter: those parts are copied from the source by design.
261
+ if (ownedBlocks === 0 && ownedFields === 0) return { state: 'clean' };
262
+ return { state: 'edited', blocks, fields, ownedBlocks, ownedFields };
263
+ }
264
+
265
+ /**
266
+ * Match a person's paragraphs to the CURRENT source's blocks by the source
267
+ * text they translate. Every previously written block is queued under its
268
+ * source hash, so the k-th copy of a repeated paragraph maps to the k-th copy.
269
+ *
270
+ * @param {Array<{src:string,text:string,owned:boolean}>} editBlocks - From readReviewerEdits
271
+ * @param {string[]} sourceBlocks - Current source's translatable block texts, in order
272
+ * @returns {{ keep: Map<number,string>, superseded: Array<{ paragraph: number, text: string }> }}
273
+ * keep: source block position → the person's text. superseded: edited
274
+ * paragraphs (1-based, as they were in the file) whose source paragraph
275
+ * is gone or changed.
276
+ */
277
+ function matchEditedBlocks(editBlocks, sourceBlocks) {
278
+ const queues = new Map();
279
+ editBlocks.forEach((b, i) => {
280
+ if (!queues.has(b.src)) queues.set(b.src, []);
281
+ queues.get(b.src).push({ ...b, paragraph: i + 1 });
282
+ });
283
+ const keep = new Map();
284
+ sourceBlocks.forEach((text, j) => {
285
+ const entry = queues.get(blockHash(text))?.shift();
286
+ if (entry && entry.owned) keep.set(j, entry.text);
287
+ });
288
+ const superseded = [...queues.values()].flat()
289
+ .filter(e => e.owned)
290
+ .sort((a, b) => a.paragraph - b.paragraph)
291
+ .map(e => ({ paragraph: e.paragraph, text: e.text }));
292
+ return { keep, superseded };
293
+ }
294
+
295
+ /**
296
+ * Match a person's front-matter values to the current source fields.
297
+ *
298
+ * @returns {{ keep: Object<string,string>, superseded: Array<{ field: string, text: string }> }}
299
+ */
300
+ function matchEditedFields(editFields, sourceFields) {
301
+ const keep = {};
302
+ const superseded = [];
303
+ for (const [name, f] of Object.entries(editFields)) {
304
+ if (!f.owned) continue;
305
+ const source = sourceFields[name];
306
+ if (typeof source === 'string' && blockHash(source) === f.src) keep[name] = f.text;
307
+ else superseded.push({ field: name, text: f.text });
308
+ }
309
+ return { keep, superseded };
310
+ }
311
+
312
+ /**
313
+ * Decide what a sync does with an existing translated file before it
314
+ * translates anything. Pure (no I/O): runContentSync and the cost estimator
315
+ * (countPendingContentTranslations) both call it, so the estimate can never
316
+ * count a file the sync would leave as is.
317
+ *
318
+ * @param {object} args
319
+ * @param {string} args.targetRaw - The translated file on disk
320
+ * @param {ReturnType<typeof parseWrittenRecord>} args.record - null = no record (older version)
321
+ * @param {'block'|'page'} args.segMode - The pair's content segmentation
322
+ * @param {boolean} args.replaceEdits - The operator named this file for a redo
323
+ * (--retranslate, or --redo files: / --files with --force-content)
324
+ * @param {boolean} args.sourceCurrent - The lock says the source is unchanged
325
+ * (we are here only because of --force-content)
326
+ * @returns {{ action: 'proceed', edits: object|null, replaced?: object } |
327
+ * { action: 'keep', reason: string } | { action: 'accept' } |
328
+ * { action: 'hold', reason: string, heldHash: string }}
329
+ * proceed: translate; `edits` (state 'edited') are the person's paragraphs
330
+ * and fields to keep. keep: an up-to-date file with edits that cannot be
331
+ * merged — leave it. accept: a held file was edited since — take it as
332
+ * current. hold: leave the file as is and say so (lock not advanced).
333
+ */
334
+ function assessExistingTarget({ targetRaw, record, segMode, replaceEdits, sourceCurrent }) {
335
+ if (!record) return { action: 'proceed', edits: null };
336
+ const edits = readReviewerEdits(targetRaw, record);
337
+ if (edits.state === 'clean') return { action: 'proceed', edits: null };
338
+ if (replaceEdits) return { action: 'proceed', edits: null, replaced: edits };
339
+ const cannotMerge = edits.state === 'unmappable' || (segMode === 'page' && edits.ownedBlocks > 0);
340
+ if (!cannotMerge) return { action: 'proceed', edits };
341
+ const reason = edits.state === 'unmappable'
342
+ ? edits.reason
343
+ : "its pair uses 'page' segmentation, which translates the body as one piece";
344
+ if (sourceCurrent) return { action: 'keep', reason };
345
+ const current = fileHash(targetRaw);
346
+ if (record.held && record.held !== current) return { action: 'accept' };
347
+ return { action: 'hold', reason, heldHash: current };
348
+ }
349
+
350
+ /** "3 paragraph(s) and the title" — for kept/replaced messages. */
351
+ function describeEdits({ paragraphs = 0, fields = [] }) {
352
+ const parts = [];
353
+ if (paragraphs > 0) parts.push(`${paragraphs} paragraph(s)`);
354
+ if (fields.length > 0) parts.push(`front matter ${fields.map(f => `"${f}"`).join(', ')}`);
355
+ return parts.join(' and ') || 'the file';
356
+ }
357
+
358
+ export {
359
+ assessExistingTarget,
360
+ describeEdits,
361
+ WRITTEN_PREFIX,
362
+ blockHash,
363
+ fileHash,
364
+ writtenRecordKey,
365
+ parseWrittenRecord,
366
+ buildWrittenRecord,
367
+ buildAdoptedRecord,
368
+ buildBootstrapRecord,
369
+ readReviewerEdits,
370
+ matchEditedBlocks,
371
+ matchEditedFields,
372
+ };