champollion 0.3.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +82 -37
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1709 -101
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +230 -17
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +284 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1136 -345
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +727 -79
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +975 -262
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +1102 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +575 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +257 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +71 -129
  70. package/lib/methods/llm.js +87 -33
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +102 -55
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +24 -8
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +996 -21
  117. package/lib/verify.js +1332 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/licenses.json +99 -3
  126. package/shared/method-registry.json +2 -0
  127. package/shared/metric-registry.json +96 -18
  128. package/shared/retired-model-aliases.json +9 -0
  129. package/shared/schemas/champollion-plugin.schema.json +4 -0
  130. package/shared/schemas/corpora-card.schema.json +8 -2
  131. package/shared/schemas/human-services.schema.json +1 -1
  132. package/shared/schemas/method-index-record.schema.json +67 -0
  133. package/shared/schemas/method-registry.schema.json +4 -0
  134. package/shared/schemas/metric-registry.schema.json +55 -1
  135. package/shared/schemas/retired-model-aliases.schema.json +32 -0
  136. package/shared/docent/corpus.json +0 -11739
  137. package/shared/model-aliases.json +0 -7
  138. package/shared/schemas/model-aliases.schema.json +0 -27
@@ -1,5 +1,7 @@
1
1
  /**
2
- * Content sync — translates Hugo Markdown content files.
2
+ * Content sync — translates the Markdown/MDX files of a contentDir (a Hugo
3
+ * site's content/ or any folder of Markdown), each translation written beside
4
+ * its source as <name>.<locale>.md (lib/content.js getTargetContentPath).
3
5
  *
4
6
  * WHY THIS EXISTS: This was extracted from sync.js to reduce the
5
7
  * god-module's line count and give content translation its own
@@ -35,15 +37,31 @@
35
37
  * results that pass the existing checks (non-null API result; structural
36
38
  * block-batch validation; body placeholder-corruption check on the
37
39
  * REASSEMBLED body) are stored, and --no-tm bypasses the cache entirely.
40
+ *
41
+ * EDITS MADE BY HAND (lib/content-review.js): every file this lane writes is
42
+ * also recorded in the content lock ("written:<relPath>:<locale>"), block by
43
+ * block. Before a changed source re-translates a file, the file on disk is
44
+ * compared with that record: paragraphs and front-matter fields a person
45
+ * changed are KEPT when the source text they translate is unchanged (never
46
+ * sent to the API, never cached as machine output), and the run says so. An
47
+ * edited paragraph whose own source changed is re-translated with its edited
48
+ * wording printed; edits that cannot be matched paragraph by paragraph leave
49
+ * the whole file as is until it is updated by hand or named for a redo.
38
50
  */
39
51
 
40
52
  import fs from 'node:fs';
41
53
  import path from 'node:path';
42
54
  import crypto from 'node:crypto';
43
55
  import { translateBatch, translateRawContent, getMethod } from './translate.js';
44
- import { checkContentPreservation } from './validate.js';
45
- import { loadTM, saveTM, lookupTM, lookupTMValidated, storeTM, evictTM, isTMDirty, tmSize, tmMethodKey, partitionByTM } from './tm.js';
56
+ import { checkContentPreservation, contentGateFault } from './validate.js';
57
+ import { loadTM, saveTM, lookupTM, peekTM, storeTM, evictTM, isTMDirty, tmSize, tmMethodKey, partitionByTM, setModelCarryover, setTMReads, bypassTMFor, cacheKey, describeTMChanges, adoptLegacyCoachingKeys } from './tm.js';
58
+ import {
59
+ writtenRecordKey, parseWrittenRecord, buildWrittenRecord, buildAdoptedRecord, buildBootstrapRecord,
60
+ assessExistingTarget, matchEditedBlocks, matchEditedFields, describeEdits,
61
+ } from './content-review.js';
46
62
  import { output } from './output.js';
63
+ import { missingKeyAdvice } from './missing-key.js';
64
+ import { billableContentChars, translatableBlockSources } from './content-estimate.js';
47
65
  import { DEFAULT_REGISTERS } from './registers.js';
48
66
  import { isPathContained } from './security.js';
49
67
  import { pMap } from './concurrent.js';
@@ -63,12 +81,23 @@ import {
63
81
  splitBlocks, buildBlockBatchPrompt, parseBlockBatchResponse,
64
82
  translateBlockBatchResilient, assertSegmentationMode,
65
83
  } from './segment.js';
84
+ import {
85
+ lookupContentTM, serveFieldsFromFallbackCache, translateFieldsWithFallback,
86
+ translateBlocksWithFallback, translatePageWithFallback,
87
+ newFallbackReport, addToTally, fallbackSummary, printFallbackReport, warnFallbackMajority,
88
+ refuseRepeatedCachedPage,
89
+ } from './fallback.js';
90
+ import {
91
+ readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
92
+ describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock,
93
+ } from './content-refusals.js';
94
+ import { contentRedoCommand } from './verify.js';
66
95
 
67
96
  /**
68
- * Run content sync — translate Hugo Markdown content files.
97
+ * Run content sync — translate the contentDir's Markdown/MDX files.
69
98
  *
70
99
  * @param {object} options
71
- * @param {string} options.contentDir - Path to Hugo content directory
100
+ * @param {string} options.contentDir - Path to the content directory (any folder of Markdown/MDX)
72
101
  * @param {string} options.sourceLocale - Source language code
73
102
  * @param {Map<string, object>} options.pairs - Resolved pair graph (pairKey → pairConfig)
74
103
  * @param {string[]|null} options.translatableFields - Front matter fields to translate
@@ -77,8 +106,13 @@ import {
77
106
  * (gemini, openai, anthropic, deepl, …) resolve their own env keys inside
78
107
  * their method classes — a missing OpenRouter key must NOT block them.
79
108
  * @param {boolean} options.dryRun - Whether to write files
109
+ * @param {boolean} [options.freshOnModelChange] - --fresh-on-model-change: serve
110
+ * only exact-model TM hits (default reuses other models' translations)
80
111
  * @param {boolean} [options.noTM] - Bypass the Translation Memory (--no-tm):
81
112
  * every segment goes to the API and nothing is cached.
113
+ * @param {object|null} [options.fallbackBudget] - lib/fallback.js
114
+ * createFallbackBudget(): the --max-cost guard for pairs' fallback methods
115
+ * (null = no cap).
82
116
  */
83
117
  async function runContentSync(options) {
84
118
  const {
@@ -89,12 +123,22 @@ async function runContentSync(options) {
89
123
  apiKey,
90
124
  dryRun = false,
91
125
  noTM = false,
126
+ freshOnModelChange = false,
127
+ // --force-content: re-process up-to-date files (TM serves unchanged
128
+ // text, so this rebuilds from cache rather than re-billing).
129
+ forceContent = false,
130
+ // --files / --retranslate (lib/file-scope.js); null = every file.
131
+ fileScope = null,
92
132
  cwd = process.cwd(),
93
133
  concurrency = 48,
94
134
  // The honest-fallback marker (config.js default). Written in front of a
95
135
  // block's SOURCE text when the model drops its segment twice — visible,
96
136
  // never silent, never cached (see translateBlockBatchResilient).
97
137
  fallbackPrefix = '[EN] ',
138
+ fallbackBudget = null,
139
+ // (code) => the locale's different-inputs-same-output index, shared with
140
+ // the key-value sync of the same run (lib/validate.js SharedOutputIndex).
141
+ sharedOutputsFor = null,
98
142
  } = options;
99
143
 
100
144
  if (!fs.existsSync(contentDir)) {
@@ -110,6 +154,11 @@ async function runContentSync(options) {
110
154
 
111
155
  const fieldsList = translatableFields || DEFAULT_TRANSLATABLE_FIELDS;
112
156
 
157
+ // --redo files:<glob> (= --files + --force-content) and --retranslate name
158
+ // files: the one way a run replaces edits made by hand to a translation.
159
+ // A bare --force-content / --redo content keeps them.
160
+ const namedByFiles = Boolean(fileScope && fileScope.limitsFiles);
161
+
113
162
  // Warn at most once per source file about translatable-looking front matter
114
163
  // we can't reach (arrays / nested blocks). Never silently drop them. The
115
164
  // check + add are synchronous (no await between), so this is race-free.
@@ -129,10 +178,18 @@ async function runContentSync(options) {
129
178
  //
130
179
  // Safe under pMap concurrency: storeTM is a synchronous property assignment
131
180
  // and Node is single-threaded, so writes can't interleave between awaits.
132
- const tm = noTM ? { _meta: { version: 1 } } : loadTM(cwd);
181
+ // --fresh/--no-tm: serve nothing from the cache, but cache what is paid for
182
+ // (lib/tm.js setTMReads) — same as the key-value path in sync.js.
183
+ const tm = loadTM(cwd);
184
+ if (noTM) setTMReads(tm, false);
185
+ if (freshOnModelChange) setModelCarryover(tm, false); // see lib/tm.js "Model carry-over"
186
+ // Blocks a pair cached before its coaching was keyed, with today's
187
+ // coaching, stay served (lib/tm.js adoptLegacyCoachingKeys — sync records
188
+ // it first; a dry run, which saves nothing, records it here too).
189
+ adoptLegacyCoachingKeys(tm, pairs.values());
133
190
  const tmInitialSize = tmSize(tm);
134
191
  if (noTM) {
135
- output.info('Content sync: Translation Memory disabled (--no-tm)');
192
+ output.info('Content sync: Translation Memory not read this run — content is translated fresh, and cached');
136
193
  } else if (tmInitialSize > 0) {
137
194
  output.info(`Content sync: Translation Memory loaded (${tmInitialSize} cached entries)`);
138
195
  }
@@ -171,6 +228,25 @@ async function runContentSync(options) {
171
228
  assertSegmentationMode(pairConfig.contentSegmentation, `pair "${pairKey}"`);
172
229
  }
173
230
 
231
+ // Per-pair tally of what each pair's fallback method did (lib/fallback.js)
232
+ // — one [FALLBACK] line per pair at the end, and the --json summary.
233
+ const fallbackTallies = new Map();
234
+ for (const [pairKey, pairConfig] of pairEntries) {
235
+ if (pairConfig.fallback) fallbackTallies.set(pairKey, newFallbackReport(pairConfig.fallback));
236
+ }
237
+ const tallyFallback = (pairKey, report, page = null) => {
238
+ const tally = fallbackTallies.get(pairKey);
239
+ if (!tally) return;
240
+ addToTally(tally, report);
241
+ if (page && report && (report.accepted > 0 || report.cached > 0)) notePage(pairKey, page);
242
+ };
243
+ // A page some of whose text the fallback produced (its answer or its
244
+ // cache): named in the [FALLBACK] line.
245
+ const notePage = (pairKey, page) => {
246
+ const tally = fallbackTallies.get(pairKey);
247
+ if (tally && !tally.produced.includes(page)) tally.produced.push(page);
248
+ };
249
+
174
250
  output.info(`Content sync: ${sourceFiles.length} source file(s) × ${pairEntries.length} language(s), concurrency: ${concurrency}`);
175
251
  if (dryRun) output.info('Dry-run mode — no content files will be written.');
176
252
 
@@ -179,56 +255,117 @@ async function runContentSync(options) {
179
255
  let skipped = 0;
180
256
 
181
257
  const syncStartTime = Date.now();
258
+ const failedItems = []; // { file, locale, error } — listed at the end
259
+ // Files left as is because edits made by hand could not be merged with a
260
+ // source change — { file, target, locale, reason }; listed at the end.
261
+ const heldItems = [];
262
+ let keptEditFiles = 0;
263
+ let fatalError = null;
264
+ // Blocks and front-matter fields the quality gate refused, per translation
265
+ // (lib/content-refusals.js): what each run held back and what it learned.
266
+ // Applied to the lock at the end, whatever each page's outcome.
267
+ const refusalStates = new Map(); // manifestKey → { prior, refused, filled, held, … }
268
+ let heldBackSeen = 0; // blocks/fields held back (a dry run: would be)
182
269
 
183
- for (let fileIdx = 0; fileIdx < sourceFiles.length; fileIdx++) {
184
- const sourcePath = sourceFiles[fileIdx];
185
- const relPath = path.relative(contentDir, sourcePath);
186
- const fileNum = fileIdx + 1;
187
- const totalFiles = sourceFiles.length;
188
-
189
- // ETA calculation
190
- let etaStr = '';
191
- if (fileIdx > 0) {
192
- const elapsedMs = Date.now() - syncStartTime;
193
- const msPerFile = elapsedMs / fileIdx;
194
- const remainingMs = msPerFile * (totalFiles - fileIdx);
195
- const remainingMin = Math.ceil(remainingMs / 60000);
196
- etaStr = remainingMin > 1 ? ` (~${remainingMin} min remaining)` : '';
270
+ // A written-record that does not parse must not silently turn into "no
271
+ // edits": say so, then fall back to the pre-record behaviour for that file.
272
+ const readRecord = (recordKey, label) => {
273
+ try {
274
+ return parseWrittenRecord(contentManifest[recordKey]);
275
+ } catch (err) {
276
+ output.warn(
277
+ `${CONTENT_LOCK_FILENAME}: the record of ${label} is unreadable (${err.message}) — ` +
278
+ 'edits made by hand to that translation cannot be recognised this run. It is rewritten the next time sync writes the file.'
279
+ );
280
+ return null;
197
281
  }
198
- output.info(`[${fileNum}/${totalFiles}] ${relPath}${etaStr}`);
199
-
200
- // Read source file once — shared across all locale translations
201
- const raw = fs.readFileSync(sourcePath, 'utf-8');
202
- const currentSourceHash = hashFileContent(sourcePath);
282
+ };
203
283
 
204
- // Parallelize across locales for this file
205
- const perPairResults = await pMap(pairEntries, async ([pairKey, pairConfig]) => {
206
- const code = pairConfig.target;
207
- const result = { translated: false, fallback: false, skipped: false, retranslated: false };
284
+ try {
285
+ for (let fileIdx = 0; fileIdx < sourceFiles.length; fileIdx++) {
286
+ const sourcePath = sourceFiles[fileIdx];
287
+ const relPath = path.relative(contentDir, sourcePath);
288
+ if (fileScope && !fileScope.includes(relPath)) continue;
289
+ const retranslate = fileScope ? fileScope.retranslates(relPath) : false;
290
+ const fileNum = fileIdx + 1;
291
+ const totalFiles = sourceFiles.length;
208
292
 
209
- const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
293
+ // ETA calculation
294
+ let etaStr = '';
295
+ if (fileIdx > 0) {
296
+ const elapsedMs = Date.now() - syncStartTime;
297
+ const msPerFile = elapsedMs / fileIdx;
298
+ const remainingMs = msPerFile * (totalFiles - fileIdx);
299
+ const remainingMin = Math.ceil(remainingMs / 60000);
300
+ etaStr = remainingMin > 1 ? ` (~${remainingMin} min remaining)` : '';
301
+ }
302
+ output.info(`[${fileNum}/${totalFiles}] ${relPath}${etaStr}`);
210
303
 
211
- // Security: verify target path stays within content directory
212
- if (!isPathContained(targetPath, contentDir)) {
213
- output.error(`${code} — refusing to write outside content directory`);
214
- return result;
304
+ // Read and parse the source once — shared across all locale translations
305
+ // (parsing is stateless; nothing below mutates the parsed parts).
306
+ const raw = fs.readFileSync(sourcePath, 'utf-8');
307
+ const currentSourceHash = hashFileContent(sourcePath);
308
+ const { frontMatter, rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parseContentFile(raw);
309
+ // The translatable front-matter values — the per-field TM unit, and
310
+ // what the written-record fingerprints.
311
+ const sourceFields = {};
312
+ if (hasFrontMatter) {
313
+ for (const field of fieldsList) {
314
+ if (frontMatter[field] && typeof frontMatter[field] === 'string') sourceFields[field] = frontMatter[field];
315
+ }
215
316
  }
216
317
 
217
- // Change detection for existing files
218
- const manifestKey = `${relPath}:${code}`;
318
+ // Parallelize across locales for this file
319
+ const translateOne = async ([pairKey, pairConfig]) => {
320
+ const code = pairConfig.target;
321
+ const result = { translated: false, fallback: false, skipped: false, retranslated: false, held: false, keptEdits: false };
219
322
 
220
- if (fs.existsSync(targetPath)) {
323
+ const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
324
+
325
+ // Security: verify target path stays within content directory
326
+ if (!isPathContained(targetPath, contentDir)) {
327
+ output.error(`${code} — refusing to write outside content directory`);
328
+ return result;
329
+ }
330
+
331
+ // Change detection for existing files
332
+ const manifestKey = `${relPath}:${code}`;
333
+ // What sync last left on disk for this translation — how edits a
334
+ // person made to it are told apart from ours (lib/content-review.js).
335
+ const recordKey = writtenRecordKey(manifestKey);
336
+ const targetRel = path.relative(contentDir, targetPath);
337
+ const segMode = pairConfig.contentSegmentation || 'block';
338
+ const targetExists = fs.existsSync(targetPath);
221
339
  const storedHash = contentManifest[manifestKey];
222
- if (storedHash && storedHash === currentSourceHash) {
340
+ const sourceCurrent = Boolean(storedHash) && storedHash === currentSourceHash;
341
+ let edits = null; // a person's paragraphs and fields to keep (state 'edited')
342
+
343
+ if (!retranslate && targetExists && sourceCurrent && !forceContent) {
344
+ // Up to date — not touched. A translation no record exists for
345
+ // (written by an older version) is recorded now, so edits made to
346
+ // it from here on are recognised when its source changes.
347
+ if (contentManifest[recordKey] === undefined) {
348
+ const tmKeys = [tmMethodKey(pairConfig), pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null].filter(Boolean);
349
+ updatedManifest[recordKey] = buildBootstrapRecord({
350
+ sourceBody: body,
351
+ sourceFields,
352
+ written: fs.readFileSync(targetPath, 'utf-8'),
353
+ // Exact cached machine translations of a source text (read
354
+ // directly: a comparison, not a served hit).
355
+ machineFor: (text) => tmKeys
356
+ .map(k => tm[cacheKey(text, code, k)])
357
+ .filter(e => e && typeof e.t === 'string')
358
+ .map(e => e.t),
359
+ });
360
+ }
223
361
  result.skipped = true;
224
362
  return result;
225
363
  }
226
- if (storedHash && storedHash !== currentSourceHash) {
227
- output.info(`${code} — source updated, re-translating`);
228
- result.retranslated = true;
229
- } else if (!storedHash) {
364
+
365
+ if (!retranslate && targetExists && !storedHash) {
230
366
  // No stored hash — check if this file was generated by a prior
231
- // champollion run (contains [EN] fallback markers) or is a genuine
367
+ // champollion run (a legacy '[EN] ' fallback marker, written before
368
+ // 2026-10-05 — such pages are now recorded as pending:<hash>) or is a genuine
232
369
  // hand-translated file that should be preserved.
233
370
  //
234
371
  // BUG FIX: Previously, this always skipped hashless files.
@@ -238,340 +375,813 @@ async function runContentSync(options) {
238
375
  const isLegacyFallback = existingContent.includes('[EN] ');
239
376
  if (!isLegacyFallback) {
240
377
  updatedManifest[manifestKey] = currentSourceHash;
378
+ // Recorded as a person's file: each paragraph is kept when the
379
+ // source later changes elsewhere (before, the first source edit
380
+ // replaced the whole hand translation).
381
+ updatedManifest[recordKey] = buildAdoptedRecord({ sourceBody: body, sourceFields, written: existingContent });
241
382
  result.skipped = true;
383
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'kept-hand-translated' });
242
384
  return result;
243
385
  }
244
- output.info(`${code} — replacing [EN] fallback`);
245
386
  }
246
- }
247
387
 
248
- if (dryRun) {
249
- const targetRel = path.relative(contentDir, targetPath);
250
- output.info(`Would create: ${targetRel}`);
251
- result.translated = true;
252
- return result;
253
- }
388
+ if (targetExists) {
389
+ // Before rewriting a translation: what did a person change in it?
390
+ const record = readRecord(recordKey, targetRel);
391
+ if (record) {
392
+ const targetRaw = fs.readFileSync(targetPath, 'utf-8');
393
+ const verdict = assessExistingTarget({
394
+ targetRaw,
395
+ record,
396
+ segMode,
397
+ replaceEdits: retranslate || (forceContent && namedByFiles),
398
+ sourceCurrent,
399
+ });
400
+ if (verdict.action === 'keep') {
401
+ // --force-content on an up-to-date file whose edits cannot be
402
+ // merged: nothing is out of date, so nothing is replaced.
403
+ output.info(
404
+ `${code} — kept ${targetRel} as is: it was edited by hand (${verdict.reason}). ` +
405
+ `To replace it with machine translation: champollion sync --redo files:${relPath}`
406
+ );
407
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'kept-edits' });
408
+ result.skipped = true;
409
+ return result;
410
+ }
411
+ if (verdict.action === 'accept') {
412
+ // Held on an earlier run and edited since: the person brought
413
+ // it up to date. It is current now — and theirs.
414
+ updatedManifest[manifestKey] = currentSourceHash;
415
+ updatedManifest[recordKey] = buildAdoptedRecord({ sourceBody: body, sourceFields, written: targetRaw });
416
+ output.info(`${code} — ${dryRun ? 'would take' : 'took'} your updated ${targetRel} as up to date with ${relPath}`);
417
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'accepted-edits' });
418
+ result.skipped = true;
419
+ return result;
420
+ }
421
+ if (verdict.action === 'hold') {
422
+ // Lock NOT advanced: this warning repeats every run until the
423
+ // file is brought up to date by hand or named for a redo.
424
+ updatedManifest[recordKey] = { ...contentManifest[recordKey], held: verdict.heldHash };
425
+ output.warn(
426
+ `${targetRel} was left as is: it was edited by hand (${verdict.reason}), so the changes to ` +
427
+ `${relPath} cannot be merged into it paragraph by paragraph. Bring it up to date by hand ` +
428
+ '(the next sync then takes it as current), or replace it with machine translation: ' +
429
+ `champollion sync --redo files:${relPath}`
430
+ );
431
+ heldItems.push({ file: relPath, target: targetRel, locale: code, reason: verdict.reason });
432
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'held-edits' });
433
+ result.held = true;
434
+ return result;
435
+ }
436
+ if (verdict.replaced) {
437
+ const r = verdict.replaced;
438
+ const what = r.state === 'edited'
439
+ ? describeEdits({ paragraphs: r.ownedBlocks, fields: Object.keys(r.fields).filter(f => r.fields[f].owned) })
440
+ : 'the file';
441
+ output.info(
442
+ `${code} — ${dryRun ? 'would replace' : 'replacing'} the edits made by hand to ${what} of ${targetRel} ` +
443
+ `(you named it with ${retranslate ? '--retranslate' : '--redo files: / --files'})`
444
+ );
445
+ }
446
+ edits = verdict.edits;
447
+ }
448
+ }
254
449
 
255
- // Key gate — require only what the pair's resolved method actually
256
- // needs (see readinessCache above). Fail loud with the method's own
257
- // reason instead of blaming OPENROUTER_API_KEY unconditionally.
258
- const readiness = await checkPairReadiness(pairKey, pairConfig);
259
- if (!readiness.ready) {
260
- throw new Error(
261
- `Content sync for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
262
- ` ${readiness.reason}\n` +
263
- ' Set the key it names in .env.local (or your environment) to translate content.'
264
- );
265
- }
450
+ if (retranslate) {
451
+ // --retranslate: the operator named this file — skip the lock and
452
+ // the adoption rule, and make its text miss the TM (below).
453
+ result.retranslated = targetExists;
454
+ if (!dryRun) output.info(`${code} — re-translating (--retranslate)`);
455
+ } else if (targetExists) {
456
+ if (sourceCurrent) {
457
+ output.info(`${code} — re-processing (--force-content; cached text is reused)`);
458
+ result.retranslated = true;
459
+ } else if (storedHash === pendingLockValue(currentSourceHash)) {
460
+ output.info(`${code} — re-checking what was left in the source language last time (cached text is free)`);
461
+ result.retranslated = true;
462
+ } else if (storedHash) {
463
+ output.info(`${code} — source updated, re-translating`);
464
+ result.retranslated = true;
465
+ } else {
466
+ output.info(`${code} — replacing an older '[EN] ' fallback`);
467
+ }
468
+ }
266
469
 
267
- // Parse source (shared raw content, parsing is stateless)
268
- const { frontMatter, rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parseContentFile(raw);
470
+ if (dryRun) {
471
+ const keeping = edits
472
+ ? ` (keeping the edits made by hand to ${describeEdits({ paragraphs: edits.ownedBlocks, fields: Object.keys(edits.fields).filter(f => edits.fields[f].owned) })})`
473
+ : '';
474
+ // What the quality gate refused before is held back by the real run
475
+ // (lib/content-refusals.js) — said here, not "would create".
476
+ const preview = previewHeld({
477
+ tm, code, pairConfig, fields: sourceFields, body, segMode,
478
+ holds: contentHolds(readRefusals(contentManifest, manifestKey), pairConfig, { redo: retranslate || forceContent || noTM }),
479
+ });
480
+ if (preview.pageHeld) {
481
+ output.info(`Would hold back ${targetRel}: ${preview.names.join(', ')} refused before — nothing sent, not written. `
482
+ + `Ask again: \`${contentRedoCommand(relPath, { pair: pairKey })}\``);
483
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'would-hold', held: preview.names.length });
484
+ result.heldBack = preview.names.length;
485
+ return result;
486
+ }
487
+ const holding = preview.names.length > 0
488
+ ? ` (holding back ${preview.names.join(', ')}: refused before — not sent; \`${contentRedoCommand(relPath, { pair: pairKey })}\` asks again)`
489
+ : '';
490
+ output.info(`Would ${targetExists ? 'update' : 'create'}: ${targetRel}${keeping}${holding}`);
491
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'would-translate', ...(preview.names.length > 0 && { held: preview.names.length }) });
492
+ result.translated = true;
493
+ return result;
494
+ }
269
495
 
270
- // Never silently drop translatable-looking nested/array front matter
271
- // (e.g. `related:` lists) — surface it once per source file.
272
- if (hasFrontMatter && !warnedFrontMatter.has(sourcePath)) {
273
- warnedFrontMatter.add(sourcePath);
274
- const skipped = findUntranslatableNestedFields(rawFrontMatter);
275
- if (skipped.length > 0) {
276
- output.warn(
277
- `${relPath}: front matter field(s) [${skipped.join(', ')}] are arrays/nested — ` +
278
- `left untranslated. Flatten them to top-level strings to translate, or translate by hand.`
496
+ // Key gate — require only what the pair's resolved method actually
497
+ // needs (see readinessCache above). Fail loud with the method's own
498
+ // reason instead of blaming OPENROUTER_API_KEY unconditionally.
499
+ const readiness = await checkPairReadiness(pairKey, pairConfig);
500
+ if (!readiness.ready) {
501
+ const err = new Error(
502
+ `Content sync for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
503
+ ` ${readiness.reason}\n` +
504
+ missingKeyAdvice({ reasons: [readiness.reason], setupHelp: [' Set the key it names in .env.local (or your environment) to translate content.'] }).join('\n')
279
505
  );
506
+ err.fatal = true; // every file would fail the same way — stop now
507
+ throw err;
280
508
  }
281
- }
282
509
 
283
- // TM entries are keyed on the full method key (method|model|register|
284
- // coaching) — switching any of those must re-translate, not re-serve.
285
- const tmKey = tmMethodKey(pairConfig);
510
+ // The source was parsed once per file above (frontMatter, body, …).
511
+ if (retranslate) {
512
+ // Fresh translation for every field, the body and each block of
513
+ // this file — new results still replace the cached ones.
514
+ const fields = hasFrontMatter
515
+ ? fieldsList.map(f => frontMatter[f]).filter(v => typeof v === 'string')
516
+ : [];
517
+ bypassTMFor(tm, code, [...fields, body, ...translatableBlockSources(body)]);
518
+ }
286
519
 
287
- // Translate front matter fields — TM first, API for the misses.
288
- // Each field is cached on its own source text, exactly like a
289
- // key-value sync key: a title edit re-pays only the title.
290
- const translatedFields = {};
291
- if (hasFrontMatter) {
292
- const fieldsToTranslate = {};
293
- for (const field of fieldsList) {
294
- if (frontMatter[field] && typeof frontMatter[field] === 'string') {
295
- fieldsToTranslate[field] = frontMatter[field];
520
+ // Refused before (lib/content-refusals.js): blocks and fields the gate
521
+ // refused this pair's method for are not sent to it again — unless this
522
+ // page is named for a redo (--redo files: / --redo content /
523
+ // --retranslate / --fresh). Recorded below whatever the page's outcome.
524
+ const refusal = {
525
+ file: relPath, locale: code, pageHeld: false,
526
+ prior: readRefusals(contentManifest, manifestKey),
527
+ refused: [], filled: new Set(), held: [],
528
+ fields: sourceFields,
529
+ blockSources: segMode === 'block' && body.trim() ? translatableBlockSources(body) : [],
530
+ pageSource: segMode === 'page' && body.trim() ? body : null,
531
+ };
532
+ refusalStates.set(manifestKey, refusal);
533
+ const holds = contentHolds(refusal.prior, pairConfig, { redo: retranslate || forceContent || noTM });
534
+ // A front-matter field held back fails its page, as a refused one does:
535
+ // nothing of the page is sent, nothing written, the lock not advanced.
536
+ const holdPage = (names) => {
537
+ refusal.held.push(...names);
538
+ refusal.pageHeld = true;
539
+ output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, pageHeld: true, handWritten: true }));
540
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'held-refused', held: names.length });
541
+ result.heldBack = names.length;
542
+ return result;
543
+ };
544
+ // A page translated whole that the gate refused before (and that the
545
+ // cache does not hold): held before anything of it is sent — its
546
+ // front matter included (a pure look; the body read below serves).
547
+ if (refusal.pageSource && holds.page(body) === 'held'
548
+ && ![tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])]
549
+ .some(k => peekTM(tm, body, code, k) !== null)) {
550
+ return holdPage([PAGE_NAME]);
551
+ }
552
+
553
+ // Never silently drop translatable-looking nested/array front matter
554
+ // (e.g. `related:` lists) — surface it once per source file.
555
+ if (hasFrontMatter && !warnedFrontMatter.has(sourcePath)) {
556
+ warnedFrontMatter.add(sourcePath);
557
+ const skipped = findUntranslatableNestedFields(rawFrontMatter);
558
+ if (skipped.length > 0) {
559
+ output.warn(
560
+ `${relPath}: front matter field(s) [${skipped.join(', ')}] are arrays/nested — ` +
561
+ `left untranslated. Flatten them to top-level strings to translate, or translate by hand.`
562
+ );
296
563
  }
297
564
  }
298
565
 
299
- const { hits: fmHits, misses: fmMisses } = partitionByTM(
300
- tm, fieldsToTranslate, Object.keys(fieldsToTranslate), code, tmKey
301
- );
302
- // Validate cached hits BEFORE serving. A cache is a time machine: an
303
- // entry stored before the content-preservation gate existed re-serves
304
- // exactly what that gate now rejects — hollowed titles sat here and
305
- // were re-served forever, because TM hits skipped every gate. A hit
306
- // that fails today's gate is evicted and re-billed as a miss.
307
- for (const [field, cachedValue] of Object.entries(fmHits)) {
308
- if (checkContentPreservation(fieldsToTranslate[field], cachedValue)) {
309
- evictTM(tm, fieldsToTranslate[field], code, tmKey);
310
- delete fmHits[field];
311
- fmMisses.push(field);
566
+ // This page's cached content through the repeat check, in one batch
567
+ // and before any of it is served (lib/fallback.js): a text cached for
568
+ // several different source strings is evicted, so the lookups below
569
+ // miss it and it is translated again, checked the same way. What a
570
+ // person's edits keep is not served from the cache, so not checked.
571
+ {
572
+ const keptFields = hasFrontMatter && edits && edits.ownedFields > 0
573
+ ? matchEditedFields(edits.fields, sourceFields).keep : {};
574
+ const keptParagraphs = edits && edits.blocks && edits.ownedBlocks > 0
575
+ ? matchEditedBlocks(edits.blocks, body.trim() ? translatableBlockSources(body) : []).keep : new Map();
576
+ const repeated = refuseRepeatedCachedPage({
577
+ label: `content:${relPath}`,
578
+ fields: hasFrontMatter ? Object.fromEntries(Object.entries(sourceFields).filter(([f]) => !(f in keptFields))) : {},
579
+ body,
580
+ skipBlocks: new Set(keptParagraphs.keys()),
581
+ wholeBody: keptParagraphs.size === 0,
582
+ tm, code, pairConfig,
583
+ sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
584
+ });
585
+ if (repeated.length > 0) {
586
+ output.warn(
587
+ `${relPath} → ${code}: cached ${repeated.join(', ')} held the same text as other, different source strings ` +
588
+ '(a memorized sentence, not a translation) — removed from the cache and translated again.'
589
+ );
312
590
  }
313
591
  }
314
- Object.assign(translatedFields, fmHits);
315
- tmSegmentHits += Object.keys(fmHits).length;
316
-
317
- if (fmMisses.length > 0) {
318
- output.progress(` [SYNC] ${code} front matter (${pairConfig.method})...`);
319
- const fmResult = await translateBatch(
320
- fmMisses,
321
- fieldsToTranslate,
322
- pairConfig,
323
- { apiKey, cwd, model: pairConfig.model, batchSize: pairConfig.batchSize || 30 },
324
- );
325
- if (fmResult) {
326
- // Content-preservation gate. Front matter (title, description,
327
- // summary) went from the API straight to disk AND into the TM
328
- // with no validation of any kind — which is how a hollowed
329
- // page title was written silently and then cached. Check before
330
- // either. Failing the file is consistent with the null-result
331
- // branch below: nothing written, manifest not advanced, retried.
332
- for (const [field, value] of Object.entries(fmResult)) {
333
- const sourceValue = fieldsToTranslate[field];
334
- if (typeof value !== 'string' || typeof sourceValue !== 'string') continue;
335
- const hollowed = checkContentPreservation(sourceValue, value);
336
- if (hollowed) {
337
- output.raw(' [ERR]');
338
- throw new Error(
339
- `Content sync for ${code}: front matter "${field}" — ${hollowed.reason}.\n` +
340
- ` source: ${JSON.stringify(sourceValue)}\n` +
341
- ` got: ${JSON.stringify(value)}\n` +
342
- ' Nothing was written or cached. If this is a low-coverage target language,\n' +
343
- ' the model has no vocabulary for this string — fix the prompt or the pair.'
344
- );
345
- }
592
+
593
+ // TM entries are keyed on the full method key (method|model|register|
594
+ // coaching). A method/register/coaching change re-translates; a MODEL
595
+ // change reuses the previous model's entries unless
596
+ // --fresh-on-model-change (see "Model carry-over" in lib/tm.js).
597
+ const tmKey = tmMethodKey(pairConfig);
598
+
599
+ // Translate front matter fields — TM first, API for the misses.
600
+ // Each field is cached on its own source text, exactly like a
601
+ // key-value sync key: a title edit re-pays only the title.
602
+ const translatedFields = {};
603
+ // A field left in the source language (refused, or held from an earlier
604
+ // refusal): the page is written, its lock marked pending.
605
+ let fieldsLeftInSource = false;
606
+ // Edits made by hand that this write keeps, and the ones whose
607
+ // source text changed under them (re-translated; wording printed).
608
+ const keptFieldNames = new Set();
609
+ const keptBlocks = new Set(); // source block positions
610
+ const supersededNotes = [];
611
+ if (hasFrontMatter) {
612
+ const fieldsToTranslate = { ...sourceFields };
613
+ if (edits && edits.ownedFields > 0) {
614
+ const { keep, superseded } = matchEditedFields(edits.fields, sourceFields);
615
+ for (const [field, text] of Object.entries(keep)) {
616
+ // A person's value for an unchanged source value: written as
617
+ // is, never sent to the API, never cached as machine output.
618
+ translatedFields[field] = text;
619
+ keptFieldNames.add(field);
620
+ delete fieldsToTranslate[field];
346
621
  }
347
- Object.assign(translatedFields, fmResult);
348
- // Cache only what the API actually returned (the same non-null
349
- // check that gates writing it to the target file).
350
- for (const [field, value] of Object.entries(fmResult)) {
351
- if (typeof value === 'string' && typeof fieldsToTranslate[field] === 'string') {
352
- storeTM(tm, fieldsToTranslate[field], code, tmKey, value);
353
- }
622
+ for (const s of superseded) {
623
+ supersededNotes.push(
624
+ `${targetRel}: front matter "${s.field}" had been edited by hand, but its source value has changed, ` +
625
+ `so it was re-translated. The edited wording, to re-apply if it still fits: ${JSON.stringify(s.text)}`
626
+ );
354
627
  }
355
- output.raw(' [OK]');
356
- } else {
357
- // Front matter translation failed — loud error, skip this file
358
- output.raw(' [ERR]');
359
- throw new Error(
360
- `Content sync for ${code}: front matter translation returned no results.\n` +
361
- ' Check your API key and method configuration.'
362
- );
363
628
  }
364
- }
365
- }
366
629
 
367
- // Translate body — whole-body TM first (a reverted or duplicate body
368
- // is free), then block-level TM + ONE batched API call for the missed
369
- // blocks ('block' mode, the default), or the whole-page prompt
370
- // ('page' mode). A structural failure fails the file whole; a segment
371
- // the model drops TWICE degrades to the honest '[EN] ' fallback for
372
- // just that block (never cached, lock not advanced — re-fires next
373
- // sync as a TM-cheap retry). Mirrors docusaurus-sync.js.
374
- let translatedBody = body;
375
- let bodyUsedFallback = false;
376
- if (body.trim()) {
377
- // Read-time validation: a whole-body entry cached by a gateless
378
- // pipeline must not be re-served once the gate exists (evicts on fail).
379
- const cachedBody = lookupTMValidated(tm, body, code, tmKey,
380
- (src, cached) => !checkContentPreservation(src, cached));
381
- if (cachedBody !== null) {
382
- translatedBody = cachedBody;
383
- tmSegmentHits += 1;
384
- } else {
385
- const segMode = pairConfig.contentSegmentation || 'block';
386
- const { protectedBody, blocks } = protectBlocks(body);
387
- const promptOptions = {
388
- sourceLanguageName: DEFAULT_REGISTERS[sourceLocale]?.name || sourceLocale,
389
- promptContext: pairConfig.promptContext || null,
390
- };
391
-
392
- // Block stores are deferred until the reassembled body passes the
393
- // orphaned-placeholder check — the TM must never hold a value that
394
- // would fail the gate on re-serve.
395
- const pendingBlockStores = [];
396
- let apiCalled = false;
397
-
398
- if (segMode === 'page') {
399
- // Whole-page prompt — the pre-segmentation single-call behavior.
400
- output.progress(` [SYNC] ${code} body (${pairConfig.method})...`);
401
- apiCalled = true;
402
- const prompt = buildContentPrompt(protectedBody, pairConfig, promptOptions);
403
- const bodyResult = await translateRawContent(prompt, {
404
- apiKey,
405
- cwd,
630
+ const { hits: fmHits, misses: fmMisses } = partitionByTM(
631
+ tm, fieldsToTranslate, Object.keys(fieldsToTranslate), code, tmKey
632
+ );
633
+ // Validate cached hits BEFORE serving. A cache is a time machine: an
634
+ // entry stored before the content-preservation gate existed re-serves
635
+ // exactly what that gate now rejects — hollowed titles sat here and
636
+ // were re-served forever, because TM hits skipped every gate. A hit
637
+ // that fails today's gate is evicted and re-billed as a miss.
638
+ for (const [field, cachedValue] of Object.entries(fmHits)) {
639
+ if (contentGateFault(fieldsToTranslate[field], cachedValue, pairConfig)) {
640
+ evictTM(tm, fieldsToTranslate[field], code, tmKey);
641
+ delete fmHits[field];
642
+ fmMisses.push(field);
643
+ }
644
+ }
645
+ Object.assign(translatedFields, fmHits);
646
+ tmSegmentHits += Object.keys(fmHits).length;
647
+ // Then the fallback's cache: fields it translated on an earlier run
648
+ // are reused, not re-sent to the primary (lib/fallback.js ladder).
649
+ const fbCache = serveFieldsFromFallbackCache(tm, fieldsToTranslate, fmMisses, code, pairConfig);
650
+ Object.assign(translatedFields, fbCache.hits);
651
+ tmSegmentHits += Object.keys(fbCache.hits).length;
652
+ if (Object.keys(fbCache.hits).length > 0) notePage(pairKey, relPath);
653
+ const fieldsToSend = fbCache.misses;
654
+ // Filled without a call: the cache, a person's value.
655
+ for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
656
+ // Refused before: held back (the page with it), or the fallback's only.
657
+ const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
658
+ if (heldFields.length > 0) {
659
+ // Not sent again: the field keeps its source text and the page is
660
+ // written (its lock marked pending).
661
+ fieldsLeftInSource = true;
662
+ const names = heldFields.map(f => `front matter "${f}"`);
663
+ refusal.held.push(...names);
664
+ output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true }));
665
+ for (const f of heldFields) fieldsToSend.splice(fieldsToSend.indexOf(f), 1);
666
+ }
667
+ const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
668
+
669
+ if (fieldsToSend.length > 0) {
670
+ output.progress(` [SYNC] ${code} front matter (${pairConfig.method})...`);
671
+ // The pair's method, then its fallback (if any) for every field
672
+ // it returned nothing for or hollowed. Content-preservation gate
673
+ // on every value: front matter (title, description, summary)
674
+ // once went from the API straight to disk AND into the TM with no
675
+ // validation — which is how a hollowed page title was written
676
+ // silently and then cached. Nothing is cached until no field is
677
+ // left failing; a failing field fails the file (nothing written,
678
+ // manifest not advanced, retried).
679
+ const fm = await translateFieldsWithFallback({
680
+ fields: fieldsToTranslate,
681
+ misses: fieldsToSend,
406
682
  pairConfig,
683
+ budget: fallbackBudget,
684
+ sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
685
+ label: `content:${relPath} front matter`,
686
+ translate: (keys, cfg, extra = {}) => translateBatch(
687
+ keys, fieldsToTranslate, cfg,
688
+ { apiKey, cwd, model: cfg.model, batchSize: cfg.batchSize || 30, ...extra },
689
+ ),
690
+ fallbackOnly: fallbackOnlyFields,
407
691
  });
408
- if (!bodyResult) {
409
- // Body translation returned null — loud error
692
+ tallyFallback(pairKey, fm.report, relPath);
693
+ // Refused fields are remembered (held back from the method that
694
+ // refused them), whatever happens to the page below.
695
+ for (const [field, methods] of Object.entries(fm.refusedBy || {})) {
696
+ refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
697
+ }
698
+ if (fm.hollowed.length > 0) {
699
+ // Refused, and refused again when asked with the reason: the
700
+ // field keeps its source text; the page is still written.
701
+ fieldsLeftInSource = true;
702
+ for (const h of fm.hollowed) {
703
+ const advice = h.sharedOutput
704
+ ? ` ${pairConfig.method} answers it only with a sentence it gave for other strings — `
705
+ + `${pairConfig.fallback ? 'and the fallback did not translate it either' : 'add a "fallback" method to the pair'}, or write it in ${targetRel} by hand (kept).`
706
+ : '';
707
+ output.warn(
708
+ `${relPath} → ${code}: front matter "${h.field}" kept in the source language — ${h.reason}`
709
+ + `${h.fallbackReason ? `; the fallback (${pairConfig.fallback.method}): ${h.fallbackReason}` : ''}.${advice}`
710
+ );
711
+ }
712
+ output.warn(describeNewHold({ file: relPath, pairKey, pairConfig, count: fm.hollowed.length }));
713
+ }
714
+ if (fm.noResults) {
715
+ // Front matter translation failed — loud error, skip this file
410
716
  output.raw(' [ERR]');
411
717
  throw new Error(
412
- `Content sync body for ${code}: translation returned no results.\n` +
718
+ `Content sync for ${code}: front matter translation returned no results`
719
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
413
720
  ' Check your API key and method configuration.'
414
721
  );
415
722
  }
416
- translatedBody = restoreBlocks(bodyResult, blocks);
417
- } else {
418
- // Block mode: segment the PROTECTED body (placeholders are
419
- // single tokens, so they can never be split), serve blocks from
420
- // the TM, and batch the misses into one API call.
421
- const segments = splitBlocks(protectedBody);
422
-
423
- // TM keys use each block's RESTORED source text: placeholder
424
- // numbering is positional per file, so the protected text of an
425
- // identical paragraph differs across files/edits, while the
426
- // restored text is stable and self-contained. Cached values are
427
- // likewise restored — they must never carry another file's
428
- // placeholder ids.
429
- const rendered = segments.map(seg => ({
430
- seg,
431
- source: restoreBlocks(seg.text, blocks),
432
- out: null,
433
- }));
434
-
435
- const missed = [];
436
- for (const r of rendered) {
437
- if (r.seg.type !== 'translatable') {
438
- // Separators + passthrough blocks: copied verbatim, never billed.
439
- r.out = r.source;
440
- continue;
441
- }
442
- const cached = lookupTMValidated(tm, r.source, code, tmKey,
443
- (src, c) => !checkContentPreservation(src, c));
444
- if (cached !== null) {
445
- r.out = cached;
446
- tmSegmentHits += 1;
447
- } else {
448
- missed.push(r);
449
- }
723
+ // Cache only gate-passing values, each under the TM key of the
724
+ // method that produced it.
725
+ for (const st of fm.stores) storeTM(tm, st.text, code, st.tmKey, st.value);
726
+ // A field only the fallback may be asked for that it did not
727
+ // translate (skipped by --max-cost, or no answer): still refused by
728
+ // the pair's method, so the page is held, as with no fallback.
729
+ const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
730
+ if (unfilled.length > 0) {
731
+ fieldsLeftInSource = true;
732
+ const names = unfilled.map(f => `front matter "${f}"`);
733
+ refusal.held.push(...names);
734
+ output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true }));
450
735
  }
736
+ Object.assign(translatedFields, fm.translated);
737
+ for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
738
+ output.raw(' [OK]');
739
+ }
740
+ }
741
+
742
+ // Translate body — whole-body TM first (a reverted or duplicate body
743
+ // is free), then block-level TM + ONE batched API call for the missed
744
+ // blocks ('block' mode, the default), or the whole-page prompt
745
+ // ('page' mode). A structural failure fails the file whole; a segment
746
+ // the model drops TWICE degrades to the honest '[EN] ' fallback for
747
+ // just that block (never cached, lock not advanced — re-fires next
748
+ // sync as a TM-cheap retry). Mirrors docusaurus-sync.js.
749
+ let translatedBody = body;
750
+ let bodyUsedFallback = false;
751
+ // Paragraphs a person edited whose source paragraph is unchanged:
752
+ // kept word for word (block mode only — assessExistingTarget never
753
+ // lets a 'page'-mode file with edited paragraphs get this far). The
754
+ // rest are reported, even when the source body is now empty.
755
+ let keepPlan = null;
756
+ if (edits && edits.blocks && edits.ownedBlocks > 0) {
757
+ keepPlan = matchEditedBlocks(edits.blocks, body.trim() ? translatableBlockSources(body) : []);
758
+ for (const s of keepPlan.superseded) {
759
+ supersededNotes.push(
760
+ `${targetRel}: paragraph ${s.paragraph} had been edited by hand, but the source paragraph it ` +
761
+ 'translates has changed, so it was re-translated. The edited wording, to re-apply if it still fits:\n' +
762
+ s.text.split('\n').map(line => ` ${line}`).join('\n')
763
+ );
764
+ }
765
+ }
766
+ if (body.trim()) {
767
+ // Read-time validation: a whole-body entry cached by a gateless
768
+ // pipeline must not be re-served once the gate exists (evicts on fail).
769
+ // The pair's cache, then its fallback's (lib/fallback.js ladder).
770
+ // Not when a person's paragraphs are kept: a whole-body hit would
771
+ // put the machine wording back over them.
772
+ const cachedBody = keepPlan && keepPlan.keep.size > 0
773
+ ? null
774
+ : lookupContentTM(tm, body, code, pairConfig);
775
+ if (cachedBody !== null) {
776
+ translatedBody = cachedBody.text;
777
+ tmSegmentHits += 1;
778
+ for (const src of refusal.blockSources) refusal.filled.add(blockUnit(src));
779
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
780
+ } else {
781
+ const { protectedBody, blocks } = protectBlocks(body);
782
+ const promptOptions = {
783
+ sourceLanguageName: DEFAULT_REGISTERS[sourceLocale]?.name || sourceLocale,
784
+ promptContext: pairConfig.promptContext || null,
785
+ protectedTerms: pairConfig.protectedTerms || [],
786
+ };
787
+
788
+ // Block stores are deferred until the reassembled body passes the
789
+ // orphaned-placeholder check — the TM must never hold a value that
790
+ // would fail the gate on re-serve. Each carries the TM key of the
791
+ // method that produced it (the pair's, or its fallback's).
792
+ const pendingBlockStores = [];
793
+ let apiCalled = false;
794
+ // The whole body is cached under ONE method's key, so only when
795
+ // one method produced all of it (a body mixing the primary's and
796
+ // the fallback's blocks is cached block by block only).
797
+ let wholeBodyKey = tmKey;
798
+ let mixedProducers = false;
799
+ // Page mode: whose whole page the checks below refuse (the lane
800
+ // remembers it — lib/content-refusals.js "page").
801
+ let pageRefusedBy = null;
451
802
 
452
- if (missed.length > 0) {
453
- output.progress(` [SYNC] ${code} body (${missed.length} block(s), ${pairConfig.method})...`);
803
+ if (segMode === 'page') {
804
+ // Refused before by this method (and the cache does not hold
805
+ // it): not sent again (lib/content-refusals.js).
806
+ const pageHold = holds.page(body);
807
+ if (pageHold === 'held') return holdPage([PAGE_NAME]);
808
+ // Whole-page prompt — the pre-segmentation single-call behavior.
809
+ output.progress(` [SYNC] ${code} body (${pairConfig.method})...`);
454
810
  apiCalled = true;
455
- // Terminology context for the block-batch prompt: the page's
456
- // title (front matter first, else the first H1 in the body).
457
- const pageTitle =
458
- (hasFrontMatter && typeof frontMatter.title === 'string' ? frontMatter.title : null)
459
- || (body.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? null);
460
- // Self-repair ladder (translateBlockBatchResilient): full
461
- // batch → one missing-segments-only retry → honest
462
- // '[EN] '-prefixed source for anything still missing. A
463
- // duplicate/unknown marker (untrustworthy mapping) or an
464
- // empty first response still fails the file whole.
465
- let batchOutcome;
466
- try {
467
- batchOutcome = await translateBlockBatchResilient({
468
- texts: missed.map(r => r.seg.text),
469
- buildPrompt: (texts) => buildBlockBatchPrompt(
470
- texts, pairConfig, { ...promptOptions, pageTitle }),
471
- callModel: (prompt) => translateRawContent(prompt, {
472
- apiKey,
473
- cwd,
474
- pairConfig,
475
- }),
476
- fallbackPrefix,
811
+ const runPage = (cfg) => translateRawContent(buildContentPrompt(protectedBody, cfg, promptOptions), {
812
+ apiKey,
813
+ cwd,
814
+ pairConfig: cfg,
815
+ });
816
+ let bodyResult;
817
+ if (pairConfig.fallback) {
818
+ // The fallback translates the page when the primary returns
819
+ // nothing, damages a placeholder or hollows it. When both
820
+ // fail, the primary's page goes to the checks below, which
821
+ // fail the file exactly as without a fallback.
822
+ // Refused before by the pair's method: only the fallback is asked.
823
+ const page = await translatePageWithFallback({
824
+ body, blocks, pairConfig, runPage, budget: fallbackBudget, fallbackOnly: pageHold === 'fallback-only',
477
825
  });
478
- } catch (batchErr) {
826
+ tallyFallback(pairKey, page.report, relPath);
827
+ if (page.refusedBy.length > 0) {
828
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: page.refusedBy });
829
+ }
830
+ if (pageHold === 'fallback-only' && page.body === null) {
831
+ // The fallback did not translate it: the pair's method
832
+ // refused it before, so it is not asked — refused now by
833
+ // the fallback too, or held (no answer, or skipped by
834
+ // --max-cost).
835
+ if (page.refusedBy.length === 0) return holdPage([PAGE_NAME]);
836
+ output.raw(' [ERR]');
837
+ const err = new Error(
838
+ `Content sync body for ${code}: the fallback (${pairConfig.fallback.method}) failed the whole-page checks too.\n` +
839
+ ` Nothing was written or cached. ${describeNewHold({ file: relPath, pairKey, pairConfig, count: 1 })}`
840
+ );
841
+ err.heldNext = true;
842
+ throw err;
843
+ }
844
+ bodyResult = page.body ?? page.primaryBody;
845
+ if (page.body !== null) wholeBodyKey = page.tmKey;
846
+ else if (page.refusedBy.length > 0) pageRefusedBy = page.refusedBy;
847
+ } else {
848
+ const raw = await runPage(pairConfig);
849
+ bodyResult = raw ? restoreBlocks(raw, blocks) : null;
850
+ if (bodyResult) pageRefusedBy = [tmKey];
851
+ }
852
+ if (!bodyResult) {
853
+ // Body translation returned null — loud error
479
854
  output.raw(' [ERR]');
480
- throw batchErr;
855
+ throw new Error(
856
+ `Content sync body for ${code}: translation returned no results`
857
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
858
+ ' Check your API key and method configuration.'
859
+ );
860
+ }
861
+ translatedBody = bodyResult;
862
+ } else {
863
+ // Block mode: segment the PROTECTED body (placeholders are
864
+ // single tokens, so they can never be split), serve blocks from
865
+ // the TM, and batch the misses into one API call.
866
+ const segments = splitBlocks(protectedBody);
867
+
868
+ // TM keys use each block's RESTORED source text: placeholder
869
+ // numbering is positional per file, so the protected text of an
870
+ // identical paragraph differs across files/edits, while the
871
+ // restored text is stable and self-contained. Cached values are
872
+ // likewise restored — they must never carry another file's
873
+ // placeholder ids.
874
+ const rendered = segments.map(seg => ({
875
+ seg,
876
+ source: restoreBlocks(seg.text, blocks),
877
+ out: null,
878
+ }));
879
+
880
+ const missed = [];
881
+ const heldBlocks = []; // refused before: not sent, the source text kept
882
+ let position = 0; // index among translatable blocks (= translatableBlockSources order)
883
+ for (const r of rendered) {
884
+ if (r.seg.type !== 'translatable') {
885
+ // Separators + passthrough blocks: copied verbatim, never billed.
886
+ r.out = r.source;
887
+ continue;
888
+ }
889
+ const j = position++;
890
+ // Its name in the repeat check (content:<file>#<n>, as verify names it).
891
+ r.pos = j;
892
+ if (keepPlan && keepPlan.keep.has(j)) {
893
+ // A person's paragraph: written as is, not billed, not cached.
894
+ r.out = keepPlan.keep.get(j);
895
+ keptBlocks.add(j);
896
+ refusal.filled.add(blockUnit(r.source));
897
+ continue;
898
+ }
899
+ const cached = lookupContentTM(tm, r.source, code, pairConfig);
900
+ if (cached !== null) {
901
+ r.out = cached.text;
902
+ tmSegmentHits += 1;
903
+ if (cached.fromFallback) { mixedProducers = true; notePage(pairKey, relPath); }
904
+ refusal.filled.add(blockUnit(r.source));
905
+ continue;
906
+ }
907
+ const hold = holds.block(r.source);
908
+ if (hold === 'held') {
909
+ // Refused before by this method (and its fallback): the
910
+ // honest last resort stays, nothing is sent or billed.
911
+ r.out = r.source;
912
+ heldBlocks.push(r);
913
+ continue;
914
+ }
915
+ r.fallbackOnly = hold === 'fallback-only';
916
+ missed.push(r);
481
917
  }
482
- const { blocks: translatedBlocks, fellBack } = batchOutcome;
483
- const fellBackSet = new Set(fellBack);
484
- if (fellBack.length > 0) {
918
+ if (heldBlocks.length > 0) {
485
919
  bodyUsedFallback = true;
486
- output.raw(' [FALLBACK]');
487
- output.warn(
488
- `Content sync body for ${code}: ${fellBack.length} of ${missed.length} ` +
489
- `block(s) missing from the model response after a retry — written as ` +
490
- `'${fallbackPrefix}'-prefixed source. Not cached, lock not advanced: ` +
491
- `the next sync retries just those block(s).`
492
- );
920
+ const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
921
+ refusal.held.push(...names);
922
+ result.heldBack = (result.heldBack || 0) + names.length;
923
+ output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true }));
493
924
  }
494
- missed.forEach((r, i) => {
495
- r.out = restoreBlocks(translatedBlocks[i], blocks);
925
+
926
+ if (missed.length > 0) {
927
+ output.progress(` [SYNC] ${code} body (${missed.length} block(s), ${pairConfig.method})...`);
928
+ apiCalled = true;
929
+ // Terminology context for the block-batch prompt: the page's
930
+ // title (front matter first, else the first H1 in the body).
931
+ const pageTitle =
932
+ (hasFrontMatter && typeof frontMatter.title === 'string' ? frontMatter.title : null)
933
+ || (body.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? null);
934
+ // Self-repair ladder (translateBlockBatchResilient): full
935
+ // batch → one missing-segments-only retry → (with a fallback
936
+ // method: one batch through it for every block the primary
937
+ // dropped or damaged — lib/fallback.js) → the source text,
938
+ // unmarked, for anything still missing. A
939
+ // duplicate/unknown marker (untrustworthy mapping) or an
940
+ // empty first response still fails the file whole when no
941
+ // fallback rescues it.
942
+ let batchOutcome;
943
+ try {
944
+ batchOutcome = await translateBlocksWithFallback({
945
+ missed,
946
+ blocks,
947
+ pairConfig,
948
+ budget: fallbackBudget,
949
+ sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
950
+ label: `content:${relPath}`,
951
+ runBatch: (texts, cfg) => translateBlockBatchResilient({
952
+ texts,
953
+ buildPrompt: (t) => buildBlockBatchPrompt(t, cfg, { ...promptOptions, pageTitle }),
954
+ callModel: (prompt) => translateRawContent(prompt, {
955
+ apiKey,
956
+ cwd,
957
+ pairConfig: cfg,
958
+ }),
959
+ }),
960
+ fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
961
+ });
962
+ } catch (batchErr) {
963
+ output.raw(' [ERR]');
964
+ throw batchErr;
965
+ }
966
+ tallyFallback(pairKey, batchOutcome.report, relPath);
967
+ if (batchOutcome.fromFallback > 0) mixedProducers = true;
968
+ const { fellBack } = batchOutcome;
969
+ // Refused blocks are remembered; translated ones drop a record.
970
+ const fellSet = new Set(fellBack);
971
+ const newlyHeld = [];
972
+ missed.forEach((r, i) => {
973
+ if (!fellSet.has(i)) { refusal.filled.add(blockUnit(r.source)); return; }
974
+ const methods = batchOutcome.refusedBy?.[i] || [];
975
+ if (methods.length > 0) {
976
+ refusal.refused.push({ unit: blockUnit(r.source), source: r.source, methods });
977
+ newlyHeld.push(i);
978
+ }
979
+ });
980
+ if ((batchOutcome.sharedOutput || []).length > 0) {
981
+ output.warn(
982
+ `Content sync body for ${code}: ${batchOutcome.sharedOutput.length} block(s) of ${relPath} came back as the same text ` +
983
+ 'the model gave for other, different source strings (a memorized sentence, not a translation) — refused' +
984
+ (pairConfig.fallback
985
+ ? `; sent to the fallback (${pairConfig.fallback.method}).`
986
+ : `. ${pairConfig.method} can only answer them with that sentence: add a "fallback" method to the pair, ` +
987
+ 'or write those paragraphs by hand (a hand-written paragraph is kept).')
988
+ );
989
+ }
990
+ // Blocks the quality gate refused — the key-value gate's checks
991
+ // (length inflation, echo, truncation, script), per block.
992
+ const refusedByGate = batchOutcome.refused || [];
993
+ if (refusedByGate.length > 0) {
994
+ output.warn(
995
+ `Content sync body for ${code}: ${refusedByGate.length} block(s) of ${relPath} refused by the quality gate, also when asked again with the reason — `
996
+ + refusedByGate.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
997
+ + `${refusedByGate.length > 3 ? '; …' : ''}`
998
+ + (pairConfig.fallback
999
+ ? '.'
1000
+ : `. Add a "fallback" method to the pair (it is asked for what ${pairConfig.method}'s answer is refused for), `
1001
+ + 'or write those paragraphs by hand (a hand-written paragraph is kept).')
1002
+ );
1003
+ }
1004
+ if (fellBack.length > 0) {
1005
+ bodyUsedFallback = true;
1006
+ output.raw(' [PARTIAL]');
1007
+ // Refused as a memorized sentence (said just above) is not
1008
+ // "missing from the response".
1009
+ const refusedShared = (batchOutcome.sharedOutput || []).length + refusedByGate.length;
1010
+ const why = batchOutcome.report?.attempted > 0
1011
+ ? `neither the primary (${pairConfig.method}) nor its fallback (${pairConfig.fallback.method}) translated safely`
1012
+ : refusedShared >= fellBack.length
1013
+ ? 'refused (above)'
1014
+ : refusedShared > 0
1015
+ ? `refused (${refusedShared}, above) or missing from the model response after a retry`
1016
+ : 'missing from the model response after a retry';
1017
+ output.warn(
1018
+ `Content sync body for ${code}: ${fellBack.length} of ${missed.length} ` +
1019
+ `block(s) ${why} — left in the source language, unmarked (\`champollion status\` lists them). ` +
1020
+ describeFallenBack({
1021
+ file: relPath, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: batchOutcome.refusedBy,
1022
+ })
1023
+ );
1024
+ }
496
1025
  // Fallen-back segments are never TM-cached — an error cached
497
1026
  // is an error forever (they re-bill on the next sync).
498
- if (!fellBackSet.has(i)) {
499
- pendingBlockStores.push({ source: r.source, translation: r.out });
500
- }
501
- });
1027
+ missed.forEach((r, i) => { r.out = batchOutcome.outs[i]; });
1028
+ pendingBlockStores.push(...batchOutcome.stores);
1029
+ }
1030
+
1031
+ // Reassemble in order with the source's exact separators.
1032
+ translatedBody = rendered.map(r => r.out).join('');
502
1033
  }
503
1034
 
504
- // Reassemble in order with the source's exact separators.
505
- translatedBody = rendered.map(r => r.out).join('');
506
- }
1035
+ // A page translated whole that fails the checks below is
1036
+ // remembered as refused: the next plain sync does not send it to
1037
+ // the same method again (lib/content-refusals.js "page").
1038
+ const refusePage = (err) => {
1039
+ if (!pageRefusedBy) return err;
1040
+ if (!refusal.refused.some(r => r.unit === PAGE_UNIT)) {
1041
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: pageRefusedBy });
1042
+ }
1043
+ err.message += `\n ${describeNewHold({ file: relPath, pairKey, pairConfig, count: 1 })}`;
1044
+ err.heldNext = true;
1045
+ return err;
1046
+ };
507
1047
 
508
- // Orphaned-placeholder check on the REASSEMBLED body — the same
509
- // gate for both modes.
510
- if (hasOrphanedPlaceholders(translatedBody)) {
511
- // Placeholder corruption — loud error, skip this file
512
- output.error('PLACEHOLDER CORRUPTION');
513
- throw new Error(
514
- `Content sync body for ${code}: placeholder corruption detected.\n` +
515
- ' Code blocks were corrupted during translation. Retry or report this issue.'
516
- );
517
- }
1048
+ // Orphaned-placeholder check on the REASSEMBLED body — the same
1049
+ // gate for both modes.
1050
+ if (hasOrphanedPlaceholders(translatedBody)) {
1051
+ // Placeholder corruption — loud error, skip this file
1052
+ output.error('PLACEHOLDER CORRUPTION');
1053
+ throw refusePage(new Error(
1054
+ `Content sync body for ${code}: placeholder corruption detected.\n` +
1055
+ ' Code blocks were corrupted during translation. Retry or report this issue.'
1056
+ ));
1057
+ }
518
1058
 
519
- // Content-preservation check on the REASSEMBLED body. This lane
520
- // never ran the quality gate at all — translateBatch/
521
- // translateRawContent output went straight to disk AND into the TM
522
- // — so a body hollowed of its letters was written silently and then
523
- // re-served from cache forever. Throwing here matches the
524
- // placeholder gate above: the file is skipped, its manifest entry is
525
- // not advanced, and nothing is cached, so the next sync retries.
526
- const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
527
- if (bodyHollowed) {
528
- output.error('CONTENT LOSS');
529
- throw new Error(
530
- `Content sync body for ${code}: ${bodyHollowed.reason}.\n` +
531
- ' Nothing was written or cached. If this is a low-coverage target language,\n' +
532
- ' the model has no vocabulary for this text — fix the prompt or the pair, not the gate.'
533
- );
534
- }
1059
+ // Content-preservation check on the REASSEMBLED body. This lane
1060
+ // never ran the quality gate at all — translateBatch/
1061
+ // translateRawContent output went straight to disk AND into the TM
1062
+ // — so a body hollowed of its letters was written silently and then
1063
+ // re-served from cache forever. Throwing here matches the
1064
+ // placeholder gate above: the file is skipped, its manifest entry is
1065
+ // not advanced, and nothing is cached, so the next sync retries.
1066
+ const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
1067
+ if (bodyHollowed) {
1068
+ output.error('CONTENT LOSS');
1069
+ throw refusePage(new Error(
1070
+ `Content sync body for ${code}: ${bodyHollowed.reason}.\n` +
1071
+ ' Nothing was written or cached. If this is a low-coverage target language,\n' +
1072
+ ' the model has no vocabulary for this text — fix the prompt or the pair, not the gate.'
1073
+ ));
1074
+ }
1075
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
535
1076
 
536
- // Store per-block AND whole-body entries only AFTER the corruption
537
- // check — the whole-body entry makes reverts/duplicates free. A
538
- // fallback body is NEVER stored whole: it contains untranslated
539
- // '[EN] ' text, and an error cached is an error forever.
540
- for (const s of pendingBlockStores) {
541
- storeTM(tm, s.source, code, tmKey, s.translation);
542
- }
543
- if (!bodyUsedFallback) {
544
- storeTM(tm, body, code, tmKey, translatedBody);
1077
+ // Store per-block AND whole-body entries only AFTER the corruption
1078
+ // check — the whole-body entry makes reverts/duplicates free. A
1079
+ // fallback body is NEVER stored whole: it contains untranslated
1080
+ // '[EN] ' text, and an error cached is an error forever.
1081
+ for (const s of pendingBlockStores) {
1082
+ storeTM(tm, s.source, code, s.tmKey, s.translation);
1083
+ }
1084
+ // Never a whole-body entry holding a person's paragraphs: the TM
1085
+ // holds machine output only, under the method that produced it.
1086
+ if (!bodyUsedFallback && !mixedProducers && keptBlocks.size === 0) {
1087
+ storeTM(tm, body, code, wholeBodyKey, translatedBody);
1088
+ }
1089
+ if (apiCalled && !bodyUsedFallback) output.raw(' [OK]');
545
1090
  }
546
- if (apiCalled && !bodyUsedFallback) output.raw(' [OK]');
547
1091
  }
548
- }
549
1092
 
550
- // Reassemble and write
551
- const assembled = reassembleContentFile({
552
- rawFrontMatter, translatedFields, translatedBody,
553
- hasFrontMatter, frontMatterFormat,
554
- });
555
- fs.mkdirSync(path.dirname(targetPath), { recursive: true });
556
- fs.writeFileSync(targetPath, assembled, 'utf-8');
557
-
558
- result.translated = true;
559
- // A fallback body keeps its OLD manifest entry so the file re-fires
560
- // next sync — every good block is a TM hit, only the fallen-back
561
- // segment re-bills. Self-healing at bounded cost.
562
- if (!bodyUsedFallback) {
563
- updatedManifest[manifestKey] = currentSourceHash;
564
- }
1093
+ // Reassemble and write
1094
+ const assembled = reassembleContentFile({
1095
+ rawFrontMatter, translatedFields, translatedBody,
1096
+ hasFrontMatter, frontMatterFormat,
1097
+ });
1098
+ fs.mkdirSync(path.dirname(targetPath), { recursive: true });
1099
+ fs.writeFileSync(targetPath, assembled, 'utf-8');
1100
+
1101
+ // Record what is now on disk, block by block, with the paragraphs
1102
+ // and fields that are a person's marked — so the next sync can tell
1103
+ // a later edit from this write, and keeps these ones again.
1104
+ updatedManifest[recordKey] = buildWrittenRecord({
1105
+ sourceBody: body, sourceFields, written: assembled,
1106
+ ownedBlocks: keptBlocks, ownedFields: keptFieldNames,
1107
+ });
1108
+
1109
+ result.translated = true;
1110
+ const keptCount = keptBlocks.size + keptFieldNames.size;
1111
+ output.event('file', {
1112
+ lane: 'content', file: relPath, locale: code,
1113
+ status: bodyUsedFallback ? 'fallback' : 'translated',
1114
+ ...(keptCount > 0 && { keptEdits: keptCount }),
1115
+ });
1116
+ if (keptCount > 0) {
1117
+ result.keptEdits = true;
1118
+ output.info(
1119
+ `${code} — kept the edits made by hand to ${describeEdits({ paragraphs: keptBlocks.size, fields: [...keptFieldNames] })} ` +
1120
+ `of ${targetRel}. To re-translate them: champollion sync --redo files:${relPath}`
1121
+ );
1122
+ }
1123
+ for (const note of supersededNotes) output.warn(note);
1124
+ // A fallback body keeps its OLD manifest entry so the file re-fires
1125
+ // next sync — every good block is a TM hit, only the fallen-back
1126
+ // segment re-bills. Self-healing at bounded cost.
1127
+ // Something left in the source language: `pending:<hash>` — the page
1128
+ // is processed again next sync (cache-free; held units not re-sent)
1129
+ // and stays the tool's, not taken for a person's file.
1130
+ updatedManifest[manifestKey] = !bodyUsedFallback && !fieldsLeftInSource
1131
+ ? currentSourceHash : pendingLockValue(currentSourceHash);
1132
+
1133
+ return result;
1134
+ };
1135
+
1136
+ // One (file × locale) failing must not stop the others — or discard
1137
+ // their work: the run carries on, and the lock + TM are saved below for
1138
+ // everything that succeeded (dogfood 2026-08-28, finding 5). Only a
1139
+ // FATAL error (a method that cannot run at all) stops the run.
1140
+ const perPairResults = await pMap(pairEntries, async (entry) => {
1141
+ try {
1142
+ return await translateOne(entry);
1143
+ } catch (err) {
1144
+ if (err.fatal) throw err;
1145
+ const code = entry[1].target;
1146
+ output.error(`${relPath} → ${code} — ${err.message}`);
1147
+ // heldNext: refused by the quality gate — held back from now on,
1148
+ // not retried by the next plain sync (lib/content-refusals.js).
1149
+ failedItems.push({ file: relPath, locale: code, error: err.message, ...(err.heldNext && { heldNext: true }) });
1150
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'failed', error: err.message });
1151
+ return { translated: false, fallback: false, skipped: false, retranslated: false };
1152
+ }
1153
+ }, { concurrency });
565
1154
 
566
- return result;
567
- }, { concurrency });
1155
+ // Aggregate per-pair results into totals
1156
+ for (const r of perPairResults) {
1157
+ if (r.translated) translated++;
1158
+ if (r.skipped) skipped++;
1159
+ if (r.retranslated) retranslated++;
1160
+ if (r.keptEdits) keptEditFiles++;
1161
+ if (r.heldBack) heldBackSeen += r.heldBack;
1162
+ }
1163
+ }
1164
+ } catch (err) {
1165
+ // A fatal error still lets the saves below run: files finished before
1166
+ // it paid for their translations, and their lock entries + TM entries
1167
+ // are what make the retry cheap.
1168
+ fatalError = err;
1169
+ }
568
1170
 
569
- // Aggregate per-pair results into totals
570
- for (const r of perPairResults) {
571
- if (r.translated) translated++;
572
- if (r.skipped) skipped++;
573
- if (r.retranslated) retranslated++;
1171
+ // What each translation's refusals are now (lib/content-refusals.js):
1172
+ // earlier ones that still apply, plus this run's — held back from the
1173
+ // method that refused them until a redo names the page, the source text
1174
+ // changes, or the model/method does.
1175
+ const heldBackItems = [];
1176
+ let heldBack = 0;
1177
+ let refusedNow = 0;
1178
+ for (const [manifestKey, st] of refusalStates) {
1179
+ storeRefusals(updatedManifest, manifestKey, nextRefusals(st.prior, st));
1180
+ if (st.held.length > 0) {
1181
+ heldBack += st.held.length;
1182
+ heldBackItems.push({ file: st.file, locale: st.locale, units: st.held, ...(st.pageHeld && { pageNotWritten: true }) });
574
1183
  }
1184
+ refusedNow += new Set(st.refused.map(r => r.unit)).size;
575
1185
  }
576
1186
 
577
1187
  // Write updated content manifest (skip in dry-run)
@@ -585,23 +1195,85 @@ async function runContentSync(options) {
585
1195
 
586
1196
  // Persist TM if it was mutated during this content sync (same rationale as
587
1197
  // the key-value path in sync.js: dirty tracking, not size comparison).
588
- // Skip when --no-tm is active — nothing was cached, nothing to save.
589
- if (!dryRun && !noTM && isTMDirty(tm)) {
590
- const tmFinalSize = tmSize(tm);
591
- const delta = tmFinalSize - tmInitialSize;
1198
+ // Under --fresh/--no-tm too: what was paid for is cached.
1199
+ if (!dryRun && isTMDirty(tm)) {
592
1200
  saveTM(cwd, tm);
593
- output.info(`[TM] Saved ${tmFinalSize} entries (${delta >= 0 ? '+' + delta : delta} this sync)`);
1201
+ output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
594
1202
  }
595
1203
 
596
1204
  const totalCreated = translated;
597
- if (totalCreated > 0 || skipped > 0 || retranslated > 0) {
1205
+ if (totalCreated > 0 || skipped > 0 || retranslated > 0 || heldItems.length > 0 || (dryRun && heldBackSeen > 0)) {
598
1206
  const retranslateNote = retranslated > 0 ? ` (${retranslated} re-translated)` : '';
1207
+ const keptNote = keptEditFiles > 0 ? `, ${keptEditFiles} keeping edits made by hand` : '';
1208
+ const heldNote = heldItems.length > 0 ? `, ${heldItems.length} left as is (edited by hand — listed below)` : '';
599
1209
  if (dryRun) {
600
- output.info(`Would have created ${totalCreated} content file(s)${retranslateNote}, ${skipped} unchanged.`);
1210
+ const heldBackNote = heldBackSeen > 0 ? `; ${heldBackSeen} block(s)/field(s) refused before would be held back (not sent)` : '';
1211
+ output.info(`Would have created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}${heldBackNote}.`);
601
1212
  } else {
602
- output.ok(`Created ${totalCreated} content file(s)${retranslateNote}, ${skipped} unchanged.`);
1213
+ output.ok(`Created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}.`);
1214
+ }
1215
+ }
1216
+ if (heldBack > 0) {
1217
+ // Not sent, not billed — but not translated either (exit 2, as for a
1218
+ // held-back key). Each page was named above with its own repair.
1219
+ output.warn(
1220
+ `${heldBack} content block(s)/field(s) held back in ${heldBackItems.length} translation(s) — the quality gate refused the method's `
1221
+ + 'translation of their current text before; not sent, not billed (listed above). Ask again for all of them: '
1222
+ + '`champollion sync --redo content`; for one page: `champollion sync --redo files:<page>`.'
1223
+ );
1224
+ }
1225
+ if (heldItems.length > 0) {
1226
+ // Not a failure (nothing was lost, nothing billed) — but out of date
1227
+ // with its source, so it is listed on every run until resolved.
1228
+ output.raw('');
1229
+ output.raw(` Translations left as is because they were edited by hand and their source changed (${heldItems.length}):`);
1230
+ for (const h of heldItems) output.raw(` ${h.target} (source: ${h.file})`);
1231
+ output.raw(' Update each by hand (the next sync takes it as current), or replace it: champollion sync --redo files:<source>');
1232
+ }
1233
+
1234
+ if (!dryRun) {
1235
+ for (const [pairKey, tally] of fallbackTallies) {
1236
+ printFallbackReport(pairKey, pairs.get(pairKey).method, tally, { unit: 'content segment(s)', producedLabel: 'pages' });
1237
+ warnFallbackMajority(pairKey, pairs.get(pairKey).method, tally, { unit: 'content segment(s)' });
603
1238
  }
604
1239
  }
1240
+
1241
+ if (fatalError) throw fatalError;
1242
+ if (failedItems.length > 0) {
1243
+ output.raw('');
1244
+ output.raw(` Failed content translations (${failedItems.length}) — not recorded as done; the next sync retries them`
1245
+ + `${failedItems.some(f => f.heldNext) ? ', except what the quality gate refused (held back — `--redo files:<page>` asks again)' : ''}:`);
1246
+ for (const f of failedItems) output.raw(` ${f.file} → ${f.locale}${f.heldNext ? ' (refused by the quality gate: held back)' : ''}`);
1247
+ output.error(
1248
+ `${failedItems.length} content translation(s) failed (listed above). ` +
1249
+ 'Completed files and their translations are saved; re-run sync to retry.'
1250
+ );
1251
+ }
1252
+
1253
+ // Returned, not thrown: a partly failed run still did real work, and the
1254
+ // caller folds this into the run summary and the exit code (2 = partial).
1255
+ return {
1256
+ translated,
1257
+ skipped,
1258
+ retranslated,
1259
+ failed: failedItems.length,
1260
+ failedItems: failedItems.map(({ heldNext, ...f }) => ({ ...f, state: heldNext ? 'held-back' : 'will-retry' })),
1261
+ // Blocks/fields the quality gate refused before, not sent this run (the
1262
+ // '[EN] ' text stays; a page with a held front-matter field is not
1263
+ // written), and blocks/fields it refused this run (held from the next).
1264
+ heldBack,
1265
+ heldBackItems,
1266
+ refused: refusedNow,
1267
+ // Translations written with a person's edits kept, and translations
1268
+ // left as is because their edits could not be merged with a source change.
1269
+ keptEdits: keptEditFiles,
1270
+ held: heldItems.length,
1271
+ heldItems,
1272
+ // What each pair's fallback method did (pairs with a fallback, real runs).
1273
+ ...(!dryRun && fallbackTallies.size > 0 && {
1274
+ fallback: [...fallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
1275
+ }),
1276
+ };
605
1277
  }
606
1278
 
607
1279
  // -----------------------------------------------------------------
@@ -642,6 +1314,9 @@ function readContentManifest(cwd) {
642
1314
  */
643
1315
  function writeContentManifest(cwd, manifest) {
644
1316
  const lockPath = path.join(cwd, CONTENT_LOCK_FILENAME);
1317
+ // Nothing to record and no lock yet (e.g. the method could not run at
1318
+ // all): don't leave an empty lock file in a folder the user never synced.
1319
+ if (Object.keys(manifest).length === 0 && !fs.existsSync(lockPath)) return;
645
1320
  const sorted = {};
646
1321
  for (const key of Object.keys(manifest).sort()) {
647
1322
  sorted[key] = manifest[key];
@@ -673,18 +1348,28 @@ function hashFileContent(filePath) {
673
1348
  *
674
1349
  * Local file I/O only — cheap relative to the API dollars it estimates.
675
1350
  *
676
- * @param {string} contentDir - Path to Hugo content directory
1351
+ * @param {string} contentDir - Path to the content directory
677
1352
  * @param {string} sourceLocale - Source language code
678
1353
  * @param {Array<[string, object]>} pairEntries - Pair graph entries [pairKey, pairConfig]
679
1354
  * @param {string} cwd - Project root (for the content lock file)
1355
+ * @param {object} [options]
1356
+ * @param {object} [options.tm] - Loaded TM. When given, each pending item also
1357
+ * reports billedChars: only the front-matter fields and body blocks the TM
1358
+ * does NOT hold — what the run will actually pay for (lib/content-estimate.js).
1359
+ * @param {string[]|null} [options.translatableFields] - Front-matter fields
1360
+ * (null → DEFAULT_TRANSLATABLE_FIELDS, as runContentSync)
1361
+ * @param {boolean} [options.fresh] - --fresh: nothing refused before is held back
680
1362
  * @returns {{ sourceFileCount: number, pendingTranslations: number, pendingSourceChars: number,
681
- * byTarget: Object<string, { pendingTranslations: number, pendingSourceChars: number }> }}
1363
+ * byTarget: Object<string, { pendingTranslations: number, pendingSourceChars: number,
1364
+ * billedChars: number|null }> }}
682
1365
  * pendingSourceChars sums the source file characters of every pending
683
1366
  * (file × pair) translation — the rough basis for token estimation.
684
1367
  * byTarget breaks the same counts down per target code so the cost table
685
1368
  * can price each pair with its own method.
686
1369
  */
687
- function countPendingContentTranslations(contentDir, sourceLocale, pairEntries, cwd) {
1370
+ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries, cwd, {
1371
+ tm = null, translatableFields = null, fileScope = null, forceContent = false, fresh = false,
1372
+ } = {}) {
688
1373
  const result = { sourceFileCount: 0, pendingTranslations: 0, pendingSourceChars: 0, byTarget: {} };
689
1374
  if (!contentDir || !fs.existsSync(contentDir)) return result;
690
1375
 
@@ -693,9 +1378,13 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
693
1378
  if (sourceFiles.length === 0) return result;
694
1379
 
695
1380
  const contentManifest = readContentManifest(cwd);
1381
+ const fieldsList = translatableFields || DEFAULT_TRANSLATABLE_FIELDS;
1382
+ const parsedCache = new Map(); // sourcePath → { parsed, blocks } (locale-independent)
696
1383
 
697
1384
  for (const sourcePath of sourceFiles) {
698
1385
  const relPath = path.relative(contentDir, sourcePath);
1386
+ if (fileScope && !fileScope.includes(relPath)) continue;
1387
+ const retranslate = fileScope ? fileScope.retranslates(relPath) : false;
699
1388
  const raw = fs.readFileSync(sourcePath, 'utf-8');
700
1389
  const currentSourceHash = crypto.createHash('sha256').update(raw, 'utf-8').digest('hex');
701
1390
 
@@ -705,27 +1394,129 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
705
1394
  if (!isPathContained(targetPath, contentDir)) continue;
706
1395
 
707
1396
  const manifestKey = `${relPath}:${code}`;
708
- if (fs.existsSync(targetPath)) {
709
- const storedHash = contentManifest[manifestKey];
710
- if (storedHash && storedHash === currentSourceHash) continue; // unchanged — skipped by sync
1397
+ const targetExists = fs.existsSync(targetPath);
1398
+ const storedHash = contentManifest[manifestKey];
1399
+ if (!retranslate && targetExists) {
1400
+ if (storedHash && storedHash === currentSourceHash && !forceContent) continue; // unchanged — skipped by sync
711
1401
  if (!storedHash) {
712
1402
  // Hashless target: sync preserves it unless it is a legacy [EN] fallback
713
1403
  const existingContent = fs.readFileSync(targetPath, 'utf-8');
714
1404
  if (!existingContent.includes('[EN] ')) continue;
715
1405
  }
716
1406
  }
1407
+ if (targetExists) {
1408
+ // The same decision runContentSync makes (lib/content-review.js): a
1409
+ // translation edited by hand that the sync would leave as is costs
1410
+ // nothing. An unreadable record counts as pending (sync proceeds).
1411
+ let record = null;
1412
+ try { record = parseWrittenRecord(contentManifest[writtenRecordKey(manifestKey)]); } catch { record = null; }
1413
+ if (record) {
1414
+ const verdict = assessExistingTarget({
1415
+ targetRaw: fs.readFileSync(targetPath, 'utf-8'),
1416
+ record,
1417
+ segMode: pairConfig.contentSegmentation || 'block',
1418
+ replaceEdits: retranslate || (forceContent && Boolean(fileScope && fileScope.limitsFiles)),
1419
+ sourceCurrent: Boolean(storedHash) && storedHash === currentSourceHash,
1420
+ });
1421
+ if (verdict.action !== 'proceed') continue;
1422
+ }
1423
+ }
717
1424
 
718
1425
  result.pendingTranslations += 1;
719
1426
  result.pendingSourceChars += raw.length;
720
1427
  if (!result.byTarget[code]) {
721
- result.byTarget[code] = { pendingTranslations: 0, pendingSourceChars: 0 };
1428
+ result.byTarget[code] = { pendingTranslations: 0, pendingSourceChars: 0, billedChars: tm ? 0 : null };
722
1429
  }
723
1430
  result.byTarget[code].pendingTranslations += 1;
724
1431
  result.byTarget[code].pendingSourceChars += raw.length;
1432
+
1433
+ if (tm) {
1434
+ // The same TM ladder runContentSync runs: per field, whole body,
1435
+ // then per block — only misses are billed.
1436
+ if (!parsedCache.has(sourcePath)) parsedCache.set(sourcePath, { parsed: parseContentFile(raw), blocks: null });
1437
+ const cached = parsedCache.get(sourcePath);
1438
+ const fields = {};
1439
+ if (cached.parsed.hasFrontMatter) {
1440
+ for (const field of fieldsList) {
1441
+ const v = cached.parsed.frontMatter[field];
1442
+ if (v && typeof v === 'string') fields[field] = v;
1443
+ }
1444
+ }
1445
+ const { billedChars, totalChars } = billableContentChars({
1446
+ tm,
1447
+ code,
1448
+ tmKey: tmMethodKey(pairConfig),
1449
+ fallbackTmKey: pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null,
1450
+ fields,
1451
+ body: cached.parsed.body,
1452
+ segMode: pairConfig.contentSegmentation || 'block',
1453
+ blockSources: () => (cached.blocks ??= translatableBlockSources(cached.parsed.body)),
1454
+ // What the gate refused before is not sent to the method again
1455
+ // (lib/content-refusals.js) — the same holds the sync applies.
1456
+ holds: contentHolds(readRefusals(contentManifest, manifestKey), pairConfig, {
1457
+ redo: retranslate || forceContent || fresh,
1458
+ }),
1459
+ });
1460
+ // --retranslate bypasses the TM for this file: all of it is billed.
1461
+ result.byTarget[code].billedChars += retranslate ? totalChars : billedChars;
1462
+ }
725
1463
  }
726
1464
  }
727
1465
 
728
1466
  return result;
729
1467
  }
730
1468
 
731
- export { runContentSync, countPendingContentTranslations, readContentManifest, CONTENT_LOCK_FILENAME };
1469
+ /**
1470
+ * The state of the content lane (contentDir) per target locale, for
1471
+ * `champollion status` — which had shown only the key-value lane, so a
1472
+ * newsletter folder and its files' state were invisible (Round 7, school
1473
+ * persona). Local file reads only; never fails (a diagnostic).
1474
+ *
1475
+ * translated — the target exists and was made from the current source
1476
+ * (the content lock's hash matches) and nothing left untranslated
1477
+ * outOfDate — the target was made from an older source text
1478
+ * pending — no target yet, or parts the quality gate refused left in
1479
+ * the source language (the lock says `pending:<hash>`, or a
1480
+ * legacy '[EN] ' marker from before 2026-10-05 is in the file)
1481
+ * unrecorded — a target the lock has no record of (made by hand or by
1482
+ * another tool): sync keeps it as it is
1483
+ *
1484
+ * @param {string} contentDir
1485
+ * @param {string} sourceLocale
1486
+ * @param {Array<[string, object]>} pairEntries
1487
+ * @param {string} cwd
1488
+ * @param {{ fallbackPrefix?: string }} [opts] - the legacy marker to look for
1489
+ * @returns {{ dir: string, files: number, locales: Object<string, { translated: number,
1490
+ * outOfDate: string[], pending: string[], unrecorded: string[] }> }|null}
1491
+ */
1492
+ function contentStatus(contentDir, sourceLocale, pairEntries, cwd, { fallbackPrefix = '[EN] ' } = {}) {
1493
+ if (!contentDir || !fs.existsSync(contentDir)) return null;
1494
+ let sourceFiles = [];
1495
+ try { sourceFiles = discoverContentFiles(contentDir, sourceLocale); } catch { return null; }
1496
+ const manifest = readContentManifest(cwd);
1497
+ const out = { dir: path.relative(cwd, contentDir) || '.', files: sourceFiles.length, locales: {} };
1498
+ for (const [, pairConfig] of pairEntries) {
1499
+ out.locales[pairConfig.target] ??= { translated: 0, outOfDate: [], pending: [], unrecorded: [] };
1500
+ }
1501
+ for (const sourcePath of sourceFiles) {
1502
+ const relPath = path.relative(contentDir, sourcePath).split(path.sep).join('/');
1503
+ let hash;
1504
+ try { hash = hashFileContent(sourcePath); } catch { continue; }
1505
+ for (const [, pairConfig] of pairEntries) {
1506
+ const code = pairConfig.target;
1507
+ const state = out.locales[code];
1508
+ const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
1509
+ if (!fs.existsSync(targetPath)) { state.pending.push(relPath); continue; }
1510
+ let text = '';
1511
+ try { text = fs.readFileSync(targetPath, 'utf-8'); } catch { /* unreadable: by its record only */ }
1512
+ const stored = manifest[`${path.relative(contentDir, sourcePath)}:${code}`];
1513
+ if (isPendingLock(stored) || text.includes(fallbackPrefix)) state.pending.push(relPath);
1514
+ else if (!stored) state.unrecorded.push(relPath);
1515
+ else if (stored !== hash) state.outOfDate.push(relPath);
1516
+ else state.translated += 1;
1517
+ }
1518
+ }
1519
+ return out;
1520
+ }
1521
+
1522
+ export { runContentSync, countPendingContentTranslations, readContentManifest, contentStatus, CONTENT_LOCK_FILENAME };