champollion 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +83 -36
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1689 -87
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +196 -14
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +270 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1127 -344
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +726 -78
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +965 -253
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +964 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +571 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +252 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +68 -128
  70. package/lib/methods/llm.js +80 -31
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +11 -0
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +2 -1
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +880 -17
  117. package/lib/verify.js +1296 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/method-registry.json +2 -0
  126. package/shared/metric-registry.json +96 -18
  127. package/shared/schemas/champollion-plugin.schema.json +4 -0
  128. package/shared/schemas/corpora-card.schema.json +8 -2
  129. package/shared/schemas/method-index-record.schema.json +67 -0
  130. package/shared/schemas/method-registry.schema.json +4 -0
  131. package/shared/schemas/metric-registry.schema.json +55 -1
  132. package/shared/docent/corpus.json +0 -11739
@@ -1,5 +1,7 @@
1
1
  /**
2
- * Content sync — translates Hugo Markdown content files.
2
+ * Content sync — translates the Markdown/MDX files of a contentDir (a Hugo
3
+ * site's content/ or any folder of Markdown), each translation written beside
4
+ * its source as <name>.<locale>.md (lib/content.js getTargetContentPath).
3
5
  *
4
6
  * WHY THIS EXISTS: This was extracted from sync.js to reduce the
5
7
  * god-module's line count and give content translation its own
@@ -35,15 +37,31 @@
35
37
  * results that pass the existing checks (non-null API result; structural
36
38
  * block-batch validation; body placeholder-corruption check on the
37
39
  * REASSEMBLED body) are stored, and --no-tm bypasses the cache entirely.
40
+ *
41
+ * EDITS MADE BY HAND (lib/content-review.js): every file this lane writes is
42
+ * also recorded in the content lock ("written:<relPath>:<locale>"), block by
43
+ * block. Before a changed source re-translates a file, the file on disk is
44
+ * compared with that record: paragraphs and front-matter fields a person
45
+ * changed are KEPT when the source text they translate is unchanged (never
46
+ * sent to the API, never cached as machine output), and the run says so. An
47
+ * edited paragraph whose own source changed is re-translated with its edited
48
+ * wording printed; edits that cannot be matched paragraph by paragraph leave
49
+ * the whole file as is until it is updated by hand or named for a redo.
38
50
  */
39
51
 
40
52
  import fs from 'node:fs';
41
53
  import path from 'node:path';
42
54
  import crypto from 'node:crypto';
43
55
  import { translateBatch, translateRawContent, getMethod } from './translate.js';
44
- import { checkContentPreservation } from './validate.js';
45
- import { loadTM, saveTM, lookupTM, lookupTMValidated, storeTM, evictTM, isTMDirty, tmSize, tmMethodKey, partitionByTM } from './tm.js';
56
+ import { checkContentPreservation, contentGateFault } from './validate.js';
57
+ import { loadTM, saveTM, lookupTM, peekTM, storeTM, evictTM, isTMDirty, tmSize, tmMethodKey, partitionByTM, setModelCarryover, setTMReads, bypassTMFor, cacheKey, describeTMChanges, adoptLegacyCoachingKeys } from './tm.js';
58
+ import {
59
+ writtenRecordKey, parseWrittenRecord, buildWrittenRecord, buildAdoptedRecord, buildBootstrapRecord,
60
+ assessExistingTarget, matchEditedBlocks, matchEditedFields, describeEdits,
61
+ } from './content-review.js';
46
62
  import { output } from './output.js';
63
+ import { missingKeyAdvice } from './missing-key.js';
64
+ import { billableContentChars, translatableBlockSources } from './content-estimate.js';
47
65
  import { DEFAULT_REGISTERS } from './registers.js';
48
66
  import { isPathContained } from './security.js';
49
67
  import { pMap } from './concurrent.js';
@@ -63,12 +81,23 @@ import {
63
81
  splitBlocks, buildBlockBatchPrompt, parseBlockBatchResponse,
64
82
  translateBlockBatchResilient, assertSegmentationMode,
65
83
  } from './segment.js';
84
+ import {
85
+ lookupContentTM, serveFieldsFromFallbackCache, translateFieldsWithFallback,
86
+ translateBlocksWithFallback, translatePageWithFallback,
87
+ newFallbackReport, addToTally, fallbackSummary, printFallbackReport, warnFallbackMajority,
88
+ refuseRepeatedCachedPage,
89
+ } from './fallback.js';
90
+ import {
91
+ readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
92
+ describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME,
93
+ } from './content-refusals.js';
94
+ import { contentRedoCommand } from './verify.js';
66
95
 
67
96
  /**
68
- * Run content sync — translate Hugo Markdown content files.
97
+ * Run content sync — translate the contentDir's Markdown/MDX files.
69
98
  *
70
99
  * @param {object} options
71
- * @param {string} options.contentDir - Path to Hugo content directory
100
+ * @param {string} options.contentDir - Path to the content directory (any folder of Markdown/MDX)
72
101
  * @param {string} options.sourceLocale - Source language code
73
102
  * @param {Map<string, object>} options.pairs - Resolved pair graph (pairKey → pairConfig)
74
103
  * @param {string[]|null} options.translatableFields - Front matter fields to translate
@@ -77,8 +106,13 @@ import {
77
106
  * (gemini, openai, anthropic, deepl, …) resolve their own env keys inside
78
107
  * their method classes — a missing OpenRouter key must NOT block them.
79
108
  * @param {boolean} options.dryRun - Whether to write files
109
+ * @param {boolean} [options.freshOnModelChange] - --fresh-on-model-change: serve
110
+ * only exact-model TM hits (default reuses other models' translations)
80
111
  * @param {boolean} [options.noTM] - Bypass the Translation Memory (--no-tm):
81
112
  * every segment goes to the API and nothing is cached.
113
+ * @param {object|null} [options.fallbackBudget] - lib/fallback.js
114
+ * createFallbackBudget(): the --max-cost guard for pairs' fallback methods
115
+ * (null = no cap).
82
116
  */
83
117
  async function runContentSync(options) {
84
118
  const {
@@ -89,12 +123,22 @@ async function runContentSync(options) {
89
123
  apiKey,
90
124
  dryRun = false,
91
125
  noTM = false,
126
+ freshOnModelChange = false,
127
+ // --force-content: re-process up-to-date files (TM serves unchanged
128
+ // text, so this rebuilds from cache rather than re-billing).
129
+ forceContent = false,
130
+ // --files / --retranslate (lib/file-scope.js); null = every file.
131
+ fileScope = null,
92
132
  cwd = process.cwd(),
93
133
  concurrency = 48,
94
134
  // The honest-fallback marker (config.js default). Written in front of a
95
135
  // block's SOURCE text when the model drops its segment twice — visible,
96
136
  // never silent, never cached (see translateBlockBatchResilient).
97
137
  fallbackPrefix = '[EN] ',
138
+ fallbackBudget = null,
139
+ // (code) => the locale's different-inputs-same-output index, shared with
140
+ // the key-value sync of the same run (lib/validate.js SharedOutputIndex).
141
+ sharedOutputsFor = null,
98
142
  } = options;
99
143
 
100
144
  if (!fs.existsSync(contentDir)) {
@@ -110,6 +154,11 @@ async function runContentSync(options) {
110
154
 
111
155
  const fieldsList = translatableFields || DEFAULT_TRANSLATABLE_FIELDS;
112
156
 
157
+ // --redo files:<glob> (= --files + --force-content) and --retranslate name
158
+ // files: the one way a run replaces edits made by hand to a translation.
159
+ // A bare --force-content / --redo content keeps them.
160
+ const namedByFiles = Boolean(fileScope && fileScope.limitsFiles);
161
+
113
162
  // Warn at most once per source file about translatable-looking front matter
114
163
  // we can't reach (arrays / nested blocks). Never silently drop them. The
115
164
  // check + add are synchronous (no await between), so this is race-free.
@@ -129,10 +178,18 @@ async function runContentSync(options) {
129
178
  //
130
179
  // Safe under pMap concurrency: storeTM is a synchronous property assignment
131
180
  // and Node is single-threaded, so writes can't interleave between awaits.
132
- const tm = noTM ? { _meta: { version: 1 } } : loadTM(cwd);
181
+ // --fresh/--no-tm: serve nothing from the cache, but cache what is paid for
182
+ // (lib/tm.js setTMReads) — same as the key-value path in sync.js.
183
+ const tm = loadTM(cwd);
184
+ if (noTM) setTMReads(tm, false);
185
+ if (freshOnModelChange) setModelCarryover(tm, false); // see lib/tm.js "Model carry-over"
186
+ // Blocks a pair cached before its coaching was keyed, with today's
187
+ // coaching, stay served (lib/tm.js adoptLegacyCoachingKeys — sync records
188
+ // it first; a dry run, which saves nothing, records it here too).
189
+ adoptLegacyCoachingKeys(tm, pairs.values());
133
190
  const tmInitialSize = tmSize(tm);
134
191
  if (noTM) {
135
- output.info('Content sync: Translation Memory disabled (--no-tm)');
192
+ output.info('Content sync: Translation Memory not read this run — content is translated fresh, and cached');
136
193
  } else if (tmInitialSize > 0) {
137
194
  output.info(`Content sync: Translation Memory loaded (${tmInitialSize} cached entries)`);
138
195
  }
@@ -171,6 +228,25 @@ async function runContentSync(options) {
171
228
  assertSegmentationMode(pairConfig.contentSegmentation, `pair "${pairKey}"`);
172
229
  }
173
230
 
231
+ // Per-pair tally of what each pair's fallback method did (lib/fallback.js)
232
+ // — one [FALLBACK] line per pair at the end, and the --json summary.
233
+ const fallbackTallies = new Map();
234
+ for (const [pairKey, pairConfig] of pairEntries) {
235
+ if (pairConfig.fallback) fallbackTallies.set(pairKey, newFallbackReport(pairConfig.fallback));
236
+ }
237
+ const tallyFallback = (pairKey, report, page = null) => {
238
+ const tally = fallbackTallies.get(pairKey);
239
+ if (!tally) return;
240
+ addToTally(tally, report);
241
+ if (page && report && (report.accepted > 0 || report.cached > 0)) notePage(pairKey, page);
242
+ };
243
+ // A page some of whose text the fallback produced (its answer or its
244
+ // cache): named in the [FALLBACK] line.
245
+ const notePage = (pairKey, page) => {
246
+ const tally = fallbackTallies.get(pairKey);
247
+ if (tally && !tally.produced.includes(page)) tally.produced.push(page);
248
+ };
249
+
174
250
  output.info(`Content sync: ${sourceFiles.length} source file(s) × ${pairEntries.length} language(s), concurrency: ${concurrency}`);
175
251
  if (dryRun) output.info('Dry-run mode — no content files will be written.');
176
252
 
@@ -179,54 +255,114 @@ async function runContentSync(options) {
179
255
  let skipped = 0;
180
256
 
181
257
  const syncStartTime = Date.now();
258
+ const failedItems = []; // { file, locale, error } — listed at the end
259
+ // Files left as is because edits made by hand could not be merged with a
260
+ // source change — { file, target, locale, reason }; listed at the end.
261
+ const heldItems = [];
262
+ let keptEditFiles = 0;
263
+ let fatalError = null;
264
+ // Blocks and front-matter fields the quality gate refused, per translation
265
+ // (lib/content-refusals.js): what each run held back and what it learned.
266
+ // Applied to the lock at the end, whatever each page's outcome.
267
+ const refusalStates = new Map(); // manifestKey → { prior, refused, filled, held, … }
268
+ let heldBackSeen = 0; // blocks/fields held back (a dry run: would be)
182
269
 
183
- for (let fileIdx = 0; fileIdx < sourceFiles.length; fileIdx++) {
184
- const sourcePath = sourceFiles[fileIdx];
185
- const relPath = path.relative(contentDir, sourcePath);
186
- const fileNum = fileIdx + 1;
187
- const totalFiles = sourceFiles.length;
188
-
189
- // ETA calculation
190
- let etaStr = '';
191
- if (fileIdx > 0) {
192
- const elapsedMs = Date.now() - syncStartTime;
193
- const msPerFile = elapsedMs / fileIdx;
194
- const remainingMs = msPerFile * (totalFiles - fileIdx);
195
- const remainingMin = Math.ceil(remainingMs / 60000);
196
- etaStr = remainingMin > 1 ? ` (~${remainingMin} min remaining)` : '';
270
+ // A written-record that does not parse must not silently turn into "no
271
+ // edits": say so, then fall back to the pre-record behaviour for that file.
272
+ const readRecord = (recordKey, label) => {
273
+ try {
274
+ return parseWrittenRecord(contentManifest[recordKey]);
275
+ } catch (err) {
276
+ output.warn(
277
+ `${CONTENT_LOCK_FILENAME}: the record of ${label} is unreadable (${err.message}) — ` +
278
+ 'edits made by hand to that translation cannot be recognised this run. It is rewritten the next time sync writes the file.'
279
+ );
280
+ return null;
197
281
  }
198
- output.info(`[${fileNum}/${totalFiles}] ${relPath}${etaStr}`);
199
-
200
- // Read source file once — shared across all locale translations
201
- const raw = fs.readFileSync(sourcePath, 'utf-8');
202
- const currentSourceHash = hashFileContent(sourcePath);
282
+ };
203
283
 
204
- // Parallelize across locales for this file
205
- const perPairResults = await pMap(pairEntries, async ([pairKey, pairConfig]) => {
206
- const code = pairConfig.target;
207
- const result = { translated: false, fallback: false, skipped: false, retranslated: false };
284
+ try {
285
+ for (let fileIdx = 0; fileIdx < sourceFiles.length; fileIdx++) {
286
+ const sourcePath = sourceFiles[fileIdx];
287
+ const relPath = path.relative(contentDir, sourcePath);
288
+ if (fileScope && !fileScope.includes(relPath)) continue;
289
+ const retranslate = fileScope ? fileScope.retranslates(relPath) : false;
290
+ const fileNum = fileIdx + 1;
291
+ const totalFiles = sourceFiles.length;
208
292
 
209
- const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
293
+ // ETA calculation
294
+ let etaStr = '';
295
+ if (fileIdx > 0) {
296
+ const elapsedMs = Date.now() - syncStartTime;
297
+ const msPerFile = elapsedMs / fileIdx;
298
+ const remainingMs = msPerFile * (totalFiles - fileIdx);
299
+ const remainingMin = Math.ceil(remainingMs / 60000);
300
+ etaStr = remainingMin > 1 ? ` (~${remainingMin} min remaining)` : '';
301
+ }
302
+ output.info(`[${fileNum}/${totalFiles}] ${relPath}${etaStr}`);
210
303
 
211
- // Security: verify target path stays within content directory
212
- if (!isPathContained(targetPath, contentDir)) {
213
- output.error(`${code} — refusing to write outside content directory`);
214
- return result;
304
+ // Read and parse the source once — shared across all locale translations
305
+ // (parsing is stateless; nothing below mutates the parsed parts).
306
+ const raw = fs.readFileSync(sourcePath, 'utf-8');
307
+ const currentSourceHash = hashFileContent(sourcePath);
308
+ const { frontMatter, rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parseContentFile(raw);
309
+ // The translatable front-matter values — the per-field TM unit, and
310
+ // what the written-record fingerprints.
311
+ const sourceFields = {};
312
+ if (hasFrontMatter) {
313
+ for (const field of fieldsList) {
314
+ if (frontMatter[field] && typeof frontMatter[field] === 'string') sourceFields[field] = frontMatter[field];
315
+ }
215
316
  }
216
317
 
217
- // Change detection for existing files
218
- const manifestKey = `${relPath}:${code}`;
318
+ // Parallelize across locales for this file
319
+ const translateOne = async ([pairKey, pairConfig]) => {
320
+ const code = pairConfig.target;
321
+ const result = { translated: false, fallback: false, skipped: false, retranslated: false, held: false, keptEdits: false };
219
322
 
220
- if (fs.existsSync(targetPath)) {
323
+ const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
324
+
325
+ // Security: verify target path stays within content directory
326
+ if (!isPathContained(targetPath, contentDir)) {
327
+ output.error(`${code} — refusing to write outside content directory`);
328
+ return result;
329
+ }
330
+
331
+ // Change detection for existing files
332
+ const manifestKey = `${relPath}:${code}`;
333
+ // What sync last left on disk for this translation — how edits a
334
+ // person made to it are told apart from ours (lib/content-review.js).
335
+ const recordKey = writtenRecordKey(manifestKey);
336
+ const targetRel = path.relative(contentDir, targetPath);
337
+ const segMode = pairConfig.contentSegmentation || 'block';
338
+ const targetExists = fs.existsSync(targetPath);
221
339
  const storedHash = contentManifest[manifestKey];
222
- if (storedHash && storedHash === currentSourceHash) {
340
+ const sourceCurrent = Boolean(storedHash) && storedHash === currentSourceHash;
341
+ let edits = null; // a person's paragraphs and fields to keep (state 'edited')
342
+
343
+ if (!retranslate && targetExists && sourceCurrent && !forceContent) {
344
+ // Up to date — not touched. A translation no record exists for
345
+ // (written by an older version) is recorded now, so edits made to
346
+ // it from here on are recognised when its source changes.
347
+ if (contentManifest[recordKey] === undefined) {
348
+ const tmKeys = [tmMethodKey(pairConfig), pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null].filter(Boolean);
349
+ updatedManifest[recordKey] = buildBootstrapRecord({
350
+ sourceBody: body,
351
+ sourceFields,
352
+ written: fs.readFileSync(targetPath, 'utf-8'),
353
+ // Exact cached machine translations of a source text (read
354
+ // directly: a comparison, not a served hit).
355
+ machineFor: (text) => tmKeys
356
+ .map(k => tm[cacheKey(text, code, k)])
357
+ .filter(e => e && typeof e.t === 'string')
358
+ .map(e => e.t),
359
+ });
360
+ }
223
361
  result.skipped = true;
224
362
  return result;
225
363
  }
226
- if (storedHash && storedHash !== currentSourceHash) {
227
- output.info(`${code} — source updated, re-translating`);
228
- result.retranslated = true;
229
- } else if (!storedHash) {
364
+
365
+ if (!retranslate && targetExists && !storedHash) {
230
366
  // No stored hash — check if this file was generated by a prior
231
367
  // champollion run (contains [EN] fallback markers) or is a genuine
232
368
  // hand-translated file that should be preserved.
@@ -238,340 +374,807 @@ async function runContentSync(options) {
238
374
  const isLegacyFallback = existingContent.includes('[EN] ');
239
375
  if (!isLegacyFallback) {
240
376
  updatedManifest[manifestKey] = currentSourceHash;
377
+ // Recorded as a person's file: each paragraph is kept when the
378
+ // source later changes elsewhere (before, the first source edit
379
+ // replaced the whole hand translation).
380
+ updatedManifest[recordKey] = buildAdoptedRecord({ sourceBody: body, sourceFields, written: existingContent });
241
381
  result.skipped = true;
382
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'kept-hand-translated' });
242
383
  return result;
243
384
  }
244
- output.info(`${code} — replacing [EN] fallback`);
245
385
  }
246
- }
247
386
 
248
- if (dryRun) {
249
- const targetRel = path.relative(contentDir, targetPath);
250
- output.info(`Would create: ${targetRel}`);
251
- result.translated = true;
252
- return result;
253
- }
387
+ if (targetExists) {
388
+ // Before rewriting a translation: what did a person change in it?
389
+ const record = readRecord(recordKey, targetRel);
390
+ if (record) {
391
+ const targetRaw = fs.readFileSync(targetPath, 'utf-8');
392
+ const verdict = assessExistingTarget({
393
+ targetRaw,
394
+ record,
395
+ segMode,
396
+ replaceEdits: retranslate || (forceContent && namedByFiles),
397
+ sourceCurrent,
398
+ });
399
+ if (verdict.action === 'keep') {
400
+ // --force-content on an up-to-date file whose edits cannot be
401
+ // merged: nothing is out of date, so nothing is replaced.
402
+ output.info(
403
+ `${code} — kept ${targetRel} as is: it was edited by hand (${verdict.reason}). ` +
404
+ `To replace it with machine translation: champollion sync --redo files:${relPath}`
405
+ );
406
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'kept-edits' });
407
+ result.skipped = true;
408
+ return result;
409
+ }
410
+ if (verdict.action === 'accept') {
411
+ // Held on an earlier run and edited since: the person brought
412
+ // it up to date. It is current now — and theirs.
413
+ updatedManifest[manifestKey] = currentSourceHash;
414
+ updatedManifest[recordKey] = buildAdoptedRecord({ sourceBody: body, sourceFields, written: targetRaw });
415
+ output.info(`${code} — ${dryRun ? 'would take' : 'took'} your updated ${targetRel} as up to date with ${relPath}`);
416
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'accepted-edits' });
417
+ result.skipped = true;
418
+ return result;
419
+ }
420
+ if (verdict.action === 'hold') {
421
+ // Lock NOT advanced: this warning repeats every run until the
422
+ // file is brought up to date by hand or named for a redo.
423
+ updatedManifest[recordKey] = { ...contentManifest[recordKey], held: verdict.heldHash };
424
+ output.warn(
425
+ `${targetRel} was left as is: it was edited by hand (${verdict.reason}), so the changes to ` +
426
+ `${relPath} cannot be merged into it paragraph by paragraph. Bring it up to date by hand ` +
427
+ '(the next sync then takes it as current), or replace it with machine translation: ' +
428
+ `champollion sync --redo files:${relPath}`
429
+ );
430
+ heldItems.push({ file: relPath, target: targetRel, locale: code, reason: verdict.reason });
431
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'held-edits' });
432
+ result.held = true;
433
+ return result;
434
+ }
435
+ if (verdict.replaced) {
436
+ const r = verdict.replaced;
437
+ const what = r.state === 'edited'
438
+ ? describeEdits({ paragraphs: r.ownedBlocks, fields: Object.keys(r.fields).filter(f => r.fields[f].owned) })
439
+ : 'the file';
440
+ output.info(
441
+ `${code} — ${dryRun ? 'would replace' : 'replacing'} the edits made by hand to ${what} of ${targetRel} ` +
442
+ `(you named it with ${retranslate ? '--retranslate' : '--redo files: / --files'})`
443
+ );
444
+ }
445
+ edits = verdict.edits;
446
+ }
447
+ }
254
448
 
255
- // Key gate — require only what the pair's resolved method actually
256
- // needs (see readinessCache above). Fail loud with the method's own
257
- // reason instead of blaming OPENROUTER_API_KEY unconditionally.
258
- const readiness = await checkPairReadiness(pairKey, pairConfig);
259
- if (!readiness.ready) {
260
- throw new Error(
261
- `Content sync for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
262
- ` ${readiness.reason}\n` +
263
- ' Set the key it names in .env.local (or your environment) to translate content.'
264
- );
265
- }
449
+ if (retranslate) {
450
+ // --retranslate: the operator named this file — skip the lock and
451
+ // the adoption rule, and make its text miss the TM (below).
452
+ result.retranslated = targetExists;
453
+ if (!dryRun) output.info(`${code} — re-translating (--retranslate)`);
454
+ } else if (targetExists) {
455
+ if (sourceCurrent) {
456
+ output.info(`${code} — re-processing (--force-content; cached text is reused)`);
457
+ result.retranslated = true;
458
+ } else if (storedHash) {
459
+ output.info(`${code} — source updated, re-translating`);
460
+ result.retranslated = true;
461
+ } else {
462
+ output.info(`${code} — replacing [EN] fallback`);
463
+ }
464
+ }
266
465
 
267
- // Parse source (shared raw content, parsing is stateless)
268
- const { frontMatter, rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parseContentFile(raw);
466
+ if (dryRun) {
467
+ const keeping = edits
468
+ ? ` (keeping the edits made by hand to ${describeEdits({ paragraphs: edits.ownedBlocks, fields: Object.keys(edits.fields).filter(f => edits.fields[f].owned) })})`
469
+ : '';
470
+ // What the quality gate refused before is held back by the real run
471
+ // (lib/content-refusals.js) — said here, not "would create".
472
+ const preview = previewHeld({
473
+ tm, code, pairConfig, fields: sourceFields, body, segMode,
474
+ holds: contentHolds(readRefusals(contentManifest, manifestKey), pairConfig, { redo: retranslate || forceContent || noTM }),
475
+ });
476
+ if (preview.pageHeld) {
477
+ output.info(`Would hold back ${targetRel}: ${preview.names.join(', ')} refused before — nothing sent, not written. `
478
+ + `Ask again: \`${contentRedoCommand(relPath, { pair: pairKey })}\``);
479
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'would-hold', held: preview.names.length });
480
+ result.heldBack = preview.names.length;
481
+ return result;
482
+ }
483
+ const holding = preview.names.length > 0
484
+ ? ` (holding back ${preview.names.join(', ')}: refused before — not sent; \`${contentRedoCommand(relPath, { pair: pairKey })}\` asks again)`
485
+ : '';
486
+ output.info(`Would ${targetExists ? 'update' : 'create'}: ${targetRel}${keeping}${holding}`);
487
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'would-translate', ...(preview.names.length > 0 && { held: preview.names.length }) });
488
+ result.translated = true;
489
+ return result;
490
+ }
269
491
 
270
- // Never silently drop translatable-looking nested/array front matter
271
- // (e.g. `related:` lists) — surface it once per source file.
272
- if (hasFrontMatter && !warnedFrontMatter.has(sourcePath)) {
273
- warnedFrontMatter.add(sourcePath);
274
- const skipped = findUntranslatableNestedFields(rawFrontMatter);
275
- if (skipped.length > 0) {
276
- output.warn(
277
- `${relPath}: front matter field(s) [${skipped.join(', ')}] are arrays/nested — ` +
278
- `left untranslated. Flatten them to top-level strings to translate, or translate by hand.`
492
+ // Key gate — require only what the pair's resolved method actually
493
+ // needs (see readinessCache above). Fail loud with the method's own
494
+ // reason instead of blaming OPENROUTER_API_KEY unconditionally.
495
+ const readiness = await checkPairReadiness(pairKey, pairConfig);
496
+ if (!readiness.ready) {
497
+ const err = new Error(
498
+ `Content sync for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
499
+ ` ${readiness.reason}\n` +
500
+ missingKeyAdvice({ reasons: [readiness.reason], setupHelp: [' Set the key it names in .env.local (or your environment) to translate content.'] }).join('\n')
279
501
  );
502
+ err.fatal = true; // every file would fail the same way — stop now
503
+ throw err;
504
+ }
505
+
506
+ // The source was parsed once per file above (frontMatter, body, …).
507
+ if (retranslate) {
508
+ // Fresh translation for every field, the body and each block of
509
+ // this file — new results still replace the cached ones.
510
+ const fields = hasFrontMatter
511
+ ? fieldsList.map(f => frontMatter[f]).filter(v => typeof v === 'string')
512
+ : [];
513
+ bypassTMFor(tm, code, [...fields, body, ...translatableBlockSources(body)]);
280
514
  }
281
- }
282
515
 
283
- // TM entries are keyed on the full method key (method|model|register|
284
- // coaching) — switching any of those must re-translate, not re-serve.
285
- const tmKey = tmMethodKey(pairConfig);
516
+ // Refused before (lib/content-refusals.js): blocks and fields the gate
517
+ // refused this pair's method for are not sent to it again — unless this
518
+ // page is named for a redo (--redo files: / --redo content /
519
+ // --retranslate / --fresh). Recorded below whatever the page's outcome.
520
+ const refusal = {
521
+ file: relPath, locale: code, pageHeld: false,
522
+ prior: readRefusals(contentManifest, manifestKey),
523
+ refused: [], filled: new Set(), held: [],
524
+ fields: sourceFields,
525
+ blockSources: segMode === 'block' && body.trim() ? translatableBlockSources(body) : [],
526
+ pageSource: segMode === 'page' && body.trim() ? body : null,
527
+ };
528
+ refusalStates.set(manifestKey, refusal);
529
+ const holds = contentHolds(refusal.prior, pairConfig, { redo: retranslate || forceContent || noTM });
530
+ // A front-matter field held back fails its page, as a refused one does:
531
+ // nothing of the page is sent, nothing written, the lock not advanced.
532
+ const holdPage = (names) => {
533
+ refusal.held.push(...names);
534
+ refusal.pageHeld = true;
535
+ output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, pageHeld: true, handWritten: true, fallbackPrefix }));
536
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'held-refused', held: names.length });
537
+ result.heldBack = names.length;
538
+ return result;
539
+ };
540
+ // A page translated whole that the gate refused before (and that the
541
+ // cache does not hold): held before anything of it is sent — its
542
+ // front matter included (a pure look; the body read below serves).
543
+ if (refusal.pageSource && holds.page(body) === 'held'
544
+ && ![tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])]
545
+ .some(k => peekTM(tm, body, code, k) !== null)) {
546
+ return holdPage([PAGE_NAME]);
547
+ }
286
548
 
287
- // Translate front matter fields — TM first, API for the misses.
288
- // Each field is cached on its own source text, exactly like a
289
- // key-value sync key: a title edit re-pays only the title.
290
- const translatedFields = {};
291
- if (hasFrontMatter) {
292
- const fieldsToTranslate = {};
293
- for (const field of fieldsList) {
294
- if (frontMatter[field] && typeof frontMatter[field] === 'string') {
295
- fieldsToTranslate[field] = frontMatter[field];
549
+ // Never silently drop translatable-looking nested/array front matter
550
+ // (e.g. `related:` lists) — surface it once per source file.
551
+ if (hasFrontMatter && !warnedFrontMatter.has(sourcePath)) {
552
+ warnedFrontMatter.add(sourcePath);
553
+ const skipped = findUntranslatableNestedFields(rawFrontMatter);
554
+ if (skipped.length > 0) {
555
+ output.warn(
556
+ `${relPath}: front matter field(s) [${skipped.join(', ')}] are arrays/nested — ` +
557
+ `left untranslated. Flatten them to top-level strings to translate, or translate by hand.`
558
+ );
296
559
  }
297
560
  }
298
561
 
299
- const { hits: fmHits, misses: fmMisses } = partitionByTM(
300
- tm, fieldsToTranslate, Object.keys(fieldsToTranslate), code, tmKey
301
- );
302
- // Validate cached hits BEFORE serving. A cache is a time machine: an
303
- // entry stored before the content-preservation gate existed re-serves
304
- // exactly what that gate now rejects — hollowed titles sat here and
305
- // were re-served forever, because TM hits skipped every gate. A hit
306
- // that fails today's gate is evicted and re-billed as a miss.
307
- for (const [field, cachedValue] of Object.entries(fmHits)) {
308
- if (checkContentPreservation(fieldsToTranslate[field], cachedValue)) {
309
- evictTM(tm, fieldsToTranslate[field], code, tmKey);
310
- delete fmHits[field];
311
- fmMisses.push(field);
562
+ // This page's cached content through the repeat check, in one batch
563
+ // and before any of it is served (lib/fallback.js): a text cached for
564
+ // several different source strings is evicted, so the lookups below
565
+ // miss it and it is translated again, checked the same way. What a
566
+ // person's edits keep is not served from the cache, so not checked.
567
+ {
568
+ const keptFields = hasFrontMatter && edits && edits.ownedFields > 0
569
+ ? matchEditedFields(edits.fields, sourceFields).keep : {};
570
+ const keptParagraphs = edits && edits.blocks && edits.ownedBlocks > 0
571
+ ? matchEditedBlocks(edits.blocks, body.trim() ? translatableBlockSources(body) : []).keep : new Map();
572
+ const repeated = refuseRepeatedCachedPage({
573
+ label: `content:${relPath}`,
574
+ fields: hasFrontMatter ? Object.fromEntries(Object.entries(sourceFields).filter(([f]) => !(f in keptFields))) : {},
575
+ body,
576
+ skipBlocks: new Set(keptParagraphs.keys()),
577
+ wholeBody: keptParagraphs.size === 0,
578
+ tm, code, pairConfig,
579
+ sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
580
+ });
581
+ if (repeated.length > 0) {
582
+ output.warn(
583
+ `${relPath} → ${code}: cached ${repeated.join(', ')} held the same text as other, different source strings ` +
584
+ '(a memorized sentence, not a translation) — removed from the cache and translated again.'
585
+ );
312
586
  }
313
587
  }
314
- Object.assign(translatedFields, fmHits);
315
- tmSegmentHits += Object.keys(fmHits).length;
316
-
317
- if (fmMisses.length > 0) {
318
- output.progress(` [SYNC] ${code} front matter (${pairConfig.method})...`);
319
- const fmResult = await translateBatch(
320
- fmMisses,
321
- fieldsToTranslate,
322
- pairConfig,
323
- { apiKey, cwd, model: pairConfig.model, batchSize: pairConfig.batchSize || 30 },
324
- );
325
- if (fmResult) {
326
- // Content-preservation gate. Front matter (title, description,
327
- // summary) went from the API straight to disk AND into the TM
328
- // with no validation of any kind — which is how a hollowed
329
- // page title was written silently and then cached. Check before
330
- // either. Failing the file is consistent with the null-result
331
- // branch below: nothing written, manifest not advanced, retried.
332
- for (const [field, value] of Object.entries(fmResult)) {
333
- const sourceValue = fieldsToTranslate[field];
334
- if (typeof value !== 'string' || typeof sourceValue !== 'string') continue;
335
- const hollowed = checkContentPreservation(sourceValue, value);
336
- if (hollowed) {
337
- output.raw(' [ERR]');
338
- throw new Error(
339
- `Content sync for ${code}: front matter "${field}" — ${hollowed.reason}.\n` +
340
- ` source: ${JSON.stringify(sourceValue)}\n` +
341
- ` got: ${JSON.stringify(value)}\n` +
342
- ' Nothing was written or cached. If this is a low-coverage target language,\n' +
343
- ' the model has no vocabulary for this string — fix the prompt or the pair.'
344
- );
345
- }
588
+
589
+ // TM entries are keyed on the full method key (method|model|register|
590
+ // coaching). A method/register/coaching change re-translates; a MODEL
591
+ // change reuses the previous model's entries unless
592
+ // --fresh-on-model-change (see "Model carry-over" in lib/tm.js).
593
+ const tmKey = tmMethodKey(pairConfig);
594
+
595
+ // Translate front matter fields — TM first, API for the misses.
596
+ // Each field is cached on its own source text, exactly like a
597
+ // key-value sync key: a title edit re-pays only the title.
598
+ const translatedFields = {};
599
+ // Edits made by hand that this write keeps, and the ones whose
600
+ // source text changed under them (re-translated; wording printed).
601
+ const keptFieldNames = new Set();
602
+ const keptBlocks = new Set(); // source block positions
603
+ const supersededNotes = [];
604
+ if (hasFrontMatter) {
605
+ const fieldsToTranslate = { ...sourceFields };
606
+ if (edits && edits.ownedFields > 0) {
607
+ const { keep, superseded } = matchEditedFields(edits.fields, sourceFields);
608
+ for (const [field, text] of Object.entries(keep)) {
609
+ // A person's value for an unchanged source value: written as
610
+ // is, never sent to the API, never cached as machine output.
611
+ translatedFields[field] = text;
612
+ keptFieldNames.add(field);
613
+ delete fieldsToTranslate[field];
346
614
  }
347
- Object.assign(translatedFields, fmResult);
348
- // Cache only what the API actually returned (the same non-null
349
- // check that gates writing it to the target file).
350
- for (const [field, value] of Object.entries(fmResult)) {
351
- if (typeof value === 'string' && typeof fieldsToTranslate[field] === 'string') {
352
- storeTM(tm, fieldsToTranslate[field], code, tmKey, value);
353
- }
615
+ for (const s of superseded) {
616
+ supersededNotes.push(
617
+ `${targetRel}: front matter "${s.field}" had been edited by hand, but its source value has changed, ` +
618
+ `so it was re-translated. The edited wording, to re-apply if it still fits: ${JSON.stringify(s.text)}`
619
+ );
354
620
  }
355
- output.raw(' [OK]');
356
- } else {
357
- // Front matter translation failed — loud error, skip this file
358
- output.raw(' [ERR]');
359
- throw new Error(
360
- `Content sync for ${code}: front matter translation returned no results.\n` +
361
- ' Check your API key and method configuration.'
362
- );
363
621
  }
364
- }
365
- }
366
622
 
367
- // Translate body — whole-body TM first (a reverted or duplicate body
368
- // is free), then block-level TM + ONE batched API call for the missed
369
- // blocks ('block' mode, the default), or the whole-page prompt
370
- // ('page' mode). A structural failure fails the file whole; a segment
371
- // the model drops TWICE degrades to the honest '[EN] ' fallback for
372
- // just that block (never cached, lock not advanced — re-fires next
373
- // sync as a TM-cheap retry). Mirrors docusaurus-sync.js.
374
- let translatedBody = body;
375
- let bodyUsedFallback = false;
376
- if (body.trim()) {
377
- // Read-time validation: a whole-body entry cached by a gateless
378
- // pipeline must not be re-served once the gate exists (evicts on fail).
379
- const cachedBody = lookupTMValidated(tm, body, code, tmKey,
380
- (src, cached) => !checkContentPreservation(src, cached));
381
- if (cachedBody !== null) {
382
- translatedBody = cachedBody;
383
- tmSegmentHits += 1;
384
- } else {
385
- const segMode = pairConfig.contentSegmentation || 'block';
386
- const { protectedBody, blocks } = protectBlocks(body);
387
- const promptOptions = {
388
- sourceLanguageName: DEFAULT_REGISTERS[sourceLocale]?.name || sourceLocale,
389
- promptContext: pairConfig.promptContext || null,
390
- };
391
-
392
- // Block stores are deferred until the reassembled body passes the
393
- // orphaned-placeholder check — the TM must never hold a value that
394
- // would fail the gate on re-serve.
395
- const pendingBlockStores = [];
396
- let apiCalled = false;
397
-
398
- if (segMode === 'page') {
399
- // Whole-page prompt — the pre-segmentation single-call behavior.
400
- output.progress(` [SYNC] ${code} body (${pairConfig.method})...`);
401
- apiCalled = true;
402
- const prompt = buildContentPrompt(protectedBody, pairConfig, promptOptions);
403
- const bodyResult = await translateRawContent(prompt, {
404
- apiKey,
405
- cwd,
623
+ const { hits: fmHits, misses: fmMisses } = partitionByTM(
624
+ tm, fieldsToTranslate, Object.keys(fieldsToTranslate), code, tmKey
625
+ );
626
+ // Validate cached hits BEFORE serving. A cache is a time machine: an
627
+ // entry stored before the content-preservation gate existed re-serves
628
+ // exactly what that gate now rejects — hollowed titles sat here and
629
+ // were re-served forever, because TM hits skipped every gate. A hit
630
+ // that fails today's gate is evicted and re-billed as a miss.
631
+ for (const [field, cachedValue] of Object.entries(fmHits)) {
632
+ if (contentGateFault(fieldsToTranslate[field], cachedValue, pairConfig)) {
633
+ evictTM(tm, fieldsToTranslate[field], code, tmKey);
634
+ delete fmHits[field];
635
+ fmMisses.push(field);
636
+ }
637
+ }
638
+ Object.assign(translatedFields, fmHits);
639
+ tmSegmentHits += Object.keys(fmHits).length;
640
+ // Then the fallback's cache: fields it translated on an earlier run
641
+ // are reused, not re-sent to the primary (lib/fallback.js ladder).
642
+ const fbCache = serveFieldsFromFallbackCache(tm, fieldsToTranslate, fmMisses, code, pairConfig);
643
+ Object.assign(translatedFields, fbCache.hits);
644
+ tmSegmentHits += Object.keys(fbCache.hits).length;
645
+ if (Object.keys(fbCache.hits).length > 0) notePage(pairKey, relPath);
646
+ const fieldsToSend = fbCache.misses;
647
+ // Filled without a call: the cache, a person's value.
648
+ for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
649
+ // Refused before: held back (the page with it), or the fallback's only.
650
+ const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
651
+ if (heldFields.length > 0) return holdPage(heldFields.map(f => `front matter "${f}"`));
652
+ const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
653
+
654
+ if (fieldsToSend.length > 0) {
655
+ output.progress(` [SYNC] ${code} front matter (${pairConfig.method})...`);
656
+ // The pair's method, then its fallback (if any) for every field
657
+ // it returned nothing for or hollowed. Content-preservation gate
658
+ // on every value: front matter (title, description, summary)
659
+ // once went from the API straight to disk AND into the TM with no
660
+ // validation — which is how a hollowed page title was written
661
+ // silently and then cached. Nothing is cached until no field is
662
+ // left failing; a failing field fails the file (nothing written,
663
+ // manifest not advanced, retried).
664
+ const fm = await translateFieldsWithFallback({
665
+ fields: fieldsToTranslate,
666
+ misses: fieldsToSend,
406
667
  pairConfig,
668
+ budget: fallbackBudget,
669
+ sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
670
+ label: `content:${relPath} front matter`,
671
+ translate: (keys, cfg) => translateBatch(
672
+ keys, fieldsToTranslate, cfg,
673
+ { apiKey, cwd, model: cfg.model, batchSize: cfg.batchSize || 30 },
674
+ ),
675
+ fallbackOnly: fallbackOnlyFields,
407
676
  });
408
- if (!bodyResult) {
409
- // Body translation returned null — loud error
677
+ tallyFallback(pairKey, fm.report, relPath);
678
+ // Refused fields are remembered (held back from the method that
679
+ // refused them), whatever happens to the page below.
680
+ for (const [field, methods] of Object.entries(fm.refusedBy || {})) {
681
+ refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
682
+ }
683
+ if (fm.hollowed.length > 0) {
684
+ const h = fm.hollowed[0];
685
+ output.raw(' [ERR]');
686
+ // A memorized sentence: asking the same model again returns it
687
+ // again — say so, and what does help (Round 6, school persona).
688
+ const advice = h.sharedOutput
689
+ ? (pairConfig.fallback
690
+ ? ` ${pairConfig.method} can only answer this string with that sentence, and the fallback did not translate it either.\n`
691
+ + ` Write "${h.field}" in ${targetRel} by hand (a hand-written field is kept), or try another fallback.`
692
+ : ` ${pairConfig.method} can only answer this string with that sentence — asking it again returns the same.\n`
693
+ + ' Add a "fallback" method to the pair (it is asked for what the primary\'s answer is refused for),\n'
694
+ + ` or write "${h.field}" in ${targetRel} by hand (a hand-written field is kept).`)
695
+ : ' If this is a low-coverage target language,\n'
696
+ + ' the model has no vocabulary for this string — fix the prompt or the pair.';
697
+ const err = new Error(
698
+ `Content sync for ${code}: front matter "${h.field}" — ${h.reason}.\n` +
699
+ ` source: ${JSON.stringify(fieldsToTranslate[h.field])}\n` +
700
+ ` got: ${JSON.stringify(h.value)}\n` +
701
+ (h.fallbackReason ? ` the fallback (${pairConfig.fallback.method}) failed it too: ${h.fallbackReason}\n` : '') +
702
+ ' Nothing was written or cached.\n' + advice + '\n' +
703
+ ` ${describeNewHold({ file: relPath, pairKey, pairConfig, count: fm.hollowed.length })}`
704
+ );
705
+ err.heldNext = true;
706
+ throw err;
707
+ }
708
+ if (fm.noResults) {
709
+ // Front matter translation failed — loud error, skip this file
410
710
  output.raw(' [ERR]');
411
711
  throw new Error(
412
- `Content sync body for ${code}: translation returned no results.\n` +
712
+ `Content sync for ${code}: front matter translation returned no results`
713
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
413
714
  ' Check your API key and method configuration.'
414
715
  );
415
716
  }
416
- translatedBody = restoreBlocks(bodyResult, blocks);
417
- } else {
418
- // Block mode: segment the PROTECTED body (placeholders are
419
- // single tokens, so they can never be split), serve blocks from
420
- // the TM, and batch the misses into one API call.
421
- const segments = splitBlocks(protectedBody);
422
-
423
- // TM keys use each block's RESTORED source text: placeholder
424
- // numbering is positional per file, so the protected text of an
425
- // identical paragraph differs across files/edits, while the
426
- // restored text is stable and self-contained. Cached values are
427
- // likewise restored — they must never carry another file's
428
- // placeholder ids.
429
- const rendered = segments.map(seg => ({
430
- seg,
431
- source: restoreBlocks(seg.text, blocks),
432
- out: null,
433
- }));
434
-
435
- const missed = [];
436
- for (const r of rendered) {
437
- if (r.seg.type !== 'translatable') {
438
- // Separators + passthrough blocks: copied verbatim, never billed.
439
- r.out = r.source;
440
- continue;
441
- }
442
- const cached = lookupTMValidated(tm, r.source, code, tmKey,
443
- (src, c) => !checkContentPreservation(src, c));
444
- if (cached !== null) {
445
- r.out = cached;
446
- tmSegmentHits += 1;
447
- } else {
448
- missed.push(r);
449
- }
717
+ // Cache only gate-passing values, each under the TM key of the
718
+ // method that produced it.
719
+ for (const st of fm.stores) storeTM(tm, st.text, code, st.tmKey, st.value);
720
+ // A field only the fallback may be asked for that it did not
721
+ // translate (skipped by --max-cost, or no answer): still refused by
722
+ // the pair's method, so the page is held, as with no fallback.
723
+ const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
724
+ if (unfilled.length > 0) {
725
+ output.raw(' [HELD]');
726
+ return holdPage(unfilled.map(f => `front matter "${f}"`));
450
727
  }
728
+ Object.assign(translatedFields, fm.translated);
729
+ for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
730
+ output.raw(' [OK]');
731
+ }
732
+ }
733
+
734
+ // Translate body — whole-body TM first (a reverted or duplicate body
735
+ // is free), then block-level TM + ONE batched API call for the missed
736
+ // blocks ('block' mode, the default), or the whole-page prompt
737
+ // ('page' mode). A structural failure fails the file whole; a segment
738
+ // the model drops TWICE degrades to the honest '[EN] ' fallback for
739
+ // just that block (never cached, lock not advanced — re-fires next
740
+ // sync as a TM-cheap retry). Mirrors docusaurus-sync.js.
741
+ let translatedBody = body;
742
+ let bodyUsedFallback = false;
743
+ // Paragraphs a person edited whose source paragraph is unchanged:
744
+ // kept word for word (block mode only — assessExistingTarget never
745
+ // lets a 'page'-mode file with edited paragraphs get this far). The
746
+ // rest are reported, even when the source body is now empty.
747
+ let keepPlan = null;
748
+ if (edits && edits.blocks && edits.ownedBlocks > 0) {
749
+ keepPlan = matchEditedBlocks(edits.blocks, body.trim() ? translatableBlockSources(body) : []);
750
+ for (const s of keepPlan.superseded) {
751
+ supersededNotes.push(
752
+ `${targetRel}: paragraph ${s.paragraph} had been edited by hand, but the source paragraph it ` +
753
+ 'translates has changed, so it was re-translated. The edited wording, to re-apply if it still fits:\n' +
754
+ s.text.split('\n').map(line => ` ${line}`).join('\n')
755
+ );
756
+ }
757
+ }
758
+ if (body.trim()) {
759
+ // Read-time validation: a whole-body entry cached by a gateless
760
+ // pipeline must not be re-served once the gate exists (evicts on fail).
761
+ // The pair's cache, then its fallback's (lib/fallback.js ladder).
762
+ // Not when a person's paragraphs are kept: a whole-body hit would
763
+ // put the machine wording back over them.
764
+ const cachedBody = keepPlan && keepPlan.keep.size > 0
765
+ ? null
766
+ : lookupContentTM(tm, body, code, pairConfig);
767
+ if (cachedBody !== null) {
768
+ translatedBody = cachedBody.text;
769
+ tmSegmentHits += 1;
770
+ for (const src of refusal.blockSources) refusal.filled.add(blockUnit(src));
771
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
772
+ } else {
773
+ const { protectedBody, blocks } = protectBlocks(body);
774
+ const promptOptions = {
775
+ sourceLanguageName: DEFAULT_REGISTERS[sourceLocale]?.name || sourceLocale,
776
+ promptContext: pairConfig.promptContext || null,
777
+ protectedTerms: pairConfig.protectedTerms || [],
778
+ };
779
+
780
+ // Block stores are deferred until the reassembled body passes the
781
+ // orphaned-placeholder check — the TM must never hold a value that
782
+ // would fail the gate on re-serve. Each carries the TM key of the
783
+ // method that produced it (the pair's, or its fallback's).
784
+ const pendingBlockStores = [];
785
+ let apiCalled = false;
786
+ // The whole body is cached under ONE method's key, so only when
787
+ // one method produced all of it (a body mixing the primary's and
788
+ // the fallback's blocks is cached block by block only).
789
+ let wholeBodyKey = tmKey;
790
+ let mixedProducers = false;
791
+ // Page mode: whose whole page the checks below refuse (the lane
792
+ // remembers it — lib/content-refusals.js "page").
793
+ let pageRefusedBy = null;
451
794
 
452
- if (missed.length > 0) {
453
- output.progress(` [SYNC] ${code} body (${missed.length} block(s), ${pairConfig.method})...`);
795
+ if (segMode === 'page') {
796
+ // Refused before by this method (and the cache does not hold
797
+ // it): not sent again (lib/content-refusals.js).
798
+ const pageHold = holds.page(body);
799
+ if (pageHold === 'held') return holdPage([PAGE_NAME]);
800
+ // Whole-page prompt — the pre-segmentation single-call behavior.
801
+ output.progress(` [SYNC] ${code} body (${pairConfig.method})...`);
454
802
  apiCalled = true;
455
- // Terminology context for the block-batch prompt: the page's
456
- // title (front matter first, else the first H1 in the body).
457
- const pageTitle =
458
- (hasFrontMatter && typeof frontMatter.title === 'string' ? frontMatter.title : null)
459
- || (body.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? null);
460
- // Self-repair ladder (translateBlockBatchResilient): full
461
- // batch → one missing-segments-only retry → honest
462
- // '[EN] '-prefixed source for anything still missing. A
463
- // duplicate/unknown marker (untrustworthy mapping) or an
464
- // empty first response still fails the file whole.
465
- let batchOutcome;
466
- try {
467
- batchOutcome = await translateBlockBatchResilient({
468
- texts: missed.map(r => r.seg.text),
469
- buildPrompt: (texts) => buildBlockBatchPrompt(
470
- texts, pairConfig, { ...promptOptions, pageTitle }),
471
- callModel: (prompt) => translateRawContent(prompt, {
472
- apiKey,
473
- cwd,
474
- pairConfig,
475
- }),
476
- fallbackPrefix,
803
+ const runPage = (cfg) => translateRawContent(buildContentPrompt(protectedBody, cfg, promptOptions), {
804
+ apiKey,
805
+ cwd,
806
+ pairConfig: cfg,
807
+ });
808
+ let bodyResult;
809
+ if (pairConfig.fallback) {
810
+ // The fallback translates the page when the primary returns
811
+ // nothing, damages a placeholder or hollows it. When both
812
+ // fail, the primary's page goes to the checks below, which
813
+ // fail the file exactly as without a fallback.
814
+ // Refused before by the pair's method: only the fallback is asked.
815
+ const page = await translatePageWithFallback({
816
+ body, blocks, pairConfig, runPage, budget: fallbackBudget, fallbackOnly: pageHold === 'fallback-only',
477
817
  });
478
- } catch (batchErr) {
818
+ tallyFallback(pairKey, page.report, relPath);
819
+ if (page.refusedBy.length > 0) {
820
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: page.refusedBy });
821
+ }
822
+ if (pageHold === 'fallback-only' && page.body === null) {
823
+ // The fallback did not translate it: the pair's method
824
+ // refused it before, so it is not asked — refused now by
825
+ // the fallback too, or held (no answer, or skipped by
826
+ // --max-cost).
827
+ if (page.refusedBy.length === 0) return holdPage([PAGE_NAME]);
828
+ output.raw(' [ERR]');
829
+ const err = new Error(
830
+ `Content sync body for ${code}: the fallback (${pairConfig.fallback.method}) failed the whole-page checks too.\n` +
831
+ ` Nothing was written or cached. ${describeNewHold({ file: relPath, pairKey, pairConfig, count: 1 })}`
832
+ );
833
+ err.heldNext = true;
834
+ throw err;
835
+ }
836
+ bodyResult = page.body ?? page.primaryBody;
837
+ if (page.body !== null) wholeBodyKey = page.tmKey;
838
+ else if (page.refusedBy.length > 0) pageRefusedBy = page.refusedBy;
839
+ } else {
840
+ const raw = await runPage(pairConfig);
841
+ bodyResult = raw ? restoreBlocks(raw, blocks) : null;
842
+ if (bodyResult) pageRefusedBy = [tmKey];
843
+ }
844
+ if (!bodyResult) {
845
+ // Body translation returned null — loud error
479
846
  output.raw(' [ERR]');
480
- throw batchErr;
847
+ throw new Error(
848
+ `Content sync body for ${code}: translation returned no results`
849
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
850
+ ' Check your API key and method configuration.'
851
+ );
852
+ }
853
+ translatedBody = bodyResult;
854
+ } else {
855
+ // Block mode: segment the PROTECTED body (placeholders are
856
+ // single tokens, so they can never be split), serve blocks from
857
+ // the TM, and batch the misses into one API call.
858
+ const segments = splitBlocks(protectedBody);
859
+
860
+ // TM keys use each block's RESTORED source text: placeholder
861
+ // numbering is positional per file, so the protected text of an
862
+ // identical paragraph differs across files/edits, while the
863
+ // restored text is stable and self-contained. Cached values are
864
+ // likewise restored — they must never carry another file's
865
+ // placeholder ids.
866
+ const rendered = segments.map(seg => ({
867
+ seg,
868
+ source: restoreBlocks(seg.text, blocks),
869
+ out: null,
870
+ }));
871
+
872
+ const missed = [];
873
+ const heldBlocks = []; // refused before: not sent, '[EN] ' kept
874
+ let position = 0; // index among translatable blocks (= translatableBlockSources order)
875
+ for (const r of rendered) {
876
+ if (r.seg.type !== 'translatable') {
877
+ // Separators + passthrough blocks: copied verbatim, never billed.
878
+ r.out = r.source;
879
+ continue;
880
+ }
881
+ const j = position++;
882
+ // Its name in the repeat check (content:<file>#<n>, as verify names it).
883
+ r.pos = j;
884
+ if (keepPlan && keepPlan.keep.has(j)) {
885
+ // A person's paragraph: written as is, not billed, not cached.
886
+ r.out = keepPlan.keep.get(j);
887
+ keptBlocks.add(j);
888
+ refusal.filled.add(blockUnit(r.source));
889
+ continue;
890
+ }
891
+ const cached = lookupContentTM(tm, r.source, code, pairConfig);
892
+ if (cached !== null) {
893
+ r.out = cached.text;
894
+ tmSegmentHits += 1;
895
+ if (cached.fromFallback) { mixedProducers = true; notePage(pairKey, relPath); }
896
+ refusal.filled.add(blockUnit(r.source));
897
+ continue;
898
+ }
899
+ const hold = holds.block(r.source);
900
+ if (hold === 'held') {
901
+ // Refused before by this method (and its fallback): the
902
+ // honest last resort stays, nothing is sent or billed.
903
+ r.out = restoreBlocks(fallbackPrefix + r.seg.text, blocks);
904
+ heldBlocks.push(r);
905
+ continue;
906
+ }
907
+ r.fallbackOnly = hold === 'fallback-only';
908
+ missed.push(r);
481
909
  }
482
- const { blocks: translatedBlocks, fellBack } = batchOutcome;
483
- const fellBackSet = new Set(fellBack);
484
- if (fellBack.length > 0) {
910
+ if (heldBlocks.length > 0) {
485
911
  bodyUsedFallback = true;
486
- output.raw(' [FALLBACK]');
487
- output.warn(
488
- `Content sync body for ${code}: ${fellBack.length} of ${missed.length} ` +
489
- `block(s) missing from the model response after a retry — written as ` +
490
- `'${fallbackPrefix}'-prefixed source. Not cached, lock not advanced: ` +
491
- `the next sync retries just those block(s).`
492
- );
912
+ const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
913
+ refusal.held.push(...names);
914
+ result.heldBack = (result.heldBack || 0) + names.length;
915
+ output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true, fallbackPrefix }));
493
916
  }
494
- missed.forEach((r, i) => {
495
- r.out = restoreBlocks(translatedBlocks[i], blocks);
917
+
918
+ if (missed.length > 0) {
919
+ output.progress(` [SYNC] ${code} body (${missed.length} block(s), ${pairConfig.method})...`);
920
+ apiCalled = true;
921
+ // Terminology context for the block-batch prompt: the page's
922
+ // title (front matter first, else the first H1 in the body).
923
+ const pageTitle =
924
+ (hasFrontMatter && typeof frontMatter.title === 'string' ? frontMatter.title : null)
925
+ || (body.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? null);
926
+ // Self-repair ladder (translateBlockBatchResilient): full
927
+ // batch → one missing-segments-only retry → (with a fallback
928
+ // method: one batch through it for every block the primary
929
+ // dropped or damaged — lib/fallback.js) → honest
930
+ // '[EN] '-prefixed source for anything still missing. A
931
+ // duplicate/unknown marker (untrustworthy mapping) or an
932
+ // empty first response still fails the file whole when no
933
+ // fallback rescues it.
934
+ let batchOutcome;
935
+ try {
936
+ batchOutcome = await translateBlocksWithFallback({
937
+ missed,
938
+ blocks,
939
+ pairConfig,
940
+ fallbackPrefix,
941
+ budget: fallbackBudget,
942
+ sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
943
+ label: `content:${relPath}`,
944
+ runBatch: (texts, cfg) => translateBlockBatchResilient({
945
+ texts,
946
+ buildPrompt: (t) => buildBlockBatchPrompt(t, cfg, { ...promptOptions, pageTitle }),
947
+ callModel: (prompt) => translateRawContent(prompt, {
948
+ apiKey,
949
+ cwd,
950
+ pairConfig: cfg,
951
+ }),
952
+ fallbackPrefix,
953
+ }),
954
+ fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
955
+ });
956
+ } catch (batchErr) {
957
+ output.raw(' [ERR]');
958
+ throw batchErr;
959
+ }
960
+ tallyFallback(pairKey, batchOutcome.report, relPath);
961
+ if (batchOutcome.fromFallback > 0) mixedProducers = true;
962
+ const { fellBack } = batchOutcome;
963
+ // Refused blocks are remembered; translated ones drop a record.
964
+ const fellSet = new Set(fellBack);
965
+ const newlyHeld = [];
966
+ missed.forEach((r, i) => {
967
+ if (!fellSet.has(i)) { refusal.filled.add(blockUnit(r.source)); return; }
968
+ const methods = batchOutcome.refusedBy?.[i] || [];
969
+ if (methods.length > 0) {
970
+ refusal.refused.push({ unit: blockUnit(r.source), source: r.source, methods });
971
+ newlyHeld.push(i);
972
+ }
973
+ });
974
+ if ((batchOutcome.sharedOutput || []).length > 0) {
975
+ output.warn(
976
+ `Content sync body for ${code}: ${batchOutcome.sharedOutput.length} block(s) of ${relPath} came back as the same text ` +
977
+ 'the model gave for other, different source strings (a memorized sentence, not a translation) — refused' +
978
+ (pairConfig.fallback
979
+ ? `; sent to the fallback (${pairConfig.fallback.method}).`
980
+ : `. ${pairConfig.method} can only answer them with that sentence: add a "fallback" method to the pair, ` +
981
+ 'or write those paragraphs by hand (a hand-written paragraph is kept).')
982
+ );
983
+ }
984
+ // Blocks the quality gate refused — the key-value gate's checks
985
+ // (length inflation, echo, truncation, script), per block.
986
+ const refusedByGate = batchOutcome.refused || [];
987
+ if (refusedByGate.length > 0) {
988
+ output.warn(
989
+ `Content sync body for ${code}: ${refusedByGate.length} block(s) of ${relPath} refused by the quality gate — `
990
+ + refusedByGate.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
991
+ + `${refusedByGate.length > 3 ? '; …' : ''}`
992
+ + (pairConfig.fallback
993
+ ? '.'
994
+ : `. Add a "fallback" method to the pair (it is asked for what ${pairConfig.method}'s answer is refused for), `
995
+ + 'or write those paragraphs by hand (a hand-written paragraph is kept).')
996
+ );
997
+ }
998
+ if (fellBack.length > 0) {
999
+ bodyUsedFallback = true;
1000
+ output.raw(' [EN]');
1001
+ // Refused as a memorized sentence (said just above) is not
1002
+ // "missing from the response".
1003
+ const refusedShared = (batchOutcome.sharedOutput || []).length + refusedByGate.length;
1004
+ const why = batchOutcome.report?.attempted > 0
1005
+ ? `neither the primary (${pairConfig.method}) nor its fallback (${pairConfig.fallback.method}) translated safely`
1006
+ : refusedShared >= fellBack.length
1007
+ ? 'refused (above)'
1008
+ : refusedShared > 0
1009
+ ? `refused (${refusedShared}, above) or missing from the model response after a retry`
1010
+ : 'missing from the model response after a retry';
1011
+ output.warn(
1012
+ `Content sync body for ${code}: ${fellBack.length} of ${missed.length} ` +
1013
+ `block(s) ${why} — written as ` +
1014
+ `'${fallbackPrefix}'-prefixed source. ` +
1015
+ describeFallenBack({
1016
+ file: relPath, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: batchOutcome.refusedBy,
1017
+ })
1018
+ );
1019
+ }
496
1020
  // Fallen-back segments are never TM-cached — an error cached
497
1021
  // is an error forever (they re-bill on the next sync).
498
- if (!fellBackSet.has(i)) {
499
- pendingBlockStores.push({ source: r.source, translation: r.out });
500
- }
501
- });
1022
+ missed.forEach((r, i) => { r.out = batchOutcome.outs[i]; });
1023
+ pendingBlockStores.push(...batchOutcome.stores);
1024
+ }
1025
+
1026
+ // Reassemble in order with the source's exact separators.
1027
+ translatedBody = rendered.map(r => r.out).join('');
502
1028
  }
503
1029
 
504
- // Reassemble in order with the source's exact separators.
505
- translatedBody = rendered.map(r => r.out).join('');
506
- }
1030
+ // A page translated whole that fails the checks below is
1031
+ // remembered as refused: the next plain sync does not send it to
1032
+ // the same method again (lib/content-refusals.js "page").
1033
+ const refusePage = (err) => {
1034
+ if (!pageRefusedBy) return err;
1035
+ if (!refusal.refused.some(r => r.unit === PAGE_UNIT)) {
1036
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: pageRefusedBy });
1037
+ }
1038
+ err.message += `\n ${describeNewHold({ file: relPath, pairKey, pairConfig, count: 1 })}`;
1039
+ err.heldNext = true;
1040
+ return err;
1041
+ };
507
1042
 
508
- // Orphaned-placeholder check on the REASSEMBLED body — the same
509
- // gate for both modes.
510
- if (hasOrphanedPlaceholders(translatedBody)) {
511
- // Placeholder corruption — loud error, skip this file
512
- output.error('PLACEHOLDER CORRUPTION');
513
- throw new Error(
514
- `Content sync body for ${code}: placeholder corruption detected.\n` +
515
- ' Code blocks were corrupted during translation. Retry or report this issue.'
516
- );
517
- }
1043
+ // Orphaned-placeholder check on the REASSEMBLED body — the same
1044
+ // gate for both modes.
1045
+ if (hasOrphanedPlaceholders(translatedBody)) {
1046
+ // Placeholder corruption — loud error, skip this file
1047
+ output.error('PLACEHOLDER CORRUPTION');
1048
+ throw refusePage(new Error(
1049
+ `Content sync body for ${code}: placeholder corruption detected.\n` +
1050
+ ' Code blocks were corrupted during translation. Retry or report this issue.'
1051
+ ));
1052
+ }
518
1053
 
519
- // Content-preservation check on the REASSEMBLED body. This lane
520
- // never ran the quality gate at all — translateBatch/
521
- // translateRawContent output went straight to disk AND into the TM
522
- // — so a body hollowed of its letters was written silently and then
523
- // re-served from cache forever. Throwing here matches the
524
- // placeholder gate above: the file is skipped, its manifest entry is
525
- // not advanced, and nothing is cached, so the next sync retries.
526
- const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
527
- if (bodyHollowed) {
528
- output.error('CONTENT LOSS');
529
- throw new Error(
530
- `Content sync body for ${code}: ${bodyHollowed.reason}.\n` +
531
- ' Nothing was written or cached. If this is a low-coverage target language,\n' +
532
- ' the model has no vocabulary for this text — fix the prompt or the pair, not the gate.'
533
- );
534
- }
1054
+ // Content-preservation check on the REASSEMBLED body. This lane
1055
+ // never ran the quality gate at all — translateBatch/
1056
+ // translateRawContent output went straight to disk AND into the TM
1057
+ // — so a body hollowed of its letters was written silently and then
1058
+ // re-served from cache forever. Throwing here matches the
1059
+ // placeholder gate above: the file is skipped, its manifest entry is
1060
+ // not advanced, and nothing is cached, so the next sync retries.
1061
+ const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
1062
+ if (bodyHollowed) {
1063
+ output.error('CONTENT LOSS');
1064
+ throw refusePage(new Error(
1065
+ `Content sync body for ${code}: ${bodyHollowed.reason}.\n` +
1066
+ ' Nothing was written or cached. If this is a low-coverage target language,\n' +
1067
+ ' the model has no vocabulary for this text — fix the prompt or the pair, not the gate.'
1068
+ ));
1069
+ }
1070
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
535
1071
 
536
- // Store per-block AND whole-body entries only AFTER the corruption
537
- // check — the whole-body entry makes reverts/duplicates free. A
538
- // fallback body is NEVER stored whole: it contains untranslated
539
- // '[EN] ' text, and an error cached is an error forever.
540
- for (const s of pendingBlockStores) {
541
- storeTM(tm, s.source, code, tmKey, s.translation);
542
- }
543
- if (!bodyUsedFallback) {
544
- storeTM(tm, body, code, tmKey, translatedBody);
1072
+ // Store per-block AND whole-body entries only AFTER the corruption
1073
+ // check — the whole-body entry makes reverts/duplicates free. A
1074
+ // fallback body is NEVER stored whole: it contains untranslated
1075
+ // '[EN] ' text, and an error cached is an error forever.
1076
+ for (const s of pendingBlockStores) {
1077
+ storeTM(tm, s.source, code, s.tmKey, s.translation);
1078
+ }
1079
+ // Never a whole-body entry holding a person's paragraphs: the TM
1080
+ // holds machine output only, under the method that produced it.
1081
+ if (!bodyUsedFallback && !mixedProducers && keptBlocks.size === 0) {
1082
+ storeTM(tm, body, code, wholeBodyKey, translatedBody);
1083
+ }
1084
+ if (apiCalled && !bodyUsedFallback) output.raw(' [OK]');
545
1085
  }
546
- if (apiCalled && !bodyUsedFallback) output.raw(' [OK]');
547
1086
  }
548
- }
549
1087
 
550
- // Reassemble and write
551
- const assembled = reassembleContentFile({
552
- rawFrontMatter, translatedFields, translatedBody,
553
- hasFrontMatter, frontMatterFormat,
554
- });
555
- fs.mkdirSync(path.dirname(targetPath), { recursive: true });
556
- fs.writeFileSync(targetPath, assembled, 'utf-8');
557
-
558
- result.translated = true;
559
- // A fallback body keeps its OLD manifest entry so the file re-fires
560
- // next sync — every good block is a TM hit, only the fallen-back
561
- // segment re-bills. Self-healing at bounded cost.
562
- if (!bodyUsedFallback) {
563
- updatedManifest[manifestKey] = currentSourceHash;
564
- }
1088
+ // Reassemble and write
1089
+ const assembled = reassembleContentFile({
1090
+ rawFrontMatter, translatedFields, translatedBody,
1091
+ hasFrontMatter, frontMatterFormat,
1092
+ });
1093
+ fs.mkdirSync(path.dirname(targetPath), { recursive: true });
1094
+ fs.writeFileSync(targetPath, assembled, 'utf-8');
1095
+
1096
+ // Record what is now on disk, block by block, with the paragraphs
1097
+ // and fields that are a person's marked — so the next sync can tell
1098
+ // a later edit from this write, and keeps these ones again.
1099
+ updatedManifest[recordKey] = buildWrittenRecord({
1100
+ sourceBody: body, sourceFields, written: assembled,
1101
+ ownedBlocks: keptBlocks, ownedFields: keptFieldNames,
1102
+ });
1103
+
1104
+ result.translated = true;
1105
+ const keptCount = keptBlocks.size + keptFieldNames.size;
1106
+ output.event('file', {
1107
+ lane: 'content', file: relPath, locale: code,
1108
+ status: bodyUsedFallback ? 'fallback' : 'translated',
1109
+ ...(keptCount > 0 && { keptEdits: keptCount }),
1110
+ });
1111
+ if (keptCount > 0) {
1112
+ result.keptEdits = true;
1113
+ output.info(
1114
+ `${code} — kept the edits made by hand to ${describeEdits({ paragraphs: keptBlocks.size, fields: [...keptFieldNames] })} ` +
1115
+ `of ${targetRel}. To re-translate them: champollion sync --redo files:${relPath}`
1116
+ );
1117
+ }
1118
+ for (const note of supersededNotes) output.warn(note);
1119
+ // A fallback body keeps its OLD manifest entry so the file re-fires
1120
+ // next sync — every good block is a TM hit, only the fallen-back
1121
+ // segment re-bills. Self-healing at bounded cost.
1122
+ if (!bodyUsedFallback) {
1123
+ updatedManifest[manifestKey] = currentSourceHash;
1124
+ }
1125
+
1126
+ return result;
1127
+ };
1128
+
1129
+ // One (file × locale) failing must not stop the others — or discard
1130
+ // their work: the run carries on, and the lock + TM are saved below for
1131
+ // everything that succeeded (dogfood 2026-08-28, finding 5). Only a
1132
+ // FATAL error (a method that cannot run at all) stops the run.
1133
+ const perPairResults = await pMap(pairEntries, async (entry) => {
1134
+ try {
1135
+ return await translateOne(entry);
1136
+ } catch (err) {
1137
+ if (err.fatal) throw err;
1138
+ const code = entry[1].target;
1139
+ output.error(`${relPath} → ${code} — ${err.message}`);
1140
+ // heldNext: refused by the quality gate — held back from now on,
1141
+ // not retried by the next plain sync (lib/content-refusals.js).
1142
+ failedItems.push({ file: relPath, locale: code, error: err.message, ...(err.heldNext && { heldNext: true }) });
1143
+ output.event('file', { lane: 'content', file: relPath, locale: code, status: 'failed', error: err.message });
1144
+ return { translated: false, fallback: false, skipped: false, retranslated: false };
1145
+ }
1146
+ }, { concurrency });
565
1147
 
566
- return result;
567
- }, { concurrency });
1148
+ // Aggregate per-pair results into totals
1149
+ for (const r of perPairResults) {
1150
+ if (r.translated) translated++;
1151
+ if (r.skipped) skipped++;
1152
+ if (r.retranslated) retranslated++;
1153
+ if (r.keptEdits) keptEditFiles++;
1154
+ if (r.heldBack) heldBackSeen += r.heldBack;
1155
+ }
1156
+ }
1157
+ } catch (err) {
1158
+ // A fatal error still lets the saves below run: files finished before
1159
+ // it paid for their translations, and their lock entries + TM entries
1160
+ // are what make the retry cheap.
1161
+ fatalError = err;
1162
+ }
568
1163
 
569
- // Aggregate per-pair results into totals
570
- for (const r of perPairResults) {
571
- if (r.translated) translated++;
572
- if (r.skipped) skipped++;
573
- if (r.retranslated) retranslated++;
1164
+ // What each translation's refusals are now (lib/content-refusals.js):
1165
+ // earlier ones that still apply, plus this run's — held back from the
1166
+ // method that refused them until a redo names the page, the source text
1167
+ // changes, or the model/method does.
1168
+ const heldBackItems = [];
1169
+ let heldBack = 0;
1170
+ let refusedNow = 0;
1171
+ for (const [manifestKey, st] of refusalStates) {
1172
+ storeRefusals(updatedManifest, manifestKey, nextRefusals(st.prior, st));
1173
+ if (st.held.length > 0) {
1174
+ heldBack += st.held.length;
1175
+ heldBackItems.push({ file: st.file, locale: st.locale, units: st.held, ...(st.pageHeld && { pageNotWritten: true }) });
574
1176
  }
1177
+ refusedNow += new Set(st.refused.map(r => r.unit)).size;
575
1178
  }
576
1179
 
577
1180
  // Write updated content manifest (skip in dry-run)
@@ -585,23 +1188,85 @@ async function runContentSync(options) {
585
1188
 
586
1189
  // Persist TM if it was mutated during this content sync (same rationale as
587
1190
  // the key-value path in sync.js: dirty tracking, not size comparison).
588
- // Skip when --no-tm is active — nothing was cached, nothing to save.
589
- if (!dryRun && !noTM && isTMDirty(tm)) {
590
- const tmFinalSize = tmSize(tm);
591
- const delta = tmFinalSize - tmInitialSize;
1191
+ // Under --fresh/--no-tm too: what was paid for is cached.
1192
+ if (!dryRun && isTMDirty(tm)) {
592
1193
  saveTM(cwd, tm);
593
- output.info(`[TM] Saved ${tmFinalSize} entries (${delta >= 0 ? '+' + delta : delta} this sync)`);
1194
+ output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
594
1195
  }
595
1196
 
596
1197
  const totalCreated = translated;
597
- if (totalCreated > 0 || skipped > 0 || retranslated > 0) {
1198
+ if (totalCreated > 0 || skipped > 0 || retranslated > 0 || heldItems.length > 0 || (dryRun && heldBackSeen > 0)) {
598
1199
  const retranslateNote = retranslated > 0 ? ` (${retranslated} re-translated)` : '';
1200
+ const keptNote = keptEditFiles > 0 ? `, ${keptEditFiles} keeping edits made by hand` : '';
1201
+ const heldNote = heldItems.length > 0 ? `, ${heldItems.length} left as is (edited by hand — listed below)` : '';
599
1202
  if (dryRun) {
600
- output.info(`Would have created ${totalCreated} content file(s)${retranslateNote}, ${skipped} unchanged.`);
1203
+ const heldBackNote = heldBackSeen > 0 ? `; ${heldBackSeen} block(s)/field(s) refused before would be held back (not sent)` : '';
1204
+ output.info(`Would have created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}${heldBackNote}.`);
601
1205
  } else {
602
- output.ok(`Created ${totalCreated} content file(s)${retranslateNote}, ${skipped} unchanged.`);
1206
+ output.ok(`Created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}.`);
1207
+ }
1208
+ }
1209
+ if (heldBack > 0) {
1210
+ // Not sent, not billed — but not translated either (exit 2, as for a
1211
+ // held-back key). Each page was named above with its own repair.
1212
+ output.warn(
1213
+ `${heldBack} content block(s)/field(s) held back in ${heldBackItems.length} translation(s) — the quality gate refused the method's `
1214
+ + 'translation of their current text before; not sent, not billed (listed above). Ask again for all of them: '
1215
+ + '`champollion sync --redo content`; for one page: `champollion sync --redo files:<page>`.'
1216
+ );
1217
+ }
1218
+ if (heldItems.length > 0) {
1219
+ // Not a failure (nothing was lost, nothing billed) — but out of date
1220
+ // with its source, so it is listed on every run until resolved.
1221
+ output.raw('');
1222
+ output.raw(` Translations left as is because they were edited by hand and their source changed (${heldItems.length}):`);
1223
+ for (const h of heldItems) output.raw(` ${h.target} (source: ${h.file})`);
1224
+ output.raw(' Update each by hand (the next sync takes it as current), or replace it: champollion sync --redo files:<source>');
1225
+ }
1226
+
1227
+ if (!dryRun) {
1228
+ for (const [pairKey, tally] of fallbackTallies) {
1229
+ printFallbackReport(pairKey, pairs.get(pairKey).method, tally, { unit: 'content segment(s)', producedLabel: 'pages' });
1230
+ warnFallbackMajority(pairKey, pairs.get(pairKey).method, tally, { unit: 'content segment(s)' });
603
1231
  }
604
1232
  }
1233
+
1234
+ if (fatalError) throw fatalError;
1235
+ if (failedItems.length > 0) {
1236
+ output.raw('');
1237
+ output.raw(` Failed content translations (${failedItems.length}) — not recorded as done; the next sync retries them`
1238
+ + `${failedItems.some(f => f.heldNext) ? ', except what the quality gate refused (held back — `--redo files:<page>` asks again)' : ''}:`);
1239
+ for (const f of failedItems) output.raw(` ${f.file} → ${f.locale}${f.heldNext ? ' (refused by the quality gate: held back)' : ''}`);
1240
+ output.error(
1241
+ `${failedItems.length} content translation(s) failed (listed above). ` +
1242
+ 'Completed files and their translations are saved; re-run sync to retry.'
1243
+ );
1244
+ }
1245
+
1246
+ // Returned, not thrown: a partly failed run still did real work, and the
1247
+ // caller folds this into the run summary and the exit code (2 = partial).
1248
+ return {
1249
+ translated,
1250
+ skipped,
1251
+ retranslated,
1252
+ failed: failedItems.length,
1253
+ failedItems: failedItems.map(({ heldNext, ...f }) => ({ ...f, state: heldNext ? 'held-back' : 'will-retry' })),
1254
+ // Blocks/fields the quality gate refused before, not sent this run (the
1255
+ // '[EN] ' text stays; a page with a held front-matter field is not
1256
+ // written), and blocks/fields it refused this run (held from the next).
1257
+ heldBack,
1258
+ heldBackItems,
1259
+ refused: refusedNow,
1260
+ // Translations written with a person's edits kept, and translations
1261
+ // left as is because their edits could not be merged with a source change.
1262
+ keptEdits: keptEditFiles,
1263
+ held: heldItems.length,
1264
+ heldItems,
1265
+ // What each pair's fallback method did (pairs with a fallback, real runs).
1266
+ ...(!dryRun && fallbackTallies.size > 0 && {
1267
+ fallback: [...fallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
1268
+ }),
1269
+ };
605
1270
  }
606
1271
 
607
1272
  // -----------------------------------------------------------------
@@ -642,6 +1307,9 @@ function readContentManifest(cwd) {
642
1307
  */
643
1308
  function writeContentManifest(cwd, manifest) {
644
1309
  const lockPath = path.join(cwd, CONTENT_LOCK_FILENAME);
1310
+ // Nothing to record and no lock yet (e.g. the method could not run at
1311
+ // all): don't leave an empty lock file in a folder the user never synced.
1312
+ if (Object.keys(manifest).length === 0 && !fs.existsSync(lockPath)) return;
645
1313
  const sorted = {};
646
1314
  for (const key of Object.keys(manifest).sort()) {
647
1315
  sorted[key] = manifest[key];
@@ -673,18 +1341,28 @@ function hashFileContent(filePath) {
673
1341
  *
674
1342
  * Local file I/O only — cheap relative to the API dollars it estimates.
675
1343
  *
676
- * @param {string} contentDir - Path to Hugo content directory
1344
+ * @param {string} contentDir - Path to the content directory
677
1345
  * @param {string} sourceLocale - Source language code
678
1346
  * @param {Array<[string, object]>} pairEntries - Pair graph entries [pairKey, pairConfig]
679
1347
  * @param {string} cwd - Project root (for the content lock file)
1348
+ * @param {object} [options]
1349
+ * @param {object} [options.tm] - Loaded TM. When given, each pending item also
1350
+ * reports billedChars: only the front-matter fields and body blocks the TM
1351
+ * does NOT hold — what the run will actually pay for (lib/content-estimate.js).
1352
+ * @param {string[]|null} [options.translatableFields] - Front-matter fields
1353
+ * (null → DEFAULT_TRANSLATABLE_FIELDS, as runContentSync)
1354
+ * @param {boolean} [options.fresh] - --fresh: nothing refused before is held back
680
1355
  * @returns {{ sourceFileCount: number, pendingTranslations: number, pendingSourceChars: number,
681
- * byTarget: Object<string, { pendingTranslations: number, pendingSourceChars: number }> }}
1356
+ * byTarget: Object<string, { pendingTranslations: number, pendingSourceChars: number,
1357
+ * billedChars: number|null }> }}
682
1358
  * pendingSourceChars sums the source file characters of every pending
683
1359
  * (file × pair) translation — the rough basis for token estimation.
684
1360
  * byTarget breaks the same counts down per target code so the cost table
685
1361
  * can price each pair with its own method.
686
1362
  */
687
- function countPendingContentTranslations(contentDir, sourceLocale, pairEntries, cwd) {
1363
+ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries, cwd, {
1364
+ tm = null, translatableFields = null, fileScope = null, forceContent = false, fresh = false,
1365
+ } = {}) {
688
1366
  const result = { sourceFileCount: 0, pendingTranslations: 0, pendingSourceChars: 0, byTarget: {} };
689
1367
  if (!contentDir || !fs.existsSync(contentDir)) return result;
690
1368
 
@@ -693,9 +1371,13 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
693
1371
  if (sourceFiles.length === 0) return result;
694
1372
 
695
1373
  const contentManifest = readContentManifest(cwd);
1374
+ const fieldsList = translatableFields || DEFAULT_TRANSLATABLE_FIELDS;
1375
+ const parsedCache = new Map(); // sourcePath → { parsed, blocks } (locale-independent)
696
1376
 
697
1377
  for (const sourcePath of sourceFiles) {
698
1378
  const relPath = path.relative(contentDir, sourcePath);
1379
+ if (fileScope && !fileScope.includes(relPath)) continue;
1380
+ const retranslate = fileScope ? fileScope.retranslates(relPath) : false;
699
1381
  const raw = fs.readFileSync(sourcePath, 'utf-8');
700
1382
  const currentSourceHash = crypto.createHash('sha256').update(raw, 'utf-8').digest('hex');
701
1383
 
@@ -705,27 +1387,128 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
705
1387
  if (!isPathContained(targetPath, contentDir)) continue;
706
1388
 
707
1389
  const manifestKey = `${relPath}:${code}`;
708
- if (fs.existsSync(targetPath)) {
709
- const storedHash = contentManifest[manifestKey];
710
- if (storedHash && storedHash === currentSourceHash) continue; // unchanged — skipped by sync
1390
+ const targetExists = fs.existsSync(targetPath);
1391
+ const storedHash = contentManifest[manifestKey];
1392
+ if (!retranslate && targetExists) {
1393
+ if (storedHash && storedHash === currentSourceHash && !forceContent) continue; // unchanged — skipped by sync
711
1394
  if (!storedHash) {
712
1395
  // Hashless target: sync preserves it unless it is a legacy [EN] fallback
713
1396
  const existingContent = fs.readFileSync(targetPath, 'utf-8');
714
1397
  if (!existingContent.includes('[EN] ')) continue;
715
1398
  }
716
1399
  }
1400
+ if (targetExists) {
1401
+ // The same decision runContentSync makes (lib/content-review.js): a
1402
+ // translation edited by hand that the sync would leave as is costs
1403
+ // nothing. An unreadable record counts as pending (sync proceeds).
1404
+ let record = null;
1405
+ try { record = parseWrittenRecord(contentManifest[writtenRecordKey(manifestKey)]); } catch { record = null; }
1406
+ if (record) {
1407
+ const verdict = assessExistingTarget({
1408
+ targetRaw: fs.readFileSync(targetPath, 'utf-8'),
1409
+ record,
1410
+ segMode: pairConfig.contentSegmentation || 'block',
1411
+ replaceEdits: retranslate || (forceContent && Boolean(fileScope && fileScope.limitsFiles)),
1412
+ sourceCurrent: Boolean(storedHash) && storedHash === currentSourceHash,
1413
+ });
1414
+ if (verdict.action !== 'proceed') continue;
1415
+ }
1416
+ }
717
1417
 
718
1418
  result.pendingTranslations += 1;
719
1419
  result.pendingSourceChars += raw.length;
720
1420
  if (!result.byTarget[code]) {
721
- result.byTarget[code] = { pendingTranslations: 0, pendingSourceChars: 0 };
1421
+ result.byTarget[code] = { pendingTranslations: 0, pendingSourceChars: 0, billedChars: tm ? 0 : null };
722
1422
  }
723
1423
  result.byTarget[code].pendingTranslations += 1;
724
1424
  result.byTarget[code].pendingSourceChars += raw.length;
1425
+
1426
+ if (tm) {
1427
+ // The same TM ladder runContentSync runs: per field, whole body,
1428
+ // then per block — only misses are billed.
1429
+ if (!parsedCache.has(sourcePath)) parsedCache.set(sourcePath, { parsed: parseContentFile(raw), blocks: null });
1430
+ const cached = parsedCache.get(sourcePath);
1431
+ const fields = {};
1432
+ if (cached.parsed.hasFrontMatter) {
1433
+ for (const field of fieldsList) {
1434
+ const v = cached.parsed.frontMatter[field];
1435
+ if (v && typeof v === 'string') fields[field] = v;
1436
+ }
1437
+ }
1438
+ const { billedChars, totalChars } = billableContentChars({
1439
+ tm,
1440
+ code,
1441
+ tmKey: tmMethodKey(pairConfig),
1442
+ fallbackTmKey: pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null,
1443
+ fields,
1444
+ body: cached.parsed.body,
1445
+ segMode: pairConfig.contentSegmentation || 'block',
1446
+ blockSources: () => (cached.blocks ??= translatableBlockSources(cached.parsed.body)),
1447
+ // What the gate refused before is not sent to the method again
1448
+ // (lib/content-refusals.js) — the same holds the sync applies.
1449
+ holds: contentHolds(readRefusals(contentManifest, manifestKey), pairConfig, {
1450
+ redo: retranslate || forceContent || fresh,
1451
+ }),
1452
+ });
1453
+ // --retranslate bypasses the TM for this file: all of it is billed.
1454
+ result.byTarget[code].billedChars += retranslate ? totalChars : billedChars;
1455
+ }
725
1456
  }
726
1457
  }
727
1458
 
728
1459
  return result;
729
1460
  }
730
1461
 
731
- export { runContentSync, countPendingContentTranslations, readContentManifest, CONTENT_LOCK_FILENAME };
1462
+ /**
1463
+ * The state of the content lane (contentDir) per target locale, for
1464
+ * `champollion status` — which had shown only the key-value lane, so a
1465
+ * newsletter folder and its files' state were invisible (Round 7, school
1466
+ * persona). Local file reads only; never fails (a diagnostic).
1467
+ *
1468
+ * translated — the target exists and was made from the current source
1469
+ * (the content lock's hash matches), with no '[EN] ' block
1470
+ * outOfDate — the target was made from an older source text
1471
+ * pending — no target yet, or blocks written as the '[EN] ' last
1472
+ * resort (the next sync asks for them again)
1473
+ * unrecorded — a target the lock has no record of (made by hand or by
1474
+ * another tool): sync keeps it as it is
1475
+ *
1476
+ * @param {string} contentDir
1477
+ * @param {string} sourceLocale
1478
+ * @param {Array<[string, object]>} pairEntries
1479
+ * @param {string} cwd
1480
+ * @param {{ fallbackPrefix?: string }} [opts]
1481
+ * @returns {{ dir: string, files: number, locales: Object<string, { translated: number,
1482
+ * outOfDate: string[], pending: string[], unrecorded: string[] }> }|null}
1483
+ */
1484
+ function contentStatus(contentDir, sourceLocale, pairEntries, cwd, { fallbackPrefix = '[EN] ' } = {}) {
1485
+ if (!contentDir || !fs.existsSync(contentDir)) return null;
1486
+ let sourceFiles = [];
1487
+ try { sourceFiles = discoverContentFiles(contentDir, sourceLocale); } catch { return null; }
1488
+ const manifest = readContentManifest(cwd);
1489
+ const out = { dir: path.relative(cwd, contentDir) || '.', files: sourceFiles.length, locales: {} };
1490
+ for (const [, pairConfig] of pairEntries) {
1491
+ out.locales[pairConfig.target] ??= { translated: 0, outOfDate: [], pending: [], unrecorded: [] };
1492
+ }
1493
+ for (const sourcePath of sourceFiles) {
1494
+ const relPath = path.relative(contentDir, sourcePath).split(path.sep).join('/');
1495
+ let hash;
1496
+ try { hash = hashFileContent(sourcePath); } catch { continue; }
1497
+ for (const [, pairConfig] of pairEntries) {
1498
+ const code = pairConfig.target;
1499
+ const state = out.locales[code];
1500
+ const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
1501
+ if (!fs.existsSync(targetPath)) { state.pending.push(relPath); continue; }
1502
+ let text = '';
1503
+ try { text = fs.readFileSync(targetPath, 'utf-8'); } catch { /* unreadable: by its record only */ }
1504
+ const stored = manifest[`${path.relative(contentDir, sourcePath)}:${code}`];
1505
+ if (text.includes(fallbackPrefix)) state.pending.push(relPath);
1506
+ else if (!stored) state.unrecorded.push(relPath);
1507
+ else if (stored !== hash) state.outOfDate.push(relPath);
1508
+ else state.translated += 1;
1509
+ }
1510
+ }
1511
+ return out;
1512
+ }
1513
+
1514
+ export { runContentSync, countPendingContentTranslations, readContentManifest, contentStatus, CONTENT_LOCK_FILENAME };