champollion 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/README.md +52 -37
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +83 -36
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +51 -3
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +289 -88
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1689 -87
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +649 -130
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +16 -10
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +196 -14
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +270 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1127 -344
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +197 -38
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +726 -78
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +965 -253
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +964 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +571 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +252 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +68 -128
  70. package/lib/methods/llm.js +80 -31
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +11 -0
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +193 -106
  100. package/lib/seal.mjs +6 -5
  101. package/lib/sealed-qualifier.mjs +2 -2
  102. package/lib/segment.js +2 -1
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +880 -17
  117. package/lib/verify.js +1296 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +3 -2
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/DATA-SOVEREIGNTY.md +19 -20
  123. package/shared/LANGUAGE-CARD-FIELDS.md +1 -1
  124. package/shared/cards-fallback.json +1 -1
  125. package/shared/catalogue/card-config.json +1 -1
  126. package/shared/curated-orthography-conventions.json +26 -8
  127. package/shared/docent/faq.en.json +14 -16
  128. package/shared/docent/system-prompt.md +17 -19
  129. package/shared/explainers/tc-features.json +15 -15
  130. package/shared/gettext-plural-forms.json +45 -0
  131. package/shared/human-services.json +1 -1
  132. package/shared/method-registry.json +2 -0
  133. package/shared/metric-registry.json +96 -18
  134. package/shared/schemas/champollion-plugin.schema.json +4 -0
  135. package/shared/schemas/corpora-card.schema.json +20 -10
  136. package/shared/schemas/human-services.schema.json +2 -2
  137. package/shared/schemas/language-card.schema.json +1 -1
  138. package/shared/schemas/method-card.schema.json +1 -1
  139. package/shared/schemas/method-index-record.schema.json +67 -0
  140. package/shared/schemas/method-registry.schema.json +4 -0
  141. package/shared/schemas/metric-registry.schema.json +55 -1
  142. package/shared/docent/corpus.json +0 -11333
@@ -29,29 +29,54 @@ import {
29
29
  discoverDocusaurusContentFiles, getDocusaurusTargetPath,
30
30
  findUntranslatableNestedFields,
31
31
  } from './content.js';
32
- import { translateRawContent } from './translate.js';
32
+ import { translateRawContent, getMethod } from './translate.js';
33
33
  import {
34
34
  splitBlocks, buildBlockBatchPrompt, parseBlockBatchResponse,
35
35
  translateBlockBatchResilient,
36
36
  assertSegmentationMode,
37
37
  } from './segment.js';
38
38
  import { DEFAULT_REGISTERS } from './registers.js';
39
- import { DEFAULT_JSON_CONCURRENCY, EST_CHARS_PER_KEY } from './config.js';
39
+ import { DEFAULT_JSON_CONCURRENCY } from './config.js';
40
40
  import { compileNoTranslate } from './no-translate.js';
41
- import { checkContentPreservation } from './validate.js';
41
+ import { checkContentPreservation, contentGateFault, SharedOutputIndex } from './validate.js';
42
42
  import { convertScript, applyScriptFallback } from './scripts.js';
43
43
  import { pMap } from './concurrent.js';
44
44
  import {
45
45
  loadTM, saveTM, tmSize, isTMDirty,
46
- lookupTM, lookupTMValidated, storeTM, evictTM, partitionByTM, tmMethodKey,
46
+ lookupTM, peekTM, storeTM, evictTM, partitionByTM, tmMethodKey, setModelCarryover, bypassTMFor,
47
+ describeTMChanges, adoptLegacyCoachingKeys,
47
48
  } from './tm.js';
48
49
  import {
49
- readManifest, writeManifest, detectChangedKeys, hashValue,
50
+ readLock, writeManifest, detectChangedKeys, hashValue,
50
51
  } from './hash.js';
52
+ import {
53
+ LockState, planQueue, recordRefusal, describeHeldKeys, describeFallbackOnlyKeys, keyFateNote, describeHeldNext,
54
+ shortSourceHash,
55
+ } from './locale-state.js';
56
+ import { redoCommand } from './verify.js';
51
57
  import { CONTENT_LOCK_FILENAME } from './content-sync.js';
52
- import { parseMaxCost, abortForMaxCost, printCostTable } from './cost-report.js';
58
+ import {
59
+ parseMaxCost, abortForMaxCost, maxCostVerdict, reportDryRunMaxCost, printCostTable, warnModelSwitchStrandedTM, priceContentChars,
60
+ preflightStopReason,
61
+ summarizeEstimate,
62
+ } from './cost-report.js';
63
+ import { billableContentChars, translatableBlockSources } from './content-estimate.js';
64
+ import { compileFileScope } from './file-scope.js';
53
65
  import { output } from './output.js';
54
- import { translateAndValidate } from './translate-pair.js';
66
+ import { missingKeyAdvice } from './missing-key.js';
67
+ import { translateWithFallback } from './translate-pair.js';
68
+ import {
69
+ tmKeysForPair, tmHoldsValue, createFallbackBudget, newFallbackReport, addToTally,
70
+ fallbackSummary, printFallbackReport, warnFallbackMajority, lookupContentTM, serveFieldsFromFallbackCache,
71
+ translateFieldsWithFallback, translateBlocksWithFallback, translatePageWithFallback,
72
+ refuseRepeatedCachedPage,
73
+ } from './fallback.js';
74
+ import { walkFiles } from './locale-layout.js';
75
+ import {
76
+ readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
77
+ describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME,
78
+ } from './content-refusals.js';
79
+ import { applyNamedKeyRule, reportUnmatchedKeys, unmatchedKeysSummary, reportNamedFromCache } from './named-keys.js';
55
80
 
56
81
  /**
57
82
  * Discover all JSON locale files in a Docusaurus i18n source directory.
@@ -63,20 +88,64 @@ import { translateAndValidate } from './translate-pair.js';
63
88
  * @returns {string[]} Absolute paths to JSON files
64
89
  */
65
90
  function discoverDocusaurusJSONFiles(sourceLocaleDir) {
66
- const files = [];
67
- function walk(dir) {
68
- if (!fs.existsSync(dir)) return;
69
- for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
70
- const fullPath = path.join(dir, entry.name);
71
- if (entry.isDirectory()) {
72
- walk(fullPath);
73
- } else if (entry.isFile() && entry.name.endsWith('.json')) {
74
- files.push(fullPath);
75
- }
76
- }
91
+ // The shared folder-per-locale walk (lib/locale-layout.js). Docusaurus
92
+ // keeps its historical reach: hidden entries are NOT skipped here.
93
+ return walkFiles(sourceLocaleDir, name => name.endsWith('.json'), { skipHidden: false });
94
+ }
95
+
96
+ /**
97
+ * What one (UI-string file × locale) queue becomes once the lock's refusal
98
+ * records are consulted — the key-value lane's rule (lib/locale-state.js
99
+ * planQueue), shared by the estimate and the sync so the estimate prices
100
+ * exactly what the run sends. A key the gate refused for its current source
101
+ * text is held back from the method that refused it (its fallback, when it
102
+ * has not refused it, is asked instead) unless it is named for a redo
103
+ * (`--redo keys:`), queued by `--redo all`, or the run is `--fresh`.
104
+ *
105
+ * Docusaurus records no written values, so no value counts as a person's
106
+ * edit here (a bulk redo re-translates every UI string, as it always did),
107
+ * and there is no pending retry: a UI string refused under a redo is held
108
+ * back by the next plain sync, as a content block is.
109
+ *
110
+ * @param {object} p
111
+ * @param {object} p.diff - diffLocale result for the file × locale
112
+ * @param {object} p.sourceFlat
113
+ * @param {object} p.targetFlat
114
+ * @param {string} p.nsPrefix - "docusaurus:<relPath>:" (the lock's key space)
115
+ * @param {object} p.localeState - LockState.of(code) / .peek(code)
116
+ * @param {object} p.pairConfig
117
+ * @param {{ named: Set<string>, bulk: boolean, fresh: boolean }} p.redo
118
+ * @param {string} p.fallbackPrefix
119
+ * @returns {{ held: string[], heldFromPrimary: string[] }}
120
+ */
121
+ function planUIStrings({ diff, sourceFlat, targetFlat, nsPrefix, localeState, pairConfig, redo, fallbackPrefix }) {
122
+ const plan = planQueue({
123
+ diff, sourceFlat, targetFlat, lockKeyOf: (k) => nsPrefix + k, localeState,
124
+ named: redo.named, bulk: redo.bulk, pending: new Set(), fresh: redo.fresh, pairConfig,
125
+ classify: () => 'machine',
126
+ fallbackPrefix,
127
+ });
128
+ return { held: plan.held, heldFromPrimary: plan.heldFromPrimary };
129
+ }
130
+
131
+ /**
132
+ * UI strings refused for an older source text that the file still holds as
133
+ * the English copy of that text (Docusaurus needs every id, so an untranslated
134
+ * one is written as its source message): the source was edited since, which
135
+ * lifts the hold — queued as changed. Without this the edit was invisible: the
136
+ * refused string never had its source hash recorded, and the old English no
137
+ * longer equals the new source, so nothing queued it.
138
+ *
139
+ * @returns {string[]}
140
+ */
141
+ function editedSinceRefused(sourceFlat, targetFlat, nsPrefix, localeState) {
142
+ const out = [];
143
+ for (const [k, src] of Object.entries(sourceFlat)) {
144
+ const rec = localeState.refused[nsPrefix + k];
145
+ if (!rec || typeof src !== 'string' || typeof targetFlat[k] !== 'string') continue;
146
+ if (rec.source !== shortSourceHash(src) && shortSourceHash(targetFlat[k]) === rec.source) out.push(k);
77
147
  }
78
- walk(sourceLocaleDir);
79
- return files.sort();
148
+ return out;
80
149
  }
81
150
 
82
151
  /**
@@ -97,12 +166,13 @@ function discoverDocusaurusJSONFiles(sourceLocaleDir) {
97
166
  * @param {object} config - Resolved config (localesDir)
98
167
  * @param {object} manifest - Content lock manifest (hash per file × locale)
99
168
  * @param {boolean} forceContent - --force-content: re-process up-to-date files
169
+ * @param {import('./file-scope.js').FileScope|null} [fileScope] - --files / --retranslate
100
170
  * too (action 're-translate'), WITHOUT clearing the manifest — a target
101
171
  * with no lock entry and no '[EN] ' markers is a genuine hand-translated
102
172
  * file, and force must never overwrite human work with machine output.
103
173
  * @returns {{ workItems: Array<object>, totalContentSkipped: number, recordedHashes: object }}
104
174
  */
105
- function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent) {
175
+ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent, fileScope = null) {
106
176
  const workItems = [];
107
177
  let totalContentSkipped = 0;
108
178
  const recordedHashes = {};
@@ -115,6 +185,9 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
115
185
 
116
186
  for (const sourcePath of sourceFiles) {
117
187
  const relPath = path.relative(sourceContentDir, sourcePath);
188
+ const label = `${dirName}/${relPath}`;
189
+ if (fileScope && !fileScope.includes(label)) continue;
190
+ const retranslate = fileScope ? fileScope.retranslates(label) : false;
118
191
 
119
192
  // Read + parse source file once per source path. The staleness hash
120
193
  // covers the RAW file (same as the Hugo twin's hashFileContent): a
@@ -142,7 +215,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
142
215
  }
143
216
  const { parsed, fieldsToTranslate, pageTitle, sourceHash } = sourceCache.get(sourcePath);
144
217
 
145
- for (const [, pairConfig] of pairEntries) {
218
+ for (const [pairKey, pairConfig] of pairEntries) {
146
219
  const code = pairConfig.target;
147
220
  const targetPath = getDocusaurusTargetPath(
148
221
  sourcePath, sourceContentDir, code, config.localesDir, pluginName
@@ -156,7 +229,11 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
156
229
  const manifestKey = `docusaurus:${dirName}/${relPath}:${code}`;
157
230
  let action = 'new'; // 'new' | 'changed' | 're-translate'
158
231
 
159
- if (fs.existsSync(targetPath)) {
232
+ if (retranslate) {
233
+ // --retranslate: named by the operator — no lock skip, no
234
+ // adoption; the caller makes its text miss the TM.
235
+ if (fs.existsSync(targetPath)) action = 're-translate';
236
+ } else if (fs.existsSync(targetPath)) {
160
237
  const storedHash = manifest[manifestKey];
161
238
  if (storedHash) {
162
239
  if (storedHash === sourceHash) {
@@ -190,7 +267,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
190
267
  workItems.push({
191
268
  sourcePath, parsed, fieldsToTranslate, pageTitle, sourceHash,
192
269
  relPath, dirName,
193
- pairConfig, code, targetPath, manifestKey, pluginName, action,
270
+ pairKey, pairConfig, code, targetPath, manifestKey, pluginName, action, retranslate,
194
271
  });
195
272
  }
196
273
  }
@@ -240,7 +317,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
240
317
  * Structured estimate, or null when estimation itself failed (callers
241
318
  * with --max-cost must fail safe: unknown ≠ free)
242
319
  */
243
- async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate) {
320
+ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate, lockState = null, redo = null) {
244
321
  try {
245
322
  const { estimateCost } = await import('./pairs.js');
246
323
  const costEstimates = [];
@@ -251,6 +328,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
251
328
  const sourceJSONFiles = discoverDocusaurusJSONFiles(sourceLocaleDir);
252
329
  const missesByPair = new Map(); // pairKey → miss count
253
330
  const hitsByPair = new Map(); // pairKey → TM hit count
331
+ const heldByPair = new Map(); // pairKey → keys held back (refused before; not sent)
254
332
 
255
333
  for (const sourceFilePath of sourceJSONFiles) {
256
334
  const relPath = path.relative(sourceLocaleDir, sourceFilePath);
@@ -283,26 +361,45 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
283
361
 
284
362
  // Same confirmed-echo suppression as the Phase-1 sync below, so the
285
363
  // estimate prices exactly the keys the run will actually queue.
286
- const estTmKey = tmMethodKey(pairConfig);
364
+ // With a fallback, its cached values count too (lib/fallback.js).
365
+ const estTmKeys = tmKeysForPair(pairConfig);
366
+ const localeChanged = lockState
367
+ ? [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, lockState.peek(code))])]
368
+ : changedKeys;
287
369
  const diff = diffLocale(
288
370
  sourceFlat, existingFlat, config.fallbackPrefix,
289
- config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, changedKeys,
290
- (key, sourceValue) => lookupTM(tm, sourceValue, code, estTmKey) === sourceValue,
371
+ config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
372
+ (key, sourceValue) => tmHoldsValue(tm, sourceValue, code, estTmKeys, sourceValue),
291
373
  noTranslate.active ? noTranslate.matches : null
292
374
  );
293
375
  const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
294
376
  if (stringKeys.length === 0) continue;
295
377
 
296
- const { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmMethodKey(pairConfig));
378
+ // Refused before (the lock's records): never sent to the method that
379
+ // refused them — held keys cost nothing; a fallback is not priced.
380
+ const notSent = new Set();
381
+ if (lockState && redo) {
382
+ const plan = planUIStrings({
383
+ diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState: lockState.peek(code), pairConfig, redo,
384
+ fallbackPrefix: config.fallbackPrefix,
385
+ });
386
+ for (const k of [...plan.held, ...plan.heldFromPrimary]) notSent.add(k);
387
+ }
388
+ let { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmMethodKey(pairConfig));
389
+ if (estTmKeys.length > 1) misses = misses.filter(k => lookupTM(tm, sourceFlat[k], code, estTmKeys[1]) === null);
390
+ const held = misses.filter(k => notSent.has(k)).length;
391
+ misses = misses.filter(k => !notSent.has(k));
297
392
  missesByPair.set(pairKey, (missesByPair.get(pairKey) || 0) + misses.length);
298
- hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length));
393
+ hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length - held));
394
+ heldByPair.set(pairKey, (heldByPair.get(pairKey) || 0) + held);
299
395
  }
300
396
  }
301
397
 
302
398
  for (const [pairKey, pairConfig] of pairEntries) {
303
399
  const keysToTranslate = missesByPair.get(pairKey) || 0;
304
400
  const tmHits = hitsByPair.get(pairKey) || 0;
305
- if (keysToTranslate === 0 && tmHits === 0) continue;
401
+ const heldCount = heldByPair.get(pairKey) || 0;
402
+ if (keysToTranslate === 0 && tmHits === 0 && heldCount === 0) continue;
306
403
 
307
404
  if (keysToTranslate > 0) {
308
405
  // eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
@@ -317,8 +414,15 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
317
414
  method: pairConfig.method || 'llm',
318
415
  keys: keysToTranslate,
319
416
  tmHits,
417
+ ...(heldCount > 0 && { held: heldCount }),
320
418
  estimatedCost: estimate.estimatedCost,
321
419
  source: estimate.source,
420
+ ...(estimate.local && { local: true }),
421
+ ...(estimate.note && { note: estimate.note }),
422
+ // The rate it was priced at, its source and date (cost-report.js rateLine).
423
+ ...(estimate.rate && { rate: estimate.rate }),
424
+ // No price: the model it was looked up under, so the table can name it.
425
+ ...(estimate.estimatedCost === null && estimate.model && { model: estimate.model }),
322
426
  });
323
427
  } else {
324
428
  // Fully TM-covered: zero API calls → a KNOWN $0, even for
@@ -328,8 +432,9 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
328
432
  method: pairConfig.method || 'llm',
329
433
  keys: 0,
330
434
  tmHits,
435
+ ...(heldCount > 0 && { held: heldCount }),
331
436
  estimatedCost: 0,
332
- source: 'translation-memory',
437
+ source: tmHits > 0 ? 'translation-memory' : 'nothing-to-send',
333
438
  });
334
439
  }
335
440
  }
@@ -342,72 +447,51 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
342
447
  // block sources once (only needed for 'block' mode misses).
343
448
  const blockSourcesCache = new Map(); // sourcePath → string[]
344
449
  const charsByPair = new Map(); // pairConfig (by reference) → billed source chars
450
+ const roughCharsByPair = new Map(); // pairConfig → all source chars, as if the TM were empty
345
451
  const pendingSourceFiles = new Set();
346
452
 
347
453
  for (const item of contentWorkItems) {
348
454
  pendingSourceFiles.add(item.sourcePath);
349
455
  const code = item.code;
350
- const tmKey = tmMethodKey(item.pairConfig);
351
- let chars = 0;
352
-
353
- // Front-matter fields: each field is cached on its own source
354
- // text — only TM misses reach the API.
355
- for (const text of Object.values(item.fieldsToTranslate)) {
356
- if (lookupTM(tm, text, code, tmKey) === null) chars += text.length;
357
- }
358
-
359
- // Body: whole-body TM first (a revert or lock-loss re-run is
360
- // free), then per-block in 'block' mode — the same ladder the
361
- // sync runs below.
362
- const body = item.parsed.body;
363
- if (body.trim() && lookupTM(tm, body, code, tmKey) === null) {
364
- const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
365
- if (segMode === 'page') {
366
- chars += body.length;
367
- } else {
456
+ const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
457
+ const { billedChars: chars, totalChars } = billableContentChars({
458
+ tm,
459
+ code,
460
+ tmKey: tmMethodKey(item.pairConfig),
461
+ fallbackTmKey: item.pairConfig.fallback ? tmMethodKey(item.pairConfig.fallback) : null,
462
+ fields: item.fieldsToTranslate,
463
+ body: item.parsed.body,
464
+ segMode,
465
+ blockSources: () => {
368
466
  if (!blockSourcesCache.has(item.sourcePath)) {
369
- const { protectedBody, blocks } = protectBlocks(body);
370
- blockSourcesCache.set(
371
- item.sourcePath,
372
- splitBlocks(protectedBody)
373
- .filter(seg => seg.type === 'translatable')
374
- .map(seg => restoreBlocks(seg.text, blocks))
375
- );
376
- }
377
- for (const source of blockSourcesCache.get(item.sourcePath)) {
378
- if (lookupTM(tm, source, code, tmKey) === null) chars += source.length;
467
+ blockSourcesCache.set(item.sourcePath, translatableBlockSources(item.parsed.body));
379
468
  }
380
- }
381
- }
469
+ return blockSourcesCache.get(item.sourcePath);
470
+ },
471
+ holds: item.holds || null,
472
+ });
473
+ roughCharsByPair.set(item.pairConfig, (roughCharsByPair.get(item.pairConfig) || 0) + totalChars);
382
474
 
383
475
  if (chars > 0) {
384
476
  charsByPair.set(item.pairConfig, (charsByPair.get(item.pairConfig) || 0) + chars);
385
477
  }
386
478
  }
387
479
 
388
- let contentCost = 0;
389
- let contentUnknown = false;
390
- for (const [, pairConfig] of pairEntries) {
391
- const chars = charsByPair.get(pairConfig);
392
- // Fully TM-covered pairs are a KNOWN $0 (zero API calls) — they
393
- // must not consult estimateCost, whose unknown-pricing null would
394
- // wrongly trip hasUnknownCosts and abort a free run under a cap.
395
- if (!chars) continue;
396
- const keyEquivalents = Math.ceil(chars / EST_CHARS_PER_KEY);
397
- // eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
398
- const estimate = await estimateCost(keyEquivalents, pairConfig);
399
- if (estimate.estimatedCost !== null) {
400
- contentCost += estimate.estimatedCost;
401
- } else {
402
- contentUnknown = true;
403
- }
404
- }
480
+ const billed = await priceContentChars(charsByPair, pairEntries, estimateCost);
481
+ const rough = await priceContentChars(roughCharsByPair, pairEntries, estimateCost);
482
+ const contentCost = billed.cost;
483
+ const contentUnknown = billed.unknown;
405
484
 
406
485
  content = {
407
486
  files: pendingSourceFiles.size,
408
487
  pendingTranslations: contentWorkItems.length,
409
488
  estimatedCost: contentUnknown ? null : contentCost,
489
+ costWithoutTM: rough.unknown ? null : rough.cost,
410
490
  rough: true,
491
+ // Which pairs have no price, and why (printCostTable names them).
492
+ ...(contentUnknown && { unpriced: billed.unpriced }),
493
+ // The rates the billed part was priced at.
494
+ ...(billed.rates.length > 0 && { rates: billed.rates }),
411
495
  };
412
496
  if (contentUnknown) {
413
497
  hasUnknownCosts = true;
@@ -418,16 +502,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
418
502
 
419
503
  printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
420
504
 
421
- return {
422
- currency: 'USD',
423
- pairs: costEstimates,
424
- keyCost: content && content.estimatedCost !== null
425
- ? totalEstimatedCost - content.estimatedCost
426
- : totalEstimatedCost,
427
- content,
428
- totalEstimatedCost,
429
- hasUnknownCosts,
430
- };
505
+ return summarizeEstimate(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
431
506
  } catch (costError) {
432
507
  // Non-blocking without a cap — warn and continue. Callers enforcing
433
508
  // --max-cost must treat the null return as unknown (abort).
@@ -508,9 +583,48 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
508
583
  );
509
584
  }
510
585
 
586
+ // Keys named for a redo (--redo keys: / --force-keys) must exist — the
587
+ // rule the key-value path applies (lib/named-keys.js). A name matching no
588
+ // UI string used to re-queue nothing in silence and exit 0. Checked before
589
+ // the preflight, the estimate or any spend: with no name matching nothing
590
+ // runs; with some, those are redone and the run then fails naming the rest.
591
+ // Docusaurus message ids are per file but named bare: a name re-queues
592
+ // that id in every JSON file that has it (as the diff below applies it).
593
+ const unmatchedNamed = applyNamedKeyRule({
594
+ cliArgs, config,
595
+ known: () => {
596
+ const ids = new Set();
597
+ for (const file of discoverDocusaurusJSONFiles(sourceLocaleDir)) {
598
+ for (const key of Object.keys(extractDocusaurusMessages(JSON.parse(fs.readFileSync(file, 'utf-8'))))) {
599
+ if (!isUnsafeKey(key)) ids.add(key);
600
+ }
601
+ }
602
+ return ids;
603
+ },
604
+ });
605
+
511
606
  // Thread dryRun/audit into cliArgs so the preflight check can skip
512
607
  // when appropriate — same pattern as the main sync path in sync.js
513
- const { apiKey, pairEntries } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
608
+ const { apiKey, pairEntries, preflightFailures = [] } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
609
+
610
+ // Content needs what the pair's own method needs — a local model needs no
611
+ // OPENROUTER_API_KEY (the lane used to demand it for every pair). Same
612
+ // lazy, per-pair readiness check as the content lane (content-sync.js).
613
+ const readinessCache = new Map();
614
+ const requirePairReady = async (pairKey, pairConfig, code, what) => {
615
+ if (!readinessCache.has(pairKey)) {
616
+ const method = getMethod(pairConfig.method || 'llm', pairConfig);
617
+ readinessCache.set(pairKey, Promise.resolve(method.checkReadiness({ apiKey, cwd })));
618
+ }
619
+ const readiness = await readinessCache.get(pairKey);
620
+ if (!readiness.ready) {
621
+ throw new Error(
622
+ `Docusaurus ${what} for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
623
+ ` ${readiness.reason}\n` +
624
+ missingKeyAdvice({ reasons: [readiness.reason], setupHelp: [' Set the key it names in .env.local (or your environment) to translate content.'] }).join('\n')
625
+ );
626
+ }
627
+ };
514
628
 
515
629
  if (pairEntries.length === 0) {
516
630
  output.info('No target languages configured. Add pairs to champollion.config.json.');
@@ -542,6 +656,13 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
542
656
  } else if (tmInitialSize > 0) {
543
657
  output.info(`[TM] ${tmInitialSize} cached entries loaded`);
544
658
  }
659
+ // Model carry-over (lib/tm.js) is on unless --fresh-on-model-change.
660
+ const freshOnModelChange = !!cliArgs['fresh-on-model-change'];
661
+ if (freshOnModelChange) setModelCarryover(tm, false);
662
+ // Entries made before coaching was keyed, with today's coaching: still
663
+ // served (lib/tm.js adoptLegacyCoachingKeys).
664
+ if (!noTM) adoptLegacyCoachingKeys(tm, pairEntries.map(([, pc]) => pc));
665
+ if (!noTM) warnModelSwitchStrandedTM(tm, pairEntries.map(([, pc]) => pc), { fresh: freshOnModelChange });
545
666
 
546
667
  // Phase-1 source-hash manifest (.champollion.lock) — read up front
547
668
  // because the cost estimator mirrors Phase 1's changed-key detection.
@@ -551,8 +672,19 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
551
672
  // manifest, EDITING an English UI string never re-translated it: the
552
673
  // diff only sees missing keys and [EN] fallbacks, so a changed source
553
674
  // value looked "fully synced" forever.
554
- const lockManifest = readManifest(cwd);
675
+ const lock = readLock(cwd);
676
+ const lockManifest = lock.source;
555
677
  const updatedLockManifest = { ...lockManifest };
678
+ // The lock's per-locale record (lib/locale-state.js): which UI strings the
679
+ // quality gate refused, per (file × key × locale), for their current source
680
+ // text and the method key that produced the refused answer. A plain sync
681
+ // holds them back from that method — the key-value rule. Keyed like the
682
+ // manifest ("docusaurus:<relPath>:<id>").
683
+ const lockState = new LockState(lock.locales);
684
+ // What counts as an explicit redo (always sent, never held back): ids named
685
+ // by --redo keys: / --force-keys, every id under --redo all (--force), and
686
+ // anything under --fresh.
687
+ const redo = { named: new Set(config.forceKeys || []), bulk: !!config._forceAllKeys, fresh: noTM };
556
688
 
557
689
  // ── Content discovery + pending-work scan ─────────────────────
558
690
  // Done up front (not in Phase 2) because the cost estimate needs the
@@ -586,9 +718,29 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
586
718
  if (fs.existsSync(contentLockPath)) {
587
719
  try { docuContentManifest = JSON.parse(fs.readFileSync(contentLockPath, 'utf-8')); } catch { /* first run */ }
588
720
  }
721
+ // --files / --retranslate (lib/file-scope.js). Resolved at the scan, so a
722
+ // pattern that matches nothing fails before the estimate or any spend.
723
+ const fileScope = compileFileScope(cliArgs);
589
724
  const contentScan = scanDocusaurusContentWork(
590
- contentSources, pairEntries, config, docuContentManifest, forceContent
725
+ contentSources, pairEntries, config, docuContentManifest, forceContent, fileScope
591
726
  );
727
+ if (fileScope) fileScope.assertAllMatched();
728
+ // --retranslate files miss the TM for this run (estimate and sync alike),
729
+ // so they are priced and translated fresh; new results replace the cache.
730
+ for (const item of contentScan.workItems) {
731
+ if (!item.retranslate) continue;
732
+ bypassTMFor(tm, item.code, [
733
+ ...Object.values(item.fieldsToTranslate), item.parsed.body, ...translatableBlockSources(item.parsed.body),
734
+ ]);
735
+ }
736
+ // Blocks and front-matter fields the quality gate refused before
737
+ // (lib/content-refusals.js): not sent to the method that refused them again
738
+ // unless the page is named for a redo (--redo files: / --redo content /
739
+ // --retranslate / --fresh). The estimate and the sync read the same holds.
740
+ for (const item of contentScan.workItems) {
741
+ item.refusalsBefore = readRefusals(docuContentManifest, item.manifestKey);
742
+ item.holds = contentHolds(item.refusalsBefore, item.pairConfig, { redo: forceContent || item.retranslate || noTM });
743
+ }
592
744
 
593
745
  // ── Pre-sync cost estimation + --max-cost gate ────────────────
594
746
  // Mirrors sync.js: without a cap the estimate is informational and
@@ -598,26 +750,63 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
598
750
  // exact thing a capped user needs to see.
599
751
  const maxCost = parseMaxCost(cliArgs['max-cost']);
600
752
  const costEstimate = await printDocusaurusCostEstimate(
601
- pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate
753
+ pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate, lockState, redo
602
754
  );
755
+ // Machine-readable estimate BEFORE the gate (see sync.js).
756
+ if (costEstimate) output.event('cost', costEstimate);
757
+ // A dry run never stops at the cap, but says what the real run would do.
758
+ let dryMaxCost = null;
603
759
  if (maxCost !== null && !dryRun) {
604
- if (!costEstimate) {
605
- return abortForMaxCost(
606
- maxCost, null,
607
- 'Cost estimation failed, so --max-cost cannot be enforced (unknown is not free).'
608
- );
609
- }
610
- if (costEstimate.hasUnknownCosts) {
611
- return abortForMaxCost(
612
- maxCost, null,
613
- 'Some pairs have unknown pricing, so the total cost cannot be bounded (unknown is not free).'
614
- );
760
+ const verdict = maxCostVerdict(maxCost, costEstimate);
761
+ if (verdict.wouldStop) return abortForMaxCost(maxCost, verdict.estimatedCost, verdict.reason);
762
+ } else if (maxCost !== null) {
763
+ dryMaxCost = reportDryRunMaxCost(maxCost, costEstimate, { stopsEarlier: preflightStopReason(preflightFailures) });
764
+ }
765
+
766
+ // Fallback methods are outside the estimate (they translate what the
767
+ // primary fails, unknown in advance). Under --max-cost each fallback batch
768
+ // must fit in what the estimate left of the cap (lib/fallback.js).
769
+ if (pairEntries.some(([, p]) => p.fallback)) {
770
+ output.info(
771
+ 'Fallback methods are not in this estimate — they only translate what the primary method fails, '
772
+ + 'which is not known in advance.'
773
+ + (maxCost !== null ? ' Under --max-cost each fallback batch is priced before it runs and skipped if it would pass the cap.' : '')
774
+ );
775
+ }
776
+ const fallbackBudget = createFallbackBudget({
777
+ maxCost: dryRun ? null : maxCost,
778
+ committed: costEstimate?.knownEstimatedCost ?? 0,
779
+ cwd,
780
+ });
781
+ // Different inputs, same output (lib/validate.js SharedOutputIndex): one
782
+ // index per locale for this run's UI strings and Markdown pages — cached
783
+ // and fresh alike, as in the content lane (lib/content-sync.js).
784
+ const sharedOutputIndexes = new Map();
785
+ const sharedOutputsFor = (code) => {
786
+ if (!sharedOutputIndexes.has(code)) {
787
+ sharedOutputIndexes.set(code, new SharedOutputIndex({ protectedTerms: config.protectedTerms || [] }));
615
788
  }
616
- if (costEstimate.totalEstimatedCost > maxCost) {
617
- return abortForMaxCost(
618
- maxCost, costEstimate.totalEstimatedCost,
619
- 'Estimated translation cost exceeds the --max-cost cap.'
620
- );
789
+ return sharedOutputIndexes.get(code);
790
+ };
791
+ // Per-pair tallies of what each fallback did: JSON keys, Markdown segments.
792
+ const keyFallbackTallies = new Map();
793
+ const contentFallbackTallies = new Map();
794
+ // A page some of whose text the fallback produced (its answer or its
795
+ // cache): named in the [FALLBACK] line.
796
+ const notePage = (pairKey, page) => {
797
+ const tally = contentFallbackTallies.get(pairKey);
798
+ if (tally && !tally.produced.includes(page)) tally.produced.push(page);
799
+ };
800
+ const tallyContent = (pairKey, report, page) => {
801
+ if (!contentFallbackTallies.has(pairKey)) return;
802
+ addToTally(contentFallbackTallies.get(pairKey), report);
803
+ if (report && (report.accepted > 0 || report.cached > 0)) notePage(pairKey, page);
804
+ };
805
+ if (!dryRun) {
806
+ for (const [pairKey, pc] of pairEntries) {
807
+ if (!pc.fallback) continue;
808
+ keyFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
809
+ contentFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
621
810
  }
622
811
  }
623
812
 
@@ -628,6 +817,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
628
817
 
629
818
  let totalJSONKeys = 0;
630
819
  let totalJSONCopied = 0;
820
+ let totalJSONFailed = 0;
821
+ // UI strings held back (refused before; not sent) and what the next sync
822
+ // does with each one this run could not translate (lib/locale-state.js).
823
+ let totalJSONHeld = 0;
824
+ let totalJSONWritten = 0;
825
+ const jsonFates = { retry: [], held: [] }; // { pair, key }
826
+ // Every lock key the source still has (refused records for others are dropped).
827
+ const liveLockKeys = new Set();
631
828
 
632
829
  for (const sourceFilePath of sourceJSONFiles) {
633
830
  const relPath = path.relative(sourceLocaleDir, sourceFilePath);
@@ -649,6 +846,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
649
846
  // The manifest stores namespaced keys; detectChangedKeys expects the
650
847
  // same key space as sourceFlat, so build a per-file un-namespaced view.
651
848
  const nsPrefix = `docusaurus:${relPath}:`;
849
+ for (const key of Object.keys(sourceFlat)) liveLockKeys.add(nsPrefix + key);
652
850
  const fileOldManifest = {};
653
851
  for (const [nsKey, storedHash] of Object.entries(lockManifest)) {
654
852
  if (nsKey.startsWith(nsPrefix)) {
@@ -664,10 +862,11 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
664
862
  const pairResults = await pMap(pairEntries, async ([pairKey, pairConfig]) => {
665
863
  const code = pairConfig.target;
666
864
  const targetFilePath = path.join(config.localesDir, code, relPath);
865
+ const filename = `${code}/${relPath}`;
667
866
 
668
867
  // Security: verify target path stays within i18n directory
669
868
  if (!isPathContained(targetFilePath, config.localesDir)) {
670
- output.error(`${code}/${relPath} — refusing to write outside i18n directory`);
869
+ output.error(`${filename} — refusing to write outside i18n directory`);
671
870
  // Nothing was attempted, but nothing succeeded either: keep every
672
871
  // to-be-processed key out of the "succeeded" set so its manifest
673
872
  // hash is not advanced (an unwritable locale must re-fire).
@@ -685,11 +884,15 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
685
884
  // re-translate (they are otherwise invisible to the diff). Echo keys
686
885
  // (target === source) the TM confirms as pipeline-produced are NOT
687
886
  // requeued — see lib/diff.js isConfirmedEcho.
688
- const tmKey = tmMethodKey(pairConfig);
887
+ // A value the pair's fallback produced is cached under the fallback.
888
+ const tmKeys = tmKeysForPair(pairConfig);
889
+ const localeState = dryRun ? lockState.peek(code) : lockState.of(code);
890
+ // A refused string whose source was edited since: queued (the hold lifts).
891
+ const localeChanged = [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, localeState)])];
689
892
  const diff = diffLocale(
690
893
  sourceFlat, existingFlat, config.fallbackPrefix,
691
- config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, changedKeys,
692
- (key, sourceValue) => lookupTM(tm, sourceValue, code, tmKey) === sourceValue,
894
+ config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
895
+ (key, sourceValue) => tmHoldsValue(tm, sourceValue, code, tmKeys, sourceValue),
693
896
  noTranslate.active ? noTranslate.matches : null
694
897
  );
695
898
 
@@ -697,12 +900,38 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
697
900
  return { keys: 0, failedKeys: [] };
698
901
  }
699
902
 
903
+ // Refused before (the lock's records): held back from the method that
904
+ // refused them — the key-value rule (planUIStrings).
905
+ const plan = planUIStrings({
906
+ diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState, pairConfig, redo,
907
+ fallbackPrefix: config.fallbackPrefix,
908
+ });
909
+ const heldSet = new Set(plan.held);
910
+ const lk = (k) => nsPrefix + k;
911
+
700
912
  let keysProcessed = 0;
701
913
  let keysCopied = 0;
914
+ let keysWritten = 0; // translations written (the model, its fallback, the cache)
702
915
  const failedKeys = [];
916
+ const heldKeys = [];
917
+ const fates = { retry: [], held: [] };
703
918
 
704
919
  if (diff.toProcess.length > 0 || diff.noTranslate.length > 0) {
705
- output.info(`${code}/${relPath} — ${diffLabel(diff)}`);
920
+ // The per-file line counts what goes to the pipeline; held keys are
921
+ // said on their own line below.
922
+ const shown = heldSet.size === 0 ? diff : Object.fromEntries(Object.entries(diff).map(([k, v]) => (
923
+ [k, Array.isArray(v) && k !== 'noTranslate' && k !== 'extra' ? v.filter(x => !heldSet.has(x)) : v])));
924
+ const label = diffLabel(shown);
925
+ output.info(`${filename} — ${plan.held.length > 0 ? `${label === 'fully synced' ? '' : `${label} + `}${plan.held.length} held back` : label}`);
926
+ if (plan.held.length > 0) {
927
+ output.warn(describeHeldKeys({ filename, keys: plan.held, pairConfig, command: redoCommand(plan.held, { pair: pairKey }) }));
928
+ }
929
+ if (plan.heldFromPrimary.length > 0) {
930
+ output.info(describeFallbackOnlyKeys({ filename, keys: plan.heldFromPrimary, pairConfig }));
931
+ }
932
+ // Named for a redo but served from the cache (the model is not asked
933
+ // without --fresh): said, with the command that asks the model again.
934
+ const namedQueued = redo.fresh ? [] : diff.toProcess.filter(k => redo.named.has(k) && typeof sourceFlat[k] === 'string');
706
935
 
707
936
  if (!dryRun) {
708
937
  // Merged output starts from what's on disk, then takes the verbatim
@@ -712,6 +941,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
712
941
  const mergedFlat = { ...existingFlat };
713
942
  for (const key of diff.noTranslate) {
714
943
  mergedFlat[key] = sourceFlat[key];
944
+ delete localeState.refused[lk(key)];
715
945
  }
716
946
  keysCopied = diff.noTranslate.length;
717
947
  const writeMerged = () => {
@@ -719,32 +949,77 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
719
949
  fs.mkdirSync(path.dirname(targetFilePath), { recursive: true });
720
950
  fs.writeFileSync(targetFilePath, JSON.stringify(docuOutput, null, 2) + '\n', 'utf-8');
721
951
  };
952
+ // What the gate refused this run is remembered per key (held back
953
+ // by the next plain sync); what got no usable answer is asked again.
954
+ const settleUnfilled = (keys, refusedBy = {}) => {
955
+ for (const k of keys) {
956
+ recordRefusal(localeState, lk(k), sourceFlat[k], refusedBy[k]);
957
+ fates[(refusedBy[k] || []).length > 0 ? 'held' : 'retry'].push(k);
958
+ }
959
+ };
722
960
 
723
961
  if (keysCopied > 0) {
724
962
  const sample = diff.noTranslate.slice(0, 3)
725
963
  .map(k => `${k} (${noTranslate.reason(k, sourceFlat[k])})`)
726
964
  .join(', ');
727
965
  const more = keysCopied > 3 ? `, +${keysCopied - 3} more` : '';
728
- output.info(`${code}/${relPath} — copied ${keysCopied} no-translate key(s) verbatim: ${sample}${more}`);
966
+ output.info(`${filename} — copied ${keysCopied} no-translate key(s) verbatim: ${sample}${more}`);
729
967
  }
730
968
 
731
969
  let translated = null;
970
+ let result = null;
732
971
  const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
733
972
 
734
973
  if (stringKeys.length > 0) {
735
- // Shared pipeline: TM partition → API call → TM store → quality gate
736
- const result = await translateAndValidate(stringKeys, sourceFlat, pairConfig, pairKey, {
737
- apiKey, tm, targetCode: code, descriptions,
974
+ // Shared pipeline: TM partition → API call → quality gate → TM
975
+ // store, then the pair's fallback (if any) for what that left
976
+ // untranslated (lib/translate-pair.js translateWithFallback).
977
+ // Keys held back: the cache is still read, the method not asked.
978
+ result = await translateWithFallback(stringKeys, sourceFlat, pairConfig, pairKey, {
979
+ apiKey, tm, targetCode: code, descriptions, budget: fallbackBudget, cwd,
980
+ // One per locale, as on the standard path: a memorized sentence
981
+ // repeated across UI strings is refused (validate.js).
982
+ sharedOutputs: sharedOutputsFor(code),
983
+ noSendPrimary: new Set([...plan.held, ...plan.heldFromPrimary]),
984
+ noSendFallback: heldSet,
738
985
  });
739
986
  translated = result.translated;
987
+ if (keyFallbackTallies.has(pairKey)) addToTally(keyFallbackTallies.get(pairKey), result.fallback);
988
+ const heldHere = new Set(result.heldKeys || []);
989
+ heldKeys.push(...stringKeys.filter(k => heldHere.has(k) && !(translated && k in translated)));
740
990
 
741
991
  if (translated) {
742
- output.progress(result.failures.length > 0 ? ` [OK] (${result.failures.length} failed gate)\n` : ' [OK]\n');
743
- } else if (result.apiReturnedNull || (result.failures.length > 0 && !translated)) {
744
- output.progress(result.apiReturnedNull ? ' [ERR] translation failed\n' : ' [ERR] all failed quality gate\n');
745
- output.error(`${code}/${relPath}: Translation failed. Check API key and method configuration.`);
992
+ const notDone = stringKeys.filter(k => !(k in translated));
993
+ const refusedHere = notDone.filter(k => (result.refusedBy?.[k] || []).length > 0);
994
+ const heldNow = notDone.filter(k => heldHere.has(k));
995
+ const noAnswer = notDone.length - refusedHere.length - heldNow.length;
996
+ output.progressDone(filename, notDone.length === 0 ? '[OK]' : `[WARN] ${notDone.length} of ${stringKeys.length} key(s) not translated (${[
997
+ refusedHere.length > 0 && `${refusedHere.length} refused by the quality gate`,
998
+ heldNow.length > 0 && `${heldNow.length} held back`,
999
+ noAnswer > 0 && `${noAnswer} not returned by the method`,
1000
+ ].filter(Boolean).join(', ')})`);
1001
+ const answered = new Set(result.answeredKeys || []);
1002
+ const fromCache = namedQueued.filter(k => k in translated && !answered.has(k));
1003
+ await reportNamedFromCache({
1004
+ filename, keys: fromCache, served: translated, onDisk: existingFlat, pairConfig, cwd,
1005
+ command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
1006
+ });
1007
+ } else if (heldKeys.length === stringKeys.length) {
1008
+ // Everything queued was held back: nothing was asked, nothing failed anew.
1009
+ output.progressDone(filename, `[WARN] ${heldKeys.length} key(s) held back, not translated`);
1010
+ for (const k of heldKeys) fates.held.push(k);
1011
+ if (keysCopied > 0) writeMerged();
1012
+ return { keys: 0, copied: keysCopied, failedKeys: [], heldKeys, fates };
1013
+ } else if (result.apiReturnedNull || result.failures.length > 0) {
1014
+ output.progressDone(filename, result.apiReturnedNull ? '[ERR] translation failed' : '[ERR] all failed quality gate');
1015
+ output.error(result.apiReturnedNull
1016
+ ? `${filename}: Translation failed. Check API key and method configuration.`
1017
+ : `${filename}: all translations were rejected by the quality gate${pairConfig.fallback ? ` (and by its fallback, ${pairConfig.fallback.method})` : ''} — see the gate failures above.`);
746
1018
  if (keysCopied > 0) writeMerged();
747
- return { keys: 0, copied: keysCopied, failedKeys: stringKeys };
1019
+ const failedNow = stringKeys.filter(k => !heldKeys.includes(k));
1020
+ settleUnfilled(failedNow, result.refusedBy || {});
1021
+ for (const k of heldKeys) fates.held.push(k);
1022
+ return { keys: 0, copied: keysCopied, failedKeys: failedNow, heldKeys, fates };
748
1023
  }
749
1024
  }
750
1025
 
@@ -753,6 +1028,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
753
1028
  // for it, fallbacks first, and a value with unmappable letters
754
1029
  // stays whole in the working script (warned, not failed).
755
1030
  const scriptConverterKey = pairConfig.scriptResolution?.converterKey || null;
1031
+ const heldNow = new Set(heldKeys);
756
1032
  for (const key of diff.toProcess) {
757
1033
  if (translated && key in translated) {
758
1034
  let value = translated[key];
@@ -763,50 +1039,87 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
763
1039
  value = converted;
764
1040
  } else {
765
1041
  output.warn(
766
- `${code}/${relPath}: key "${key}" kept in working script — `
1042
+ `${filename}: key "${key}" kept in working script — `
767
1043
  + `unmapped letter(s): ${unmapped.join(', ')} (see "scriptFallback")`
768
1044
  );
769
1045
  }
770
1046
  }
771
1047
  mergedFlat[key] = value;
1048
+ keysWritten++;
1049
+ // Filled (by the model, its fallback or the cache): no longer refused.
1050
+ delete localeState.refused[lk(key)];
1051
+ } else if (heldNow.has(key)) {
1052
+ // Held back: said above; not sent, not counted as failed.
1053
+ fates.held.push(key);
772
1054
  } else if (typeof sourceFlat[key] === 'string') {
773
- output.warn(`${code}/${relPath}: key "${key}" not translated — skipping`);
1055
+ const fate = (result?.refusedBy?.[key] || []).length > 0 ? 'held' : 'retry';
1056
+ output.warn(`${filename}: key "${key}" ${keyFateNote(fate, pairConfig)}`);
774
1057
  failedKeys.push(key);
1058
+ settleUnfilled([key], result?.refusedBy || {});
775
1059
  } else {
776
1060
  mergedFlat[key] = sourceFlat[key];
777
1061
  }
778
1062
  }
779
1063
 
780
- keysProcessed = diff.toProcess.length;
1064
+ keysProcessed = diff.toProcess.length - heldKeys.length;
781
1065
 
782
1066
  // Inject back into Docusaurus format and write
783
1067
  writeMerged();
784
1068
  } else {
785
- keysProcessed = diff.toProcess.length;
1069
+ // Held keys would not be sent: not counted.
1070
+ keysProcessed = diff.toProcess.length - plan.held.length;
786
1071
  keysCopied = diff.noTranslate.length;
1072
+ if (namedQueued.length > 0) {
1073
+ // What the real run would serve from the cache (the pair's entry,
1074
+ // then its fallback's — the same partition the estimate makes).
1075
+ const served = {};
1076
+ for (const k of namedQueued.filter(x => !heldSet.has(x))) {
1077
+ for (const mk of tmKeys) {
1078
+ const v = peekTM(tm, sourceFlat[k], code, mk);
1079
+ if (v !== null) { served[k] = v; break; }
1080
+ }
1081
+ }
1082
+ const fromCache = Object.keys(served);
1083
+ await reportNamedFromCache({
1084
+ filename, keys: fromCache, served, onDisk: existingFlat, pairConfig, cwd, dryRun: true,
1085
+ command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
1086
+ });
1087
+ }
1088
+ heldKeys.push(...plan.held);
787
1089
  }
788
1090
  }
789
1091
 
790
1092
  if (diff.extra.length > 0) {
791
- output.warn(`${code}/${relPath} — ${diff.extra.length} extra key(s)`);
1093
+ output.warn(`${filename} — ${diff.extra.length} extra key(s)`);
792
1094
  }
793
1095
 
794
- return { keys: keysProcessed, copied: keysCopied, failedKeys };
1096
+ return { keys: keysProcessed, written: keysWritten, copied: keysCopied, failedKeys, heldKeys, fates };
795
1097
  }, { concurrency: jsonConcurrency });
796
1098
 
797
1099
  // Aggregate results for this JSON file
798
1100
  const fileFailedKeys = new Set();
799
- for (const r of pairResults) {
1101
+ // Held keys keep their manifest hash too (an edited source must still
1102
+ // re-fire), but they are not failures.
1103
+ const fileUnfilledKeys = new Set();
1104
+ pairResults.forEach((r, i) => {
800
1105
  totalJSONKeys += r.keys;
1106
+ totalJSONWritten += r.written || 0;
801
1107
  totalJSONCopied += r.copied || 0;
802
- for (const k of r.failedKeys || []) fileFailedKeys.add(k);
803
- }
1108
+ totalJSONHeld += (r.heldKeys || []).length;
1109
+ for (const k of r.failedKeys || []) { fileFailedKeys.add(k); fileUnfilledKeys.add(k); }
1110
+ for (const k of r.heldKeys || []) fileUnfilledKeys.add(k);
1111
+ const pair = pairEntries[i][0];
1112
+ for (const fate of ['retry', 'held']) {
1113
+ for (const key of r.fates?.[fate] || []) jsonFates[fate].push({ pair, key });
1114
+ }
1115
+ });
1116
+ totalJSONFailed += fileFailedKeys.size;
804
1117
 
805
1118
  // Update the manifest for this file: record the current hash ONLY for
806
1119
  // keys that succeeded in every locale that attempted them. A failed
807
- // key keeps its OLD hash (or none) so it is detected as changed and
808
- // RE-FIRES on the next sync — advancing the hash for a failed key
809
- // would silently mark the stale translation as current forever.
1120
+ // (or held) key keeps its OLD hash (or none) so it is detected as
1121
+ // changed and RE-FIRES on the next sync — advancing the hash for a failed
1122
+ // key would silently mark the stale translation as current forever.
810
1123
  if (!dryRun) {
811
1124
  for (const nsKey of Object.keys(updatedLockManifest)) {
812
1125
  // Own-property check: `in` walks the prototype chain, so a source
@@ -818,7 +1131,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
818
1131
  }
819
1132
  for (const [key, value] of Object.entries(sourceFlat)) {
820
1133
  const nsKey = nsPrefix + key;
821
- if (fileFailedKeys.has(key)) {
1134
+ if (fileUnfilledKeys.has(key)) {
822
1135
  // Restore/keep the pre-sync state for failed keys.
823
1136
  if (Object.prototype.hasOwnProperty.call(lockManifest, nsKey)) {
824
1137
  updatedLockManifest[nsKey] = lockManifest[nsKey];
@@ -832,25 +1145,68 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
832
1145
  }
833
1146
  }
834
1147
 
835
- // Persist the Phase 1 source-hash manifest (skip in dry-run — a preview
836
- // must not mark changed keys as resolved).
1148
+ // Persist the Phase 1 source-hash manifest and the refusal records (skip
1149
+ // in dry-run — a preview must not mark changed keys as resolved). Records
1150
+ // of UI strings the source no longer has are dropped for the locales this
1151
+ // run processed.
837
1152
  if (!dryRun && sourceJSONFiles.length > 0) {
838
- writeManifest(cwd, updatedLockManifest);
1153
+ for (const [, pc] of pairEntries) {
1154
+ const refused = lockState.of(pc.target).refused;
1155
+ for (const k of Object.keys(refused)) {
1156
+ if (k.startsWith('docusaurus:') && !liveLockKeys.has(k)) delete refused[k];
1157
+ }
1158
+ }
1159
+ writeManifest(cwd, updatedLockManifest, lockState.toJSON());
1160
+ }
1161
+ for (const [pairKey, tally] of keyFallbackTallies) {
1162
+ printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
1163
+ warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
839
1164
  }
840
1165
 
841
1166
  const copiedNote = totalJSONCopied > 0
842
1167
  ? ` (+${totalJSONCopied} copied verbatim, no-translate)`
843
1168
  : '';
844
- if (totalJSONKeys > 0) {
1169
+ const heldNote = totalJSONHeld > 0
1170
+ ? `; ${totalJSONHeld} held back (refused before; not sent, not billed)`
1171
+ : '';
1172
+ const failedNote = totalJSONFailed > 0 ? `; ${totalJSONFailed} failed (listed above)` : '';
1173
+ // A real run counts what it wrote; a dry run what it would send.
1174
+ const shownKeys = dryRun ? totalJSONKeys : totalJSONWritten;
1175
+ if (shownKeys > 0) {
845
1176
  const action = dryRun ? 'Would process' : 'Synced';
846
- output.ok(`${action} ${totalJSONKeys} JSON key(s)${copiedNote}`);
1177
+ output.ok(`${action} ${shownKeys} JSON key(s)${copiedNote}${failedNote}${heldNote}`);
1178
+ } else if (totalJSONHeld > 0 || totalJSONFailed > 0) {
1179
+ output.warn(`No JSON key ${dryRun ? 'would be' : 'was'} translated${copiedNote}${failedNote}${heldNote}`);
847
1180
  } else if (totalJSONCopied > 0) {
848
1181
  output.ok(`All JSON files fully synced${copiedNote}`);
849
1182
  } else {
850
1183
  output.ok('All JSON files fully synced');
851
1184
  }
1185
+ // What the next sync does with the UI strings this run could not translate
1186
+ // — said per outcome, in the key-value lane's words.
1187
+ if (!dryRun && (jsonFates.retry.length > 0 || jsonFates.held.length > 0)) {
1188
+ if (jsonFates.retry.length > 0) {
1189
+ output.warn(` ${jsonFates.retry.length} UI string(s) got no usable answer (missing from the response, or the method failed) — the next sync asks for them again.`);
1190
+ }
1191
+ // Ids are named bare (a name re-queues that id in every JSON file that has it).
1192
+ const heldNext = [];
1193
+ const seen = new Set();
1194
+ for (const { pair, key } of jsonFates.held) {
1195
+ if (seen.has(`${pair}\u0000${key}`)) continue;
1196
+ seen.add(`${pair}\u0000${key}`);
1197
+ heldNext.push({ pair, key });
1198
+ }
1199
+ for (const line of describeHeldNext(heldNext, (keys, pair) => redoCommand(keys, { pair }))) output.warn(line);
1200
+ }
852
1201
 
853
1202
  // ── Phase 2: Markdown content (docs + blog) ───────────────────
1203
+ let contentFailures = 0; // read after the TM save, outside the content block
1204
+ let contentTranslated = 0;
1205
+ let contentFailedItems = [];
1206
+ // Blocks/fields refused before and held back this run, and refused this run.
1207
+ let contentHeldBack = 0;
1208
+ const contentHeldBackItems = [];
1209
+ let contentRefused = 0;
854
1210
 
855
1211
  if (contentSources.length === 0) {
856
1212
  output.info('No docs/ or blog/ directories found — skipping content sync.');
@@ -872,7 +1228,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
872
1228
  const concurrency = config.contentConcurrency || 48;
873
1229
 
874
1230
  const totalWork = workItems.length;
875
- let contentFailures = 0;
1231
+ // Refusals per translation (lib/content-refusals.js), applied to the lock
1232
+ // with each item's outcome.
1233
+ const refusalStates = new Map();
1234
+ // Every failed (file × locale), with what it was left as on disk — the
1235
+ // end-of-run list an operator needs to tell "failed, previous
1236
+ // translation kept" from "failed, will retry" (dogfood 2026-08-28,
1237
+ // finding 5).
1238
+ const failedItems = [];
876
1239
  // Warn at most once per source file about front matter fields we can't
877
1240
  // translate (arrays / nested blocks like `related:`). The check + add are
878
1241
  // synchronous (no await between), so this is race-free under pMap.
@@ -905,23 +1268,67 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
905
1268
  await pMap(workItems, async (item) => {
906
1269
  const {
907
1270
  parsed, fieldsToTranslate, pageTitle, sourceHash,
908
- relPath, dirName, pairConfig, code,
1271
+ relPath, dirName, pairKey, pairConfig, code,
909
1272
  targetPath, manifestKey, action,
910
1273
  } = item;
911
1274
 
1275
+ let itemFailed = false;
1276
+ // This translation's refusals, recorded with its outcome (held,
1277
+ // failed or written) — in the incremental lock writes too.
1278
+ const settleRefusals = () => {
1279
+ const st = refusalStates.get(manifestKey);
1280
+ if (!st || dryRun) return;
1281
+ storeRefusals(updatedDocuManifest, manifestKey, nextRefusals(st.prior, st));
1282
+ manifestDirty = true;
1283
+ };
912
1284
  try {
913
1285
  if (action === 'changed') {
914
1286
  totalContentRetranslated++;
915
1287
  }
916
1288
 
917
1289
  if (dryRun) {
918
- const targetRel = path.relative(config.localesDir, targetPath);
919
- output.raw(` [DRY] ${dirName}/${relPath} → ${code}`);
920
- totalContent++;
1290
+ // What the quality gate refused before is held back by the real
1291
+ // run (lib/content-refusals.js) — said here.
1292
+ const preview = previewHeld({
1293
+ tm, code, pairConfig, fields: parsed.hasFrontMatter ? fieldsToTranslate : {}, body: parsed.body,
1294
+ segMode: pairConfig.contentSegmentation || config.contentSegmentation || 'block', holds: item.holds,
1295
+ });
1296
+ const held = preview.names.length > 0
1297
+ ? ` — ${preview.pageHeld ? 'would hold the page back:' : 'would hold back'} ${preview.names.join(', ')} (refused before; not sent)`
1298
+ : '';
1299
+ output.raw(` [DRY] ${dirName}/${relPath} → ${code}${held}`);
1300
+ output.event('file', {
1301
+ lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: preview.pageHeld ? 'would-hold' : 'would-translate', action,
1302
+ ...(preview.names.length > 0 && { held: preview.names.length }),
1303
+ });
1304
+ if (!preview.pageHeld) totalContent++;
921
1305
  return;
922
1306
  }
923
1307
 
924
1308
  const { rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parsed;
1309
+ const segModeHere = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
1310
+ const label = `${dirName}/${relPath}`;
1311
+ const refusal = {
1312
+ file: label, locale: code, pageHeld: false,
1313
+ prior: item.refusalsBefore || {},
1314
+ refused: [], filled: new Set(), held: [],
1315
+ fields: hasFrontMatter ? fieldsToTranslate : {},
1316
+ blockSources: segModeHere === 'block' && body.trim() ? translatableBlockSources(body) : [],
1317
+ pageSource: segModeHere === 'page' && body.trim() ? body : null,
1318
+ };
1319
+ refusalStates.set(manifestKey, refusal);
1320
+ const { holds } = item;
1321
+ // A page translated whole that the gate refused before (and that
1322
+ // the cache does not hold): held before anything of it is sent —
1323
+ // its front matter included (a pure look; the body read below is
1324
+ // the one that serves).
1325
+ if (refusal.pageSource && holds.page(body) === 'held'
1326
+ && ![tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])]
1327
+ .some(k => peekTM(tm, body, code, k) !== null)) {
1328
+ const err = new Error('held');
1329
+ err.heldNames = [PAGE_NAME];
1330
+ throw err;
1331
+ }
925
1332
 
926
1333
  // Never silently drop translatable-looking nested/array front matter
927
1334
  // (e.g. `related:` lists). The flat parser can't reach them — surface
@@ -942,6 +1349,25 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
942
1349
  // not re-serve. Mirrors content-sync.js.
943
1350
  const tmKey = tmMethodKey(pairConfig);
944
1351
 
1352
+ // This page's cached content through the repeat check, in one batch
1353
+ // and before any of it is served (lib/fallback.js): a text cached
1354
+ // for several different source strings is evicted, so the lookups
1355
+ // below miss it and it is translated again, checked the same way.
1356
+ const docLabel = `content:${dirName}/${relPath}`;
1357
+ const repeated = refuseRepeatedCachedPage({
1358
+ label: docLabel,
1359
+ fields: hasFrontMatter ? fieldsToTranslate : {},
1360
+ body,
1361
+ tm, code, pairConfig,
1362
+ sharedOutputs: sharedOutputsFor(code),
1363
+ });
1364
+ if (repeated.length > 0) {
1365
+ output.warn(
1366
+ `${dirName}/${relPath} → ${code}: cached ${repeated.join(', ')} held the same text as other, different source strings ` +
1367
+ '(a memorized sentence, not a translation) — removed from the cache and translated again.'
1368
+ );
1369
+ }
1370
+
945
1371
  // Translate front matter fields — TM first, API only for misses.
946
1372
  // Each field is cached on its own source text (exactly like a
947
1373
  // key-value sync key): a title edit re-pays only the title.
@@ -954,61 +1380,93 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
954
1380
  // gateless pipeline (hollowed titles were cached here) must not
955
1381
  // outlive the gate. Failing hits are evicted and re-billed.
956
1382
  for (const [field, cachedValue] of Object.entries(fmHits)) {
957
- if (checkContentPreservation(fieldsToTranslate[field], cachedValue)) {
1383
+ if (contentGateFault(fieldsToTranslate[field], cachedValue, pairConfig)) {
958
1384
  evictTM(tm, fieldsToTranslate[field], code, tmKey);
959
1385
  delete fmHits[field];
960
1386
  fmMisses.push(field);
961
1387
  }
962
1388
  }
963
1389
  Object.assign(translatedFields, fmHits);
1390
+ // Then the fallback's cache (lib/fallback.js ladder).
1391
+ const fbCache = serveFieldsFromFallbackCache(tm, fieldsToTranslate, fmMisses, code, pairConfig);
1392
+ Object.assign(translatedFields, fbCache.hits);
1393
+ if (Object.keys(fbCache.hits).length > 0) notePage(pairKey, label);
1394
+ const fieldsToSend = fbCache.misses;
1395
+ for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
1396
+ // Refused before: a held field fails its page as a refused one
1397
+ // does — nothing of it is sent, nothing written, lock not advanced.
1398
+ const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
1399
+ const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
1400
+ if (heldFields.length > 0) {
1401
+ const err = new Error('held');
1402
+ err.heldNames = heldFields.map(f => `front matter "${f}"`);
1403
+ throw err;
1404
+ }
964
1405
 
965
- if (fmMisses.length > 0) {
966
- if (!apiKey) {
967
- // No API key — fail loud
968
- throw new Error(
969
- `Docusaurus content sync for ${code}: no API key available.\n` +
970
- ' Set OPENROUTER_API_KEY in .env.local to translate content.'
1406
+ if (fieldsToSend.length > 0) {
1407
+ await requirePairReady(pairKey, pairConfig, code, 'content sync');
1408
+ // The pair's method, then its fallback (if any) for every field
1409
+ // it returned nothing for or hollowed. Content-preservation
1410
+ // gate on every value — Phase 2 front matter once reached disk
1411
+ // AND the TM with no validation at all, so a hollowed page
1412
+ // title was written silently and then cached forever. Nothing
1413
+ // is cached until no field is left failing; a failing field
1414
+ // skips the file and leaves its lock entry alone, so the next
1415
+ // sync retries it.
1416
+ const fm = await translateFieldsWithFallback({
1417
+ fields: fieldsToTranslate,
1418
+ misses: fieldsToSend,
1419
+ pairConfig,
1420
+ budget: fallbackBudget,
1421
+ sharedOutputs: sharedOutputsFor(code),
1422
+ label: `${docLabel} front matter`,
1423
+ translate: (keys, cfg) => translateBatch(
1424
+ keys, fieldsToTranslate, cfg,
1425
+ { apiKey, model: cfg.model, batchSize: cfg.batchSize || 30 },
1426
+ ),
1427
+ fallbackOnly: fallbackOnlyFields,
1428
+ });
1429
+ tallyContent(pairKey, fm.report, label);
1430
+ // Refused fields are remembered, whatever happens to the page.
1431
+ for (const [field, methods] of Object.entries(fm.refusedBy || {})) {
1432
+ refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
1433
+ }
1434
+ if (fm.hollowed.length > 0) {
1435
+ const h = fm.hollowed[0];
1436
+ const err = new Error(
1437
+ `Docusaurus content sync for ${code}: front matter "${h.field}" — ${h.reason}.\n` +
1438
+ ` source: ${JSON.stringify(fieldsToTranslate[h.field])}\n` +
1439
+ ` got: ${JSON.stringify(h.value)}\n` +
1440
+ (h.fallbackReason ? ` the fallback (${pairConfig.fallback.method}) failed it too: ${h.fallbackReason}\n` : '') +
1441
+ ' Nothing was written or cached. If this is a low-coverage target\n' +
1442
+ ' language, the model has no vocabulary for this string.\n' +
1443
+ ` ${describeNewHold({ file: label, pairKey, pairConfig, count: fm.hollowed.length })}`
971
1444
  );
1445
+ err.heldNext = true;
1446
+ throw err;
972
1447
  }
973
- const fmResult = await translateBatch(
974
- fmMisses, fieldsToTranslate, pairConfig,
975
- { apiKey, model: pairConfig.model, batchSize: pairConfig.batchSize || 30 },
976
- );
977
- if (fmResult) {
978
- // Content-preservation gate — Phase 2 front matter reached
979
- // disk AND the TM with no validation at all, so a hollowed
980
- // page title was written silently and then cached forever.
981
- // Throwing skips the file and leaves its lock entry alone,
982
- // so the next sync retries it.
983
- for (const [field, value] of Object.entries(fmResult)) {
984
- const sourceValue = fieldsToTranslate[field];
985
- if (typeof value !== 'string' || typeof sourceValue !== 'string') continue;
986
- const hollowed = checkContentPreservation(sourceValue, value);
987
- if (hollowed) {
988
- throw new Error(
989
- `Docusaurus content sync for ${code}: front matter "${field}" — ${hollowed.reason}.\n` +
990
- ` source: ${JSON.stringify(sourceValue)}\n` +
991
- ` got: ${JSON.stringify(value)}\n` +
992
- ' Nothing was written or cached. If this is a low-coverage target\n' +
993
- ' language, the model has no vocabulary for this string.'
994
- );
995
- }
996
- }
997
- Object.assign(translatedFields, fmResult);
998
- // Cache only what the API actually returned (the same
999
- // non-null check that gates writing it to the target file).
1000
- for (const [field, value] of Object.entries(fmResult)) {
1001
- if (typeof value === 'string' && typeof fieldsToTranslate[field] === 'string') {
1002
- storeTM(tm, fieldsToTranslate[field], code, tmKey, value);
1003
- }
1004
- }
1005
- } else {
1448
+ if (fm.noResults) {
1006
1449
  // Front matter translation failed — loud error
1007
1450
  throw new Error(
1008
- `Docusaurus content sync for ${code}: front matter translation returned no results.\n` +
1451
+ `Docusaurus content sync for ${code}: front matter translation returned no results`
1452
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
1009
1453
  ' Check your API key and method configuration.'
1010
1454
  );
1011
1455
  }
1456
+ // Cache only gate-passing values, each under the TM key of the
1457
+ // method that produced it.
1458
+ for (const st of fm.stores) storeTM(tm, st.text, code, st.tmKey, st.value);
1459
+ // A field only the fallback may be asked for that it did not
1460
+ // translate (skipped by --max-cost, or no answer): still refused
1461
+ // by the pair's method — the page is held, as with no fallback.
1462
+ const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
1463
+ if (unfilled.length > 0) {
1464
+ const err = new Error('held');
1465
+ err.heldNames = unfilled.map(f => `front matter "${f}"`);
1466
+ throw err;
1467
+ }
1468
+ Object.assign(translatedFields, fm.translated);
1469
+ for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
1012
1470
  }
1013
1471
  }
1014
1472
 
@@ -1020,40 +1478,92 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1020
1478
  let translatedBody = body;
1021
1479
  let bodyUsedFallback = false;
1022
1480
  if (body.trim()) {
1023
- const wholeBodyCached = lookupTMValidated(tm, body, code, tmKey,
1024
- (src, cached) => !checkContentPreservation(src, cached));
1481
+ // The pair's cache, then its fallback's (lib/fallback.js ladder).
1482
+ const wholeBodyCached = lookupContentTM(tm, body, code, pairConfig);
1025
1483
  if (wholeBodyCached !== null) {
1026
- translatedBody = wholeBodyCached;
1484
+ translatedBody = wholeBodyCached.text;
1485
+ for (const src of refusal.blockSources) refusal.filled.add(blockUnit(src));
1486
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
1027
1487
  } else {
1028
1488
  const segMode = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
1029
1489
  const { protectedBody, blocks } = protectBlocks(body);
1030
1490
  const promptOptions = {
1031
1491
  sourceLanguageName: DEFAULT_REGISTERS[inputLocale]?.name || inputLocale,
1032
1492
  promptContext: pairConfig.promptContext || null,
1493
+ protectedTerms: pairConfig.protectedTerms || [],
1033
1494
  };
1034
1495
 
1035
1496
  // Block stores are deferred until the reassembled body passes
1036
1497
  // the orphaned-placeholder check — the TM must never hold a
1037
- // value that would fail the gate on re-serve.
1498
+ // value that would fail the gate on re-serve. Each carries the
1499
+ // TM key of the method that produced it.
1038
1500
  const pendingBlockStores = [];
1501
+ // The whole body is cached under one method's key — only when
1502
+ // one method produced all of it.
1503
+ let wholeBodyKey = tmKey;
1504
+ let mixedProducers = false;
1505
+ // Page mode: whose whole page the checks below refuse (the
1506
+ // lane remembers it — lib/content-refusals.js "page").
1507
+ let pageRefusedBy = null;
1039
1508
 
1040
1509
  if (segMode === 'page') {
1041
- // Whole-page prompt — today's single-call behavior.
1042
- if (!apiKey) {
1043
- throw new Error(
1044
- `Docusaurus body translation for ${code}: no API key available.\n` +
1045
- ' Set OPENROUTER_API_KEY in .env.local to translate content.'
1046
- );
1510
+ // Whole-page prompt — today's single-call behavior. Refused
1511
+ // before by this method (and the cache does not hold it): not
1512
+ // sent again (lib/content-refusals.js).
1513
+ const pageHold = holds.page(body);
1514
+ if (pageHold === 'held') {
1515
+ const err = new Error('held');
1516
+ err.heldNames = [PAGE_NAME];
1517
+ throw err;
1518
+ }
1519
+ await requirePairReady(pairKey, pairConfig, code, 'body translation');
1520
+ const runPage = (cfg) => translateRawContent(
1521
+ buildContentPrompt(protectedBody, cfg, promptOptions), { apiKey, pairConfig: cfg });
1522
+ let bodyResult;
1523
+ if (pairConfig.fallback) {
1524
+ // When both fail, the primary's page goes to the checks
1525
+ // below, which fail the file exactly as without a fallback.
1526
+ // Refused before by the pair's method: only the fallback is asked.
1527
+ const page = await translatePageWithFallback({
1528
+ body, blocks, pairConfig, runPage, budget: fallbackBudget, fallbackOnly: pageHold === 'fallback-only',
1529
+ });
1530
+ tallyContent(pairKey, page.report, label);
1531
+ if (page.refusedBy.length > 0) {
1532
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: page.refusedBy });
1533
+ }
1534
+ if (pageHold === 'fallback-only' && page.body === null) {
1535
+ // The fallback did not translate it: the pair's method
1536
+ // refused it before, so it is not asked — refused now
1537
+ // by the fallback too, or held (no answer, or skipped
1538
+ // by --max-cost).
1539
+ if (page.refusedBy.length > 0) {
1540
+ const err = new Error(
1541
+ `Docusaurus body for ${code}: the fallback (${pairConfig.fallback.method}) failed the whole-page checks too.\n` +
1542
+ ` Nothing was written or cached. ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`
1543
+ );
1544
+ err.heldNext = true;
1545
+ throw err;
1546
+ }
1547
+ const err = new Error('held');
1548
+ err.heldNames = [PAGE_NAME];
1549
+ throw err;
1550
+ }
1551
+ bodyResult = page.body ?? page.primaryBody;
1552
+ if (page.body !== null) wholeBodyKey = page.tmKey;
1553
+ else if (page.refusedBy.length > 0) pageRefusedBy = page.refusedBy;
1554
+ } else {
1555
+ const raw = await runPage(pairConfig);
1556
+ bodyResult = raw ? restoreBlocks(raw, blocks) : null;
1557
+ if (bodyResult) pageRefusedBy = [tmKey];
1047
1558
  }
1048
- const prompt = buildContentPrompt(protectedBody, pairConfig, promptOptions);
1049
- const bodyResult = await translateRawContent(prompt, { apiKey, pairConfig });
1050
1559
  if (!bodyResult) {
1051
1560
  throw new Error(
1052
- `Docusaurus body for ${code}: translation returned no results.\n` +
1561
+ `Docusaurus body for ${code}: translation returned no results`
1562
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
1053
1563
  ' Check your API key and method configuration.'
1054
1564
  );
1055
1565
  }
1056
- translatedBody = restoreBlocks(bodyResult, blocks);
1566
+ translatedBody = bodyResult;
1057
1567
  } else {
1058
1568
  // Block mode: segment the PROTECTED body (placeholders are
1059
1569
  // single tokens, so they can never be split), serve blocks
@@ -1073,72 +1583,140 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1073
1583
  }));
1074
1584
 
1075
1585
  const missed = [];
1586
+ const heldBlocks = []; // refused before: not sent, '[EN] ' kept
1587
+ let position = 0; // place among the translatable blocks
1076
1588
  for (const r of rendered) {
1077
1589
  if (r.seg.type !== 'translatable') {
1078
1590
  // Separators + passthrough blocks: copied verbatim, never billed.
1079
1591
  r.out = r.source;
1080
1592
  continue;
1081
1593
  }
1082
- const cached = lookupTMValidated(tm, r.source, code, tmKey,
1083
- (src, c) => !checkContentPreservation(src, c));
1594
+ // Its name in the repeat check (content:<file>#<n>).
1595
+ r.pos = position++;
1596
+ const cached = lookupContentTM(tm, r.source, code, pairConfig);
1084
1597
  if (cached !== null) {
1085
- r.out = cached;
1086
- } else {
1087
- missed.push(r);
1598
+ r.out = cached.text;
1599
+ if (cached.fromFallback) { mixedProducers = true; notePage(pairKey, label); }
1600
+ refusal.filled.add(blockUnit(r.source));
1601
+ continue;
1602
+ }
1603
+ const hold = holds.block(r.source);
1604
+ if (hold === 'held') {
1605
+ // Refused before by this method (and its fallback): the
1606
+ // honest last resort stays, nothing is sent or billed.
1607
+ r.out = restoreBlocks(config.fallbackPrefix + r.seg.text, blocks);
1608
+ heldBlocks.push(r);
1609
+ continue;
1088
1610
  }
1611
+ r.fallbackOnly = hold === 'fallback-only';
1612
+ missed.push(r);
1613
+ }
1614
+ if (heldBlocks.length > 0) {
1615
+ bodyUsedFallback = true;
1616
+ const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
1617
+ refusal.held.push(...names);
1618
+ output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names, fallbackPrefix: config.fallbackPrefix }));
1089
1619
  }
1090
1620
 
1091
1621
  if (missed.length > 0) {
1092
- if (!apiKey) {
1093
- throw new Error(
1094
- `Docusaurus body translation for ${code}: no API key available.\n` +
1095
- ' Set OPENROUTER_API_KEY in .env.local to translate content.'
1096
- );
1097
- }
1622
+ await requirePairReady(pairKey, pairConfig, code, 'body translation');
1098
1623
  // Self-repair ladder (translateBlockBatchResilient): full
1099
- // batch → one missing-segments-only retry → honest
1624
+ // batch → one missing-segments-only retry → (with a
1625
+ // fallback method: one batch through it for every block the
1626
+ // primary dropped or damaged — lib/fallback.js) → honest
1100
1627
  // '[EN] '-prefixed source for anything still missing. A
1101
1628
  // duplicate/unknown marker or an empty first response
1102
- // still fails the file whole.
1103
- const { blocks: translatedBlocks, fellBack } =
1104
- await translateBlockBatchResilient({
1105
- texts: missed.map(r => r.seg.text),
1106
- buildPrompt: (texts) => buildBlockBatchPrompt(
1107
- texts, pairConfig, { ...promptOptions, pageTitle }),
1108
- callModel: (prompt) => translateRawContent(prompt, { apiKey, pairConfig }),
1629
+ // still fails the file whole when no fallback rescues it.
1630
+ const outcome = await translateBlocksWithFallback({
1631
+ missed,
1632
+ blocks,
1633
+ pairConfig,
1634
+ fallbackPrefix: config.fallbackPrefix,
1635
+ budget: fallbackBudget,
1636
+ sharedOutputs: sharedOutputsFor(code),
1637
+ label: docLabel,
1638
+ runBatch: (texts, cfg) => translateBlockBatchResilient({
1639
+ texts,
1640
+ buildPrompt: (t) => buildBlockBatchPrompt(t, cfg, { ...promptOptions, pageTitle }),
1641
+ callModel: (prompt) => translateRawContent(prompt, { apiKey, pairConfig: cfg }),
1109
1642
  fallbackPrefix: config.fallbackPrefix,
1110
- });
1111
- const fellBackSet = new Set(fellBack);
1643
+ }),
1644
+ fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
1645
+ });
1646
+ tallyContent(pairKey, outcome.report, label);
1647
+ // Refused blocks are remembered; translated ones drop a record.
1648
+ const fellSet = new Set(outcome.fellBack);
1649
+ const newlyHeld = [];
1650
+ missed.forEach((r, i) => {
1651
+ if (!fellSet.has(i)) { refusal.filled.add(blockUnit(r.source)); return; }
1652
+ const methods = outcome.refusedBy?.[i] || [];
1653
+ if (methods.length > 0) {
1654
+ refusal.refused.push({ unit: blockUnit(r.source), source: r.source, methods });
1655
+ newlyHeld.push(i);
1656
+ }
1657
+ });
1658
+ if (outcome.fromFallback > 0) mixedProducers = true;
1659
+ if ((outcome.sharedOutput || []).length > 0) {
1660
+ output.warn(
1661
+ `Docusaurus body for ${code}: ${outcome.sharedOutput.length} block(s) of ${dirName}/${relPath} came back as the same text ` +
1662
+ 'the model gave for other, different source strings (a memorized sentence, not a translation) — refused.'
1663
+ );
1664
+ }
1665
+ if ((outcome.refused || []).length > 0) {
1666
+ output.warn(
1667
+ `Docusaurus body for ${code}: ${outcome.refused.length} block(s) of ${dirName}/${relPath} refused by the quality gate — `
1668
+ + outcome.refused.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
1669
+ + `${outcome.refused.length > 3 ? '; …' : ''}.`
1670
+ );
1671
+ }
1672
+ const { fellBack } = outcome;
1112
1673
  if (fellBack.length > 0) {
1113
1674
  bodyUsedFallback = true;
1675
+ const refusedHere = (outcome.sharedOutput || []).length + (outcome.refused || []).length;
1676
+ const why = outcome.report?.attempted > 0
1677
+ ? `neither the primary (${pairConfig.method}) nor its fallback (${pairConfig.fallback.method}) translated safely`
1678
+ : refusedHere >= fellBack.length ? 'refused (above)'
1679
+ : refusedHere > 0 ? `refused (${refusedHere}, above) or missing from the model response after a retry`
1680
+ : 'missing from the model response after a retry';
1114
1681
  output.warn(
1115
1682
  `Docusaurus body for ${code}: ${fellBack.length} of ${missed.length} ` +
1116
- `block(s) missing from the model response after a retry — written as ` +
1117
- `'${config.fallbackPrefix}'-prefixed source. Not cached, lock not ` +
1118
- `advanced: the next sync retries just those block(s).`
1683
+ `block(s) ${why} — written as ` +
1684
+ `'${config.fallbackPrefix}'-prefixed source. ` +
1685
+ describeFallenBack({
1686
+ file: label, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: outcome.refusedBy,
1687
+ })
1119
1688
  );
1120
1689
  }
1121
- missed.forEach((r, i) => {
1122
- r.out = restoreBlocks(translatedBlocks[i], blocks);
1123
- // Fallen-back segments are never TM-cached — an error
1124
- // cached is an error forever (they re-bill next sync).
1125
- if (!fellBackSet.has(i)) {
1126
- pendingBlockStores.push({ source: r.source, translation: r.out });
1127
- }
1128
- });
1690
+ // Fallen-back segments are never TM-cached — an error
1691
+ // cached is an error forever (they re-bill next sync).
1692
+ missed.forEach((r, i) => { r.out = outcome.outs[i]; });
1693
+ pendingBlockStores.push(...outcome.stores);
1129
1694
  }
1130
1695
 
1131
1696
  // Reassemble in order with the source's exact separators.
1132
1697
  translatedBody = rendered.map(r => r.out).join('');
1133
1698
  }
1134
1699
 
1700
+ // A page translated whole that fails the checks below is
1701
+ // remembered as refused: the next plain sync does not send it
1702
+ // to the same method again (lib/content-refusals.js "page").
1703
+ const refusePage = (err) => {
1704
+ if (!pageRefusedBy) return err;
1705
+ if (!refusal.refused.some(r => r.unit === PAGE_UNIT)) {
1706
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: pageRefusedBy });
1707
+ }
1708
+ err.message += `\n ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`;
1709
+ err.heldNext = true;
1710
+ return err;
1711
+ };
1712
+
1135
1713
  // Orphaned-placeholder check on the REASSEMBLED body — the
1136
1714
  // same gate for both modes.
1137
1715
  if (hasOrphanedPlaceholders(translatedBody)) {
1138
- throw new Error(
1716
+ throw refusePage(new Error(
1139
1717
  `Docusaurus body for ${code}: placeholder corruption detected.\n` +
1140
1718
  ' Code blocks were corrupted during translation.'
1141
- );
1719
+ ));
1142
1720
  }
1143
1721
 
1144
1722
  // Content-preservation check on the REASSEMBLED body — same
@@ -1147,22 +1725,23 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1147
1725
  // source text and is neither cached nor lock-advanced already.
1148
1726
  const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
1149
1727
  if (bodyHollowed) {
1150
- throw new Error(
1728
+ throw refusePage(new Error(
1151
1729
  `Docusaurus body for ${code}: ${bodyHollowed.reason}.\n` +
1152
1730
  ' Nothing was written or cached. If this is a low-coverage target\n' +
1153
1731
  ' language, the model has no vocabulary for this text.'
1154
- );
1732
+ ));
1155
1733
  }
1734
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
1156
1735
 
1157
1736
  // Store per-block AND whole-body entries only after the check
1158
1737
  // passes (whole-body makes reverts/lock-loss re-runs free). A
1159
1738
  // fallback body is NEVER stored whole — it contains
1160
1739
  // untranslated '[EN] ' text.
1161
1740
  for (const s of pendingBlockStores) {
1162
- storeTM(tm, s.source, code, tmKey, s.translation);
1741
+ storeTM(tm, s.source, code, s.tmKey, s.translation);
1163
1742
  }
1164
- if (!bodyUsedFallback) {
1165
- storeTM(tm, body, code, tmKey, translatedBody);
1743
+ if (!bodyUsedFallback && !mixedProducers) {
1744
+ storeTM(tm, body, code, wholeBodyKey, translatedBody);
1166
1745
  }
1167
1746
  }
1168
1747
  }
@@ -1186,10 +1765,44 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1186
1765
  }
1187
1766
 
1188
1767
  } catch (contentErr) {
1768
+ if (contentErr.heldNames) {
1769
+ // A front-matter field refused before: held back, not failed —
1770
+ // nothing was sent, nothing written, the lock not advanced.
1771
+ const refusal = refusalStates.get(manifestKey);
1772
+ refusal.held.push(...contentErr.heldNames);
1773
+ refusal.pageHeld = true;
1774
+ settleRefusals();
1775
+ output.warn(describeHeld({
1776
+ file: `${dirName}/${relPath}`, code, pairKey, pairConfig, names: contentErr.heldNames, pageHeld: true,
1777
+ fallbackPrefix: config.fallbackPrefix,
1778
+ }));
1779
+ completed++;
1780
+ output.raw(` [${completed}/${totalWork}] ${dirName}/${relPath} → ${code} [HELD]`);
1781
+ output.event('file', { lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: 'held-refused', held: contentErr.heldNames.length, action });
1782
+ if (completed % MANIFEST_WRITE_INTERVAL === 0) writeManifestIfDirty();
1783
+ return;
1784
+ }
1785
+ itemFailed = true;
1189
1786
  contentFailures++;
1190
1787
  output.error(`${dirName}/${relPath} → ${code} — ${contentErr.message}`);
1788
+ // Nothing was written for this item, and its lock entry was not
1789
+ // advanced. If the lock already matches this source, the file on
1790
+ // disk is the PREVIOUS translation of the same text (a forced
1791
+ // re-translate that failed): it stays, and a plain sync will not
1792
+ // redo it. Otherwise the next sync retries it.
1793
+ const keptPrevious = updatedDocuManifest[manifestKey] === sourceHash && fs.existsSync(targetPath);
1794
+ failedItems.push({
1795
+ file: `${dirName}/${relPath}`,
1796
+ locale: code,
1797
+ error: contentErr.message,
1798
+ // held-back: refused by the quality gate — the next plain sync
1799
+ // does not send it again (lib/content-refusals.js).
1800
+ state: keptPrevious ? 'previous-translation-kept' : contentErr.heldNext ? 'held-back' : 'will-retry',
1801
+ });
1191
1802
  }
1192
1803
 
1804
+ settleRefusals();
1805
+
1193
1806
  // Progress reporting
1194
1807
  completed++;
1195
1808
  const pct = Math.round(100 * completed / totalWork);
@@ -1198,13 +1811,16 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1198
1811
  const remainingMs = msPerItem * (totalWork - completed);
1199
1812
  const remainingSec = Math.ceil(remainingMs / 1000);
1200
1813
  const etaStr = remainingSec > 5 ? ` (~${remainingSec}s left)` : '';
1201
- const tag = contentFailures > 0 && completed === totalWork
1814
+ // The tag describes THIS item. (It used to read the run-wide failure
1815
+ // count, so whichever item finished last was tagged FAIL.)
1816
+ const displayTag = itemFailed
1202
1817
  ? 'FAIL'
1203
1818
  : action === 're-translate' ? 'RE-TRANSLATE' : action === 'changed' ? 'CHANGED' : 'OK';
1204
- // Show [FAIL] for items that just errored (contentErr was caught above)
1205
- const itemFailed = updatedDocuManifest[manifestKey] !== sourceHash;
1206
- const displayTag = itemFailed && !dryRun ? 'FAIL' : tag;
1207
1819
  output.raw(` [${completed}/${totalWork}] (${pct}%) ${dirName}/${relPath} → ${code} [${displayTag}]${etaStr}`);
1820
+ output.event('file', {
1821
+ lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code,
1822
+ status: itemFailed ? 'failed' : dryRun ? 'would-translate' : 'translated', action,
1823
+ });
1208
1824
 
1209
1825
  // Incremental manifest write
1210
1826
  if (completed % MANIFEST_WRITE_INTERVAL === 0) {
@@ -1231,26 +1847,122 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1231
1847
  output.ok(`${action} ${totalContent} content file(s)${retranslateNote}, ${totalContentSkipped} unchanged`);
1232
1848
  }
1233
1849
 
1234
- // Fail loud if any content translations failed — do NOT exit 0
1235
- if (contentFailures > 0) {
1236
- throw new Error(
1237
- `${contentFailures} content translation(s) failed. ` +
1238
- `Re-run sync to retry failed files (completed files are cached).`
1850
+ contentTranslated = totalContent;
1851
+ contentFailedItems = failedItems;
1852
+ for (const st of refusalStates.values()) {
1853
+ if (st.held.length > 0) {
1854
+ contentHeldBack += st.held.length;
1855
+ contentHeldBackItems.push({ file: st.file, locale: st.locale, units: st.held, ...(st.pageHeld && { pageNotWritten: true }) });
1856
+ }
1857
+ contentRefused += new Set(st.refused.map(r => r.unit)).size;
1858
+ }
1859
+ if (contentHeldBack > 0) {
1860
+ output.warn(
1861
+ `${contentHeldBack} content block(s)/field(s) held back in ${contentHeldBackItems.length} translation(s) — the quality gate refused `
1862
+ + 'the method\'s translation of their current text before; not sent, not billed (listed above). Ask again for all of them: '
1863
+ + '`champollion sync --redo content`; for one page: `champollion sync --redo files:<page>`.'
1239
1864
  );
1240
1865
  }
1866
+ for (const [pairKey, tally] of contentFallbackTallies) {
1867
+ printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)', producedLabel: 'pages' });
1868
+ warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)' });
1869
+ }
1870
+ if (failedItems.length > 0) {
1871
+ output.raw('');
1872
+ output.raw(` Failed content translations (${failedItems.length}):`);
1873
+ for (const f of failedItems.sort((a, b) => (a.file + a.locale).localeCompare(b.file + b.locale))) {
1874
+ const state = f.state === 'previous-translation-kept'
1875
+ ? 'previous translation kept (it matches the current source); re-run with --force-content to try again'
1876
+ : f.state === 'held-back'
1877
+ ? `refused by the quality gate — held back: the next sync does not send it again (\`--redo files:${f.file}\` asks again)`
1878
+ : 'not recorded as done — the next sync retries it';
1879
+ output.raw(` ${f.file} → ${f.locale}: ${state}`);
1880
+ }
1881
+ }
1241
1882
  }
1242
1883
 
1243
1884
  // Save TM if it was mutated during this Docusaurus sync (stores OR
1244
1885
  // evictions — a size check would miss eviction-only runs and same-key
1245
- // replacements). Skip when --no-tm is active.
1886
+ // replacements). Skip when --no-tm is active. Saved BEFORE any failure is
1887
+ // raised: the files that succeeded paid for their translations, and
1888
+ // throwing first discarded every one of them from the cache.
1246
1889
  if (!dryRun && !noTM && isTMDirty(tm)) {
1247
- const tmFinalSize = tmSize(tm);
1248
- const delta = tmFinalSize - tmInitialSize;
1249
1890
  saveTM(cwd, tm);
1250
- output.info(`[TM] Saved ${tmFinalSize} entries (${delta >= 0 ? '+' + delta : delta} this sync)`);
1891
+ output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
1892
+ }
1893
+
1894
+ // Failures are reported, not thrown: a partly failed run still did real
1895
+ // work, and the exit code says so (2 = partial, 1 = nothing succeeded).
1896
+ if (contentFailures > 0) {
1897
+ output.error(
1898
+ `${contentFailures} content translation(s) failed (listed above). ` +
1899
+ 'Completed files and their translations are saved; re-run sync to retry.'
1900
+ );
1251
1901
  }
1252
1902
 
1903
+ // The --max-cost verdict, said once — at the end, before the summary
1904
+ // (lib/cost-report.js reportDryRunMaxCost).
1905
+ if (dryMaxCost) output[dryMaxCost.level](dryMaxCost.message);
1906
+
1907
+ // One machine-readable record of the run — the Docusaurus path never had
1908
+ // one, so --json consumers saw nothing to tell success from failure.
1909
+ output.summary({
1910
+ command: 'sync',
1911
+ format: 'docusaurus',
1912
+ dryRun,
1913
+ totalProcessed: totalJSONKeys,
1914
+ totalFailed: totalJSONFailed,
1915
+ totalCopied: totalJSONCopied,
1916
+ // UI strings held back: the gate refused this method's translation of
1917
+ // their current text before — not sent, not billed (lib/locale-state.js).
1918
+ // A dry run sends nothing either way: said above, not counted (as on
1919
+ // the key-value path).
1920
+ totalHeld: dryRun ? 0 : totalJSONHeld,
1921
+ // What each pair's fallback method did for JSON keys (pairs with a
1922
+ // fallback, real runs); Markdown segments are under content.fallback.
1923
+ ...(keyFallbackTallies.size > 0 && {
1924
+ fallback: [...keyFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
1925
+ }),
1926
+ content: {
1927
+ translated: contentTranslated,
1928
+ skipped: contentScan.totalContentSkipped,
1929
+ failed: contentFailedItems.length,
1930
+ failedItems: contentFailedItems,
1931
+ // Refused before by the method, not sent this run (lib/content-refusals.js),
1932
+ // and refused this run (held back from the next).
1933
+ heldBack: contentHeldBack,
1934
+ heldBackItems: contentHeldBackItems,
1935
+ refused: contentRefused,
1936
+ ...(contentFallbackTallies.size > 0 && {
1937
+ fallback: [...contentFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
1938
+ }),
1939
+ },
1940
+ costEstimate,
1941
+ // Dry runs with --max-cost: whether the real run would stop at the cap.
1942
+ ...(dryMaxCost && { maxCost: {
1943
+ cap: dryMaxCost.cap, estimatedCost: dryMaxCost.estimatedCost, wouldStop: dryMaxCost.wouldStop,
1944
+ ...(dryMaxCost.wouldStop && { exitCode: 2, reason: dryMaxCost.reason }),
1945
+ ...(dryMaxCost.stopsEarlier && { exitCode: 1, stopsEarlier: dryMaxCost.stopsEarlier }),
1946
+ } }),
1947
+ // Keys named for a redo that match no UI string (the run exits 1), each
1948
+ // with the closest ids that exist.
1949
+ ...(unmatchedNamed.length > 0 && { unmatchedKeys: unmatchedKeysSummary(unmatchedNamed) }),
1950
+ });
1253
1951
  output.raw('');
1952
+ // Named keys that matched nothing: said again last, where a reader (or CI)
1953
+ // looks for the verdict — and the run fails (lib/commands/sync.js).
1954
+ reportUnmatchedKeys(unmatchedNamed, cliArgs);
1955
+ return {
1956
+ totalProcessed: totalJSONKeys,
1957
+ totalFailed: totalJSONFailed,
1958
+ totalCopied: totalJSONCopied,
1959
+ totalHeld: dryRun ? 0 : totalJSONHeld,
1960
+ contentTranslated,
1961
+ contentFailed: contentFailedItems.length,
1962
+ contentHeldBack,
1963
+ contentRefused,
1964
+ unmatchedKeys: unmatchedNamed.map(m => m.name),
1965
+ };
1254
1966
  }
1255
1967
 
1256
1968
  export { runDocusaurusSync, discoverDocusaurusJSONFiles };