champollion 0.3.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +82 -37
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1709 -101
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +230 -17
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +284 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1136 -345
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +727 -79
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +975 -262
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +1102 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +575 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +257 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +71 -129
  70. package/lib/methods/llm.js +87 -33
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +102 -55
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +24 -8
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +996 -21
  117. package/lib/verify.js +1332 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/licenses.json +99 -3
  126. package/shared/method-registry.json +2 -0
  127. package/shared/metric-registry.json +96 -18
  128. package/shared/retired-model-aliases.json +9 -0
  129. package/shared/schemas/champollion-plugin.schema.json +4 -0
  130. package/shared/schemas/corpora-card.schema.json +8 -2
  131. package/shared/schemas/human-services.schema.json +1 -1
  132. package/shared/schemas/method-index-record.schema.json +67 -0
  133. package/shared/schemas/method-registry.schema.json +4 -0
  134. package/shared/schemas/metric-registry.schema.json +55 -1
  135. package/shared/schemas/retired-model-aliases.schema.json +32 -0
  136. package/shared/docent/corpus.json +0 -11739
  137. package/shared/model-aliases.json +0 -7
  138. package/shared/schemas/model-aliases.schema.json +0 -27
@@ -29,29 +29,54 @@ import {
29
29
  discoverDocusaurusContentFiles, getDocusaurusTargetPath,
30
30
  findUntranslatableNestedFields,
31
31
  } from './content.js';
32
- import { translateRawContent } from './translate.js';
32
+ import { translateRawContent, getMethod } from './translate.js';
33
33
  import {
34
34
  splitBlocks, buildBlockBatchPrompt, parseBlockBatchResponse,
35
35
  translateBlockBatchResilient,
36
36
  assertSegmentationMode,
37
37
  } from './segment.js';
38
38
  import { DEFAULT_REGISTERS } from './registers.js';
39
- import { DEFAULT_JSON_CONCURRENCY, EST_CHARS_PER_KEY } from './config.js';
39
+ import { DEFAULT_JSON_CONCURRENCY } from './config.js';
40
40
  import { compileNoTranslate } from './no-translate.js';
41
- import { checkContentPreservation } from './validate.js';
41
+ import { checkContentPreservation, contentGateFault, SharedOutputIndex } from './validate.js';
42
42
  import { convertScript, applyScriptFallback } from './scripts.js';
43
43
  import { pMap } from './concurrent.js';
44
44
  import {
45
45
  loadTM, saveTM, tmSize, isTMDirty,
46
- lookupTM, lookupTMValidated, storeTM, evictTM, partitionByTM, tmMethodKey,
46
+ lookupTM, peekTM, storeTM, evictTM, partitionByTM, tmMethodKey, setModelCarryover, bypassTMFor,
47
+ describeTMChanges, adoptLegacyCoachingKeys,
47
48
  } from './tm.js';
48
49
  import {
49
- readManifest, writeManifest, detectChangedKeys, hashValue,
50
+ readLock, writeManifest, detectChangedKeys, hashValue,
50
51
  } from './hash.js';
52
+ import {
53
+ LockState, planQueue, recordRefusal, describeHeldKeys, describeFallbackOnlyKeys, keyFateNote, describeHeldNext,
54
+ shortSourceHash,
55
+ } from './locale-state.js';
56
+ import { redoCommand } from './verify.js';
51
57
  import { CONTENT_LOCK_FILENAME } from './content-sync.js';
52
- import { parseMaxCost, abortForMaxCost, printCostTable } from './cost-report.js';
58
+ import {
59
+ parseMaxCost, abortForMaxCost, maxCostVerdict, reportDryRunMaxCost, printCostTable, warnModelSwitchStrandedTM, priceContentChars,
60
+ preflightStopReason,
61
+ summarizeEstimate,
62
+ } from './cost-report.js';
63
+ import { billableContentChars, translatableBlockSources } from './content-estimate.js';
64
+ import { compileFileScope } from './file-scope.js';
53
65
  import { output } from './output.js';
54
- import { translateAndValidate } from './translate-pair.js';
66
+ import { missingKeyAdvice } from './missing-key.js';
67
+ import { translateWithFallback } from './translate-pair.js';
68
+ import {
69
+ tmKeysForPair, tmHoldsValue, createFallbackBudget, newFallbackReport, addToTally,
70
+ fallbackSummary, printFallbackReport, warnFallbackMajority, lookupContentTM, serveFieldsFromFallbackCache,
71
+ translateFieldsWithFallback, translateBlocksWithFallback, translatePageWithFallback,
72
+ refuseRepeatedCachedPage,
73
+ } from './fallback.js';
74
+ import { walkFiles } from './locale-layout.js';
75
+ import {
76
+ readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
77
+ describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock,
78
+ } from './content-refusals.js';
79
+ import { applyNamedKeyRule, reportUnmatchedKeys, unmatchedKeysSummary, reportNamedFromCache } from './named-keys.js';
55
80
 
56
81
  /**
57
82
  * Discover all JSON locale files in a Docusaurus i18n source directory.
@@ -63,20 +88,64 @@ import { translateAndValidate } from './translate-pair.js';
63
88
  * @returns {string[]} Absolute paths to JSON files
64
89
  */
65
90
  function discoverDocusaurusJSONFiles(sourceLocaleDir) {
66
- const files = [];
67
- function walk(dir) {
68
- if (!fs.existsSync(dir)) return;
69
- for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
70
- const fullPath = path.join(dir, entry.name);
71
- if (entry.isDirectory()) {
72
- walk(fullPath);
73
- } else if (entry.isFile() && entry.name.endsWith('.json')) {
74
- files.push(fullPath);
75
- }
76
- }
91
+ // The shared folder-per-locale walk (lib/locale-layout.js). Docusaurus
92
+ // keeps its historical reach: hidden entries are NOT skipped here.
93
+ return walkFiles(sourceLocaleDir, name => name.endsWith('.json'), { skipHidden: false });
94
+ }
95
+
96
+ /**
97
+ * What one (UI-string file × locale) queue becomes once the lock's refusal
98
+ * records are consulted — the key-value lane's rule (lib/locale-state.js
99
+ * planQueue), shared by the estimate and the sync so the estimate prices
100
+ * exactly what the run sends. A key the gate refused for its current source
101
+ * text is held back from the method that refused it (its fallback, when it
102
+ * has not refused it, is asked instead) unless it is named for a redo
103
+ * (`--redo keys:`), queued by `--redo all`, or the run is `--fresh`.
104
+ *
105
+ * Docusaurus records no written values, so no value counts as a person's
106
+ * edit here (a bulk redo re-translates every UI string, as it always did),
107
+ * and there is no pending retry: a UI string refused under a redo is held
108
+ * back by the next plain sync, as a content block is.
109
+ *
110
+ * @param {object} p
111
+ * @param {object} p.diff - diffLocale result for the file × locale
112
+ * @param {object} p.sourceFlat
113
+ * @param {object} p.targetFlat
114
+ * @param {string} p.nsPrefix - "docusaurus:<relPath>:" (the lock's key space)
115
+ * @param {object} p.localeState - LockState.of(code) / .peek(code)
116
+ * @param {object} p.pairConfig
117
+ * @param {{ named: Set<string>, bulk: boolean, fresh: boolean }} p.redo
118
+ * @param {string} p.fallbackPrefix
119
+ * @returns {{ held: string[], heldFromPrimary: string[] }}
120
+ */
121
+ function planUIStrings({ diff, sourceFlat, targetFlat, nsPrefix, localeState, pairConfig, redo, fallbackPrefix }) {
122
+ const plan = planQueue({
123
+ diff, sourceFlat, targetFlat, lockKeyOf: (k) => nsPrefix + k, localeState,
124
+ named: redo.named, bulk: redo.bulk, pending: new Set(), fresh: redo.fresh, pairConfig,
125
+ classify: () => 'machine',
126
+ fallbackPrefix,
127
+ });
128
+ return { held: plan.held, heldFromPrimary: plan.heldFromPrimary };
129
+ }
130
+
131
+ /**
132
+ * UI strings refused for an older source text that the file still holds as
133
+ * the English copy of that text (Docusaurus needs every id, so an untranslated
134
+ * one is written as its source message): the source was edited since, which
135
+ * lifts the hold — queued as changed. Without this the edit was invisible: the
136
+ * refused string never had its source hash recorded, and the old English no
137
+ * longer equals the new source, so nothing queued it.
138
+ *
139
+ * @returns {string[]}
140
+ */
141
+ function editedSinceRefused(sourceFlat, targetFlat, nsPrefix, localeState) {
142
+ const out = [];
143
+ for (const [k, src] of Object.entries(sourceFlat)) {
144
+ const rec = localeState.refused[nsPrefix + k];
145
+ if (!rec || typeof src !== 'string' || typeof targetFlat[k] !== 'string') continue;
146
+ if (rec.source !== shortSourceHash(src) && shortSourceHash(targetFlat[k]) === rec.source) out.push(k);
77
147
  }
78
- walk(sourceLocaleDir);
79
- return files.sort();
148
+ return out;
80
149
  }
81
150
 
82
151
  /**
@@ -97,12 +166,13 @@ function discoverDocusaurusJSONFiles(sourceLocaleDir) {
97
166
  * @param {object} config - Resolved config (localesDir)
98
167
  * @param {object} manifest - Content lock manifest (hash per file × locale)
99
168
  * @param {boolean} forceContent - --force-content: re-process up-to-date files
169
+ * @param {import('./file-scope.js').FileScope|null} [fileScope] - --files / --retranslate
100
170
  * too (action 're-translate'), WITHOUT clearing the manifest — a target
101
171
  * with no lock entry and no '[EN] ' markers is a genuine hand-translated
102
172
  * file, and force must never overwrite human work with machine output.
103
173
  * @returns {{ workItems: Array<object>, totalContentSkipped: number, recordedHashes: object }}
104
174
  */
105
- function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent) {
175
+ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent, fileScope = null) {
106
176
  const workItems = [];
107
177
  let totalContentSkipped = 0;
108
178
  const recordedHashes = {};
@@ -115,6 +185,9 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
115
185
 
116
186
  for (const sourcePath of sourceFiles) {
117
187
  const relPath = path.relative(sourceContentDir, sourcePath);
188
+ const label = `${dirName}/${relPath}`;
189
+ if (fileScope && !fileScope.includes(label)) continue;
190
+ const retranslate = fileScope ? fileScope.retranslates(label) : false;
118
191
 
119
192
  // Read + parse source file once per source path. The staleness hash
120
193
  // covers the RAW file (same as the Hugo twin's hashFileContent): a
@@ -142,7 +215,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
142
215
  }
143
216
  const { parsed, fieldsToTranslate, pageTitle, sourceHash } = sourceCache.get(sourcePath);
144
217
 
145
- for (const [, pairConfig] of pairEntries) {
218
+ for (const [pairKey, pairConfig] of pairEntries) {
146
219
  const code = pairConfig.target;
147
220
  const targetPath = getDocusaurusTargetPath(
148
221
  sourcePath, sourceContentDir, code, config.localesDir, pluginName
@@ -156,7 +229,11 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
156
229
  const manifestKey = `docusaurus:${dirName}/${relPath}:${code}`;
157
230
  let action = 'new'; // 'new' | 'changed' | 're-translate'
158
231
 
159
- if (fs.existsSync(targetPath)) {
232
+ if (retranslate) {
233
+ // --retranslate: named by the operator — no lock skip, no
234
+ // adoption; the caller makes its text miss the TM.
235
+ if (fs.existsSync(targetPath)) action = 're-translate';
236
+ } else if (fs.existsSync(targetPath)) {
160
237
  const storedHash = manifest[manifestKey];
161
238
  if (storedHash) {
162
239
  if (storedHash === sourceHash) {
@@ -171,7 +248,9 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
171
248
  action = 'changed';
172
249
  }
173
250
  } else {
174
- // No stored hash — check for [EN] fallback markers. This runs
251
+ // No stored hash — check for legacy [EN] fallback markers (pages
252
+ // written before 2026-10-05; now such a page has a pending:<hash>
253
+ // lock entry instead). This runs
175
254
  // even under --force-content: a target with no lock entry and
176
255
  // no [EN] markers is a genuine hand-translated file, and force
177
256
  // must never overwrite human work with machine output.
@@ -190,7 +269,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
190
269
  workItems.push({
191
270
  sourcePath, parsed, fieldsToTranslate, pageTitle, sourceHash,
192
271
  relPath, dirName,
193
- pairConfig, code, targetPath, manifestKey, pluginName, action,
272
+ pairKey, pairConfig, code, targetPath, manifestKey, pluginName, action, retranslate,
194
273
  });
195
274
  }
196
275
  }
@@ -240,7 +319,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
240
319
  * Structured estimate, or null when estimation itself failed (callers
241
320
  * with --max-cost must fail safe: unknown ≠ free)
242
321
  */
243
- async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate) {
322
+ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate, lockState = null, redo = null) {
244
323
  try {
245
324
  const { estimateCost } = await import('./pairs.js');
246
325
  const costEstimates = [];
@@ -251,6 +330,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
251
330
  const sourceJSONFiles = discoverDocusaurusJSONFiles(sourceLocaleDir);
252
331
  const missesByPair = new Map(); // pairKey → miss count
253
332
  const hitsByPair = new Map(); // pairKey → TM hit count
333
+ const heldByPair = new Map(); // pairKey → keys held back (refused before; not sent)
254
334
 
255
335
  for (const sourceFilePath of sourceJSONFiles) {
256
336
  const relPath = path.relative(sourceLocaleDir, sourceFilePath);
@@ -283,26 +363,45 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
283
363
 
284
364
  // Same confirmed-echo suppression as the Phase-1 sync below, so the
285
365
  // estimate prices exactly the keys the run will actually queue.
286
- const estTmKey = tmMethodKey(pairConfig);
366
+ // With a fallback, its cached values count too (lib/fallback.js).
367
+ const estTmKeys = tmKeysForPair(pairConfig);
368
+ const localeChanged = lockState
369
+ ? [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, lockState.peek(code))])]
370
+ : changedKeys;
287
371
  const diff = diffLocale(
288
372
  sourceFlat, existingFlat, config.fallbackPrefix,
289
- config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, changedKeys,
290
- (key, sourceValue) => lookupTM(tm, sourceValue, code, estTmKey) === sourceValue,
373
+ config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
374
+ (key, sourceValue) => tmHoldsValue(tm, sourceValue, code, estTmKeys, sourceValue),
291
375
  noTranslate.active ? noTranslate.matches : null
292
376
  );
293
377
  const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
294
378
  if (stringKeys.length === 0) continue;
295
379
 
296
- const { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmMethodKey(pairConfig));
380
+ // Refused before (the lock's records): never sent to the method that
381
+ // refused them — held keys cost nothing; a fallback is not priced.
382
+ const notSent = new Set();
383
+ if (lockState && redo) {
384
+ const plan = planUIStrings({
385
+ diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState: lockState.peek(code), pairConfig, redo,
386
+ fallbackPrefix: config.fallbackPrefix,
387
+ });
388
+ for (const k of [...plan.held, ...plan.heldFromPrimary]) notSent.add(k);
389
+ }
390
+ let { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmMethodKey(pairConfig));
391
+ if (estTmKeys.length > 1) misses = misses.filter(k => lookupTM(tm, sourceFlat[k], code, estTmKeys[1]) === null);
392
+ const held = misses.filter(k => notSent.has(k)).length;
393
+ misses = misses.filter(k => !notSent.has(k));
297
394
  missesByPair.set(pairKey, (missesByPair.get(pairKey) || 0) + misses.length);
298
- hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length));
395
+ hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length - held));
396
+ heldByPair.set(pairKey, (heldByPair.get(pairKey) || 0) + held);
299
397
  }
300
398
  }
301
399
 
302
400
  for (const [pairKey, pairConfig] of pairEntries) {
303
401
  const keysToTranslate = missesByPair.get(pairKey) || 0;
304
402
  const tmHits = hitsByPair.get(pairKey) || 0;
305
- if (keysToTranslate === 0 && tmHits === 0) continue;
403
+ const heldCount = heldByPair.get(pairKey) || 0;
404
+ if (keysToTranslate === 0 && tmHits === 0 && heldCount === 0) continue;
306
405
 
307
406
  if (keysToTranslate > 0) {
308
407
  // eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
@@ -317,8 +416,15 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
317
416
  method: pairConfig.method || 'llm',
318
417
  keys: keysToTranslate,
319
418
  tmHits,
419
+ ...(heldCount > 0 && { held: heldCount }),
320
420
  estimatedCost: estimate.estimatedCost,
321
421
  source: estimate.source,
422
+ ...(estimate.local && { local: true }),
423
+ ...(estimate.note && { note: estimate.note }),
424
+ // The rate it was priced at, its source and date (cost-report.js rateLine).
425
+ ...(estimate.rate && { rate: estimate.rate }),
426
+ // No price: the model it was looked up under, so the table can name it.
427
+ ...(estimate.estimatedCost === null && estimate.model && { model: estimate.model }),
322
428
  });
323
429
  } else {
324
430
  // Fully TM-covered: zero API calls → a KNOWN $0, even for
@@ -328,8 +434,9 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
328
434
  method: pairConfig.method || 'llm',
329
435
  keys: 0,
330
436
  tmHits,
437
+ ...(heldCount > 0 && { held: heldCount }),
331
438
  estimatedCost: 0,
332
- source: 'translation-memory',
439
+ source: tmHits > 0 ? 'translation-memory' : 'nothing-to-send',
333
440
  });
334
441
  }
335
442
  }
@@ -342,72 +449,51 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
342
449
  // block sources once (only needed for 'block' mode misses).
343
450
  const blockSourcesCache = new Map(); // sourcePath → string[]
344
451
  const charsByPair = new Map(); // pairConfig (by reference) → billed source chars
452
+ const roughCharsByPair = new Map(); // pairConfig → all source chars, as if the TM were empty
345
453
  const pendingSourceFiles = new Set();
346
454
 
347
455
  for (const item of contentWorkItems) {
348
456
  pendingSourceFiles.add(item.sourcePath);
349
457
  const code = item.code;
350
- const tmKey = tmMethodKey(item.pairConfig);
351
- let chars = 0;
352
-
353
- // Front-matter fields: each field is cached on its own source
354
- // text — only TM misses reach the API.
355
- for (const text of Object.values(item.fieldsToTranslate)) {
356
- if (lookupTM(tm, text, code, tmKey) === null) chars += text.length;
357
- }
358
-
359
- // Body: whole-body TM first (a revert or lock-loss re-run is
360
- // free), then per-block in 'block' mode — the same ladder the
361
- // sync runs below.
362
- const body = item.parsed.body;
363
- if (body.trim() && lookupTM(tm, body, code, tmKey) === null) {
364
- const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
365
- if (segMode === 'page') {
366
- chars += body.length;
367
- } else {
458
+ const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
459
+ const { billedChars: chars, totalChars } = billableContentChars({
460
+ tm,
461
+ code,
462
+ tmKey: tmMethodKey(item.pairConfig),
463
+ fallbackTmKey: item.pairConfig.fallback ? tmMethodKey(item.pairConfig.fallback) : null,
464
+ fields: item.fieldsToTranslate,
465
+ body: item.parsed.body,
466
+ segMode,
467
+ blockSources: () => {
368
468
  if (!blockSourcesCache.has(item.sourcePath)) {
369
- const { protectedBody, blocks } = protectBlocks(body);
370
- blockSourcesCache.set(
371
- item.sourcePath,
372
- splitBlocks(protectedBody)
373
- .filter(seg => seg.type === 'translatable')
374
- .map(seg => restoreBlocks(seg.text, blocks))
375
- );
376
- }
377
- for (const source of blockSourcesCache.get(item.sourcePath)) {
378
- if (lookupTM(tm, source, code, tmKey) === null) chars += source.length;
469
+ blockSourcesCache.set(item.sourcePath, translatableBlockSources(item.parsed.body));
379
470
  }
380
- }
381
- }
471
+ return blockSourcesCache.get(item.sourcePath);
472
+ },
473
+ holds: item.holds || null,
474
+ });
475
+ roughCharsByPair.set(item.pairConfig, (roughCharsByPair.get(item.pairConfig) || 0) + totalChars);
382
476
 
383
477
  if (chars > 0) {
384
478
  charsByPair.set(item.pairConfig, (charsByPair.get(item.pairConfig) || 0) + chars);
385
479
  }
386
480
  }
387
481
 
388
- let contentCost = 0;
389
- let contentUnknown = false;
390
- for (const [, pairConfig] of pairEntries) {
391
- const chars = charsByPair.get(pairConfig);
392
- // Fully TM-covered pairs are a KNOWN $0 (zero API calls) — they
393
- // must not consult estimateCost, whose unknown-pricing null would
394
- // wrongly trip hasUnknownCosts and abort a free run under a cap.
395
- if (!chars) continue;
396
- const keyEquivalents = Math.ceil(chars / EST_CHARS_PER_KEY);
397
- // eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
398
- const estimate = await estimateCost(keyEquivalents, pairConfig);
399
- if (estimate.estimatedCost !== null) {
400
- contentCost += estimate.estimatedCost;
401
- } else {
402
- contentUnknown = true;
403
- }
404
- }
482
+ const billed = await priceContentChars(charsByPair, pairEntries, estimateCost);
483
+ const rough = await priceContentChars(roughCharsByPair, pairEntries, estimateCost);
484
+ const contentCost = billed.cost;
485
+ const contentUnknown = billed.unknown;
405
486
 
406
487
  content = {
407
488
  files: pendingSourceFiles.size,
408
489
  pendingTranslations: contentWorkItems.length,
409
490
  estimatedCost: contentUnknown ? null : contentCost,
491
+ costWithoutTM: rough.unknown ? null : rough.cost,
410
492
  rough: true,
493
+ // Which pairs have no price, and why (printCostTable names them).
494
+ ...(contentUnknown && { unpriced: billed.unpriced }),
495
+ // The rates the billed part was priced at.
496
+ ...(billed.rates.length > 0 && { rates: billed.rates }),
411
497
  };
412
498
  if (contentUnknown) {
413
499
  hasUnknownCosts = true;
@@ -418,16 +504,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
418
504
 
419
505
  printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
420
506
 
421
- return {
422
- currency: 'USD',
423
- pairs: costEstimates,
424
- keyCost: content && content.estimatedCost !== null
425
- ? totalEstimatedCost - content.estimatedCost
426
- : totalEstimatedCost,
427
- content,
428
- totalEstimatedCost,
429
- hasUnknownCosts,
430
- };
507
+ return summarizeEstimate(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
431
508
  } catch (costError) {
432
509
  // Non-blocking without a cap — warn and continue. Callers enforcing
433
510
  // --max-cost must treat the null return as unknown (abort).
@@ -508,9 +585,48 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
508
585
  );
509
586
  }
510
587
 
588
+ // Keys named for a redo (--redo keys: / --force-keys) must exist — the
589
+ // rule the key-value path applies (lib/named-keys.js). A name matching no
590
+ // UI string used to re-queue nothing in silence and exit 0. Checked before
591
+ // the preflight, the estimate or any spend: with no name matching nothing
592
+ // runs; with some, those are redone and the run then fails naming the rest.
593
+ // Docusaurus message ids are per file but named bare: a name re-queues
594
+ // that id in every JSON file that has it (as the diff below applies it).
595
+ const unmatchedNamed = applyNamedKeyRule({
596
+ cliArgs, config,
597
+ known: () => {
598
+ const ids = new Set();
599
+ for (const file of discoverDocusaurusJSONFiles(sourceLocaleDir)) {
600
+ for (const key of Object.keys(extractDocusaurusMessages(JSON.parse(fs.readFileSync(file, 'utf-8'))))) {
601
+ if (!isUnsafeKey(key)) ids.add(key);
602
+ }
603
+ }
604
+ return ids;
605
+ },
606
+ });
607
+
511
608
  // Thread dryRun/audit into cliArgs so the preflight check can skip
512
609
  // when appropriate — same pattern as the main sync path in sync.js
513
- const { apiKey, pairEntries } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
610
+ const { apiKey, pairEntries, preflightFailures = [] } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
611
+
612
+ // Content needs what the pair's own method needs — a local model needs no
613
+ // OPENROUTER_API_KEY (the lane used to demand it for every pair). Same
614
+ // lazy, per-pair readiness check as the content lane (content-sync.js).
615
+ const readinessCache = new Map();
616
+ const requirePairReady = async (pairKey, pairConfig, code, what) => {
617
+ if (!readinessCache.has(pairKey)) {
618
+ const method = getMethod(pairConfig.method || 'llm', pairConfig);
619
+ readinessCache.set(pairKey, Promise.resolve(method.checkReadiness({ apiKey, cwd })));
620
+ }
621
+ const readiness = await readinessCache.get(pairKey);
622
+ if (!readiness.ready) {
623
+ throw new Error(
624
+ `Docusaurus ${what} for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
625
+ ` ${readiness.reason}\n` +
626
+ missingKeyAdvice({ reasons: [readiness.reason], setupHelp: [' Set the key it names in .env.local (or your environment) to translate content.'] }).join('\n')
627
+ );
628
+ }
629
+ };
514
630
 
515
631
  if (pairEntries.length === 0) {
516
632
  output.info('No target languages configured. Add pairs to champollion.config.json.');
@@ -542,6 +658,13 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
542
658
  } else if (tmInitialSize > 0) {
543
659
  output.info(`[TM] ${tmInitialSize} cached entries loaded`);
544
660
  }
661
+ // Model carry-over (lib/tm.js) is on unless --fresh-on-model-change.
662
+ const freshOnModelChange = !!cliArgs['fresh-on-model-change'];
663
+ if (freshOnModelChange) setModelCarryover(tm, false);
664
+ // Entries made before coaching was keyed, with today's coaching: still
665
+ // served (lib/tm.js adoptLegacyCoachingKeys).
666
+ if (!noTM) adoptLegacyCoachingKeys(tm, pairEntries.map(([, pc]) => pc));
667
+ if (!noTM) warnModelSwitchStrandedTM(tm, pairEntries.map(([, pc]) => pc), { fresh: freshOnModelChange });
545
668
 
546
669
  // Phase-1 source-hash manifest (.champollion.lock) — read up front
547
670
  // because the cost estimator mirrors Phase 1's changed-key detection.
@@ -551,8 +674,19 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
551
674
  // manifest, EDITING an English UI string never re-translated it: the
552
675
  // diff only sees missing keys and [EN] fallbacks, so a changed source
553
676
  // value looked "fully synced" forever.
554
- const lockManifest = readManifest(cwd);
677
+ const lock = readLock(cwd);
678
+ const lockManifest = lock.source;
555
679
  const updatedLockManifest = { ...lockManifest };
680
+ // The lock's per-locale record (lib/locale-state.js): which UI strings the
681
+ // quality gate refused, per (file × key × locale), for their current source
682
+ // text and the method key that produced the refused answer. A plain sync
683
+ // holds them back from that method — the key-value rule. Keyed like the
684
+ // manifest ("docusaurus:<relPath>:<id>").
685
+ const lockState = new LockState(lock.locales);
686
+ // What counts as an explicit redo (always sent, never held back): ids named
687
+ // by --redo keys: / --force-keys, every id under --redo all (--force), and
688
+ // anything under --fresh.
689
+ const redo = { named: new Set(config.forceKeys || []), bulk: !!config._forceAllKeys, fresh: noTM };
556
690
 
557
691
  // ── Content discovery + pending-work scan ─────────────────────
558
692
  // Done up front (not in Phase 2) because the cost estimate needs the
@@ -586,9 +720,29 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
586
720
  if (fs.existsSync(contentLockPath)) {
587
721
  try { docuContentManifest = JSON.parse(fs.readFileSync(contentLockPath, 'utf-8')); } catch { /* first run */ }
588
722
  }
723
+ // --files / --retranslate (lib/file-scope.js). Resolved at the scan, so a
724
+ // pattern that matches nothing fails before the estimate or any spend.
725
+ const fileScope = compileFileScope(cliArgs);
589
726
  const contentScan = scanDocusaurusContentWork(
590
- contentSources, pairEntries, config, docuContentManifest, forceContent
727
+ contentSources, pairEntries, config, docuContentManifest, forceContent, fileScope
591
728
  );
729
+ if (fileScope) fileScope.assertAllMatched();
730
+ // --retranslate files miss the TM for this run (estimate and sync alike),
731
+ // so they are priced and translated fresh; new results replace the cache.
732
+ for (const item of contentScan.workItems) {
733
+ if (!item.retranslate) continue;
734
+ bypassTMFor(tm, item.code, [
735
+ ...Object.values(item.fieldsToTranslate), item.parsed.body, ...translatableBlockSources(item.parsed.body),
736
+ ]);
737
+ }
738
+ // Blocks and front-matter fields the quality gate refused before
739
+ // (lib/content-refusals.js): not sent to the method that refused them again
740
+ // unless the page is named for a redo (--redo files: / --redo content /
741
+ // --retranslate / --fresh). The estimate and the sync read the same holds.
742
+ for (const item of contentScan.workItems) {
743
+ item.refusalsBefore = readRefusals(docuContentManifest, item.manifestKey);
744
+ item.holds = contentHolds(item.refusalsBefore, item.pairConfig, { redo: forceContent || item.retranslate || noTM });
745
+ }
592
746
 
593
747
  // ── Pre-sync cost estimation + --max-cost gate ────────────────
594
748
  // Mirrors sync.js: without a cap the estimate is informational and
@@ -598,26 +752,63 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
598
752
  // exact thing a capped user needs to see.
599
753
  const maxCost = parseMaxCost(cliArgs['max-cost']);
600
754
  const costEstimate = await printDocusaurusCostEstimate(
601
- pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate
755
+ pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate, lockState, redo
602
756
  );
757
+ // Machine-readable estimate BEFORE the gate (see sync.js).
758
+ if (costEstimate) output.event('cost', costEstimate);
759
+ // A dry run never stops at the cap, but says what the real run would do.
760
+ let dryMaxCost = null;
603
761
  if (maxCost !== null && !dryRun) {
604
- if (!costEstimate) {
605
- return abortForMaxCost(
606
- maxCost, null,
607
- 'Cost estimation failed, so --max-cost cannot be enforced (unknown is not free).'
608
- );
609
- }
610
- if (costEstimate.hasUnknownCosts) {
611
- return abortForMaxCost(
612
- maxCost, null,
613
- 'Some pairs have unknown pricing, so the total cost cannot be bounded (unknown is not free).'
614
- );
762
+ const verdict = maxCostVerdict(maxCost, costEstimate);
763
+ if (verdict.wouldStop) return abortForMaxCost(maxCost, verdict.estimatedCost, verdict.reason);
764
+ } else if (maxCost !== null) {
765
+ dryMaxCost = reportDryRunMaxCost(maxCost, costEstimate, { stopsEarlier: preflightStopReason(preflightFailures) });
766
+ }
767
+
768
+ // Fallback methods are outside the estimate (they translate what the
769
+ // primary fails, unknown in advance). Under --max-cost each fallback batch
770
+ // must fit in what the estimate left of the cap (lib/fallback.js).
771
+ if (pairEntries.some(([, p]) => p.fallback)) {
772
+ output.info(
773
+ 'Fallback methods are not in this estimate — they only translate what the primary method fails, '
774
+ + 'which is not known in advance.'
775
+ + (maxCost !== null ? ' Under --max-cost each fallback batch is priced before it runs and skipped if it would pass the cap.' : '')
776
+ );
777
+ }
778
+ const fallbackBudget = createFallbackBudget({
779
+ maxCost: dryRun ? null : maxCost,
780
+ committed: costEstimate?.knownEstimatedCost ?? 0,
781
+ cwd,
782
+ });
783
+ // Different inputs, same output (lib/validate.js SharedOutputIndex): one
784
+ // index per locale for this run's UI strings and Markdown pages — cached
785
+ // and fresh alike, as in the content lane (lib/content-sync.js).
786
+ const sharedOutputIndexes = new Map();
787
+ const sharedOutputsFor = (code) => {
788
+ if (!sharedOutputIndexes.has(code)) {
789
+ sharedOutputIndexes.set(code, new SharedOutputIndex({ protectedTerms: config.protectedTerms || [] }));
615
790
  }
616
- if (costEstimate.totalEstimatedCost > maxCost) {
617
- return abortForMaxCost(
618
- maxCost, costEstimate.totalEstimatedCost,
619
- 'Estimated translation cost exceeds the --max-cost cap.'
620
- );
791
+ return sharedOutputIndexes.get(code);
792
+ };
793
+ // Per-pair tallies of what each fallback did: JSON keys, Markdown segments.
794
+ const keyFallbackTallies = new Map();
795
+ const contentFallbackTallies = new Map();
796
+ // A page some of whose text the fallback produced (its answer or its
797
+ // cache): named in the [FALLBACK] line.
798
+ const notePage = (pairKey, page) => {
799
+ const tally = contentFallbackTallies.get(pairKey);
800
+ if (tally && !tally.produced.includes(page)) tally.produced.push(page);
801
+ };
802
+ const tallyContent = (pairKey, report, page) => {
803
+ if (!contentFallbackTallies.has(pairKey)) return;
804
+ addToTally(contentFallbackTallies.get(pairKey), report);
805
+ if (report && (report.accepted > 0 || report.cached > 0)) notePage(pairKey, page);
806
+ };
807
+ if (!dryRun) {
808
+ for (const [pairKey, pc] of pairEntries) {
809
+ if (!pc.fallback) continue;
810
+ keyFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
811
+ contentFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
621
812
  }
622
813
  }
623
814
 
@@ -628,6 +819,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
628
819
 
629
820
  let totalJSONKeys = 0;
630
821
  let totalJSONCopied = 0;
822
+ let totalJSONFailed = 0;
823
+ // UI strings held back (refused before; not sent) and what the next sync
824
+ // does with each one this run could not translate (lib/locale-state.js).
825
+ let totalJSONHeld = 0;
826
+ let totalJSONWritten = 0;
827
+ const jsonFates = { retry: [], held: [] }; // { pair, key }
828
+ // Every lock key the source still has (refused records for others are dropped).
829
+ const liveLockKeys = new Set();
631
830
 
632
831
  for (const sourceFilePath of sourceJSONFiles) {
633
832
  const relPath = path.relative(sourceLocaleDir, sourceFilePath);
@@ -649,6 +848,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
649
848
  // The manifest stores namespaced keys; detectChangedKeys expects the
650
849
  // same key space as sourceFlat, so build a per-file un-namespaced view.
651
850
  const nsPrefix = `docusaurus:${relPath}:`;
851
+ for (const key of Object.keys(sourceFlat)) liveLockKeys.add(nsPrefix + key);
652
852
  const fileOldManifest = {};
653
853
  for (const [nsKey, storedHash] of Object.entries(lockManifest)) {
654
854
  if (nsKey.startsWith(nsPrefix)) {
@@ -664,10 +864,11 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
664
864
  const pairResults = await pMap(pairEntries, async ([pairKey, pairConfig]) => {
665
865
  const code = pairConfig.target;
666
866
  const targetFilePath = path.join(config.localesDir, code, relPath);
867
+ const filename = `${code}/${relPath}`;
667
868
 
668
869
  // Security: verify target path stays within i18n directory
669
870
  if (!isPathContained(targetFilePath, config.localesDir)) {
670
- output.error(`${code}/${relPath} — refusing to write outside i18n directory`);
871
+ output.error(`${filename} — refusing to write outside i18n directory`);
671
872
  // Nothing was attempted, but nothing succeeded either: keep every
672
873
  // to-be-processed key out of the "succeeded" set so its manifest
673
874
  // hash is not advanced (an unwritable locale must re-fire).
@@ -685,11 +886,15 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
685
886
  // re-translate (they are otherwise invisible to the diff). Echo keys
686
887
  // (target === source) the TM confirms as pipeline-produced are NOT
687
888
  // requeued — see lib/diff.js isConfirmedEcho.
688
- const tmKey = tmMethodKey(pairConfig);
889
+ // A value the pair's fallback produced is cached under the fallback.
890
+ const tmKeys = tmKeysForPair(pairConfig);
891
+ const localeState = dryRun ? lockState.peek(code) : lockState.of(code);
892
+ // A refused string whose source was edited since: queued (the hold lifts).
893
+ const localeChanged = [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, localeState)])];
689
894
  const diff = diffLocale(
690
895
  sourceFlat, existingFlat, config.fallbackPrefix,
691
- config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, changedKeys,
692
- (key, sourceValue) => lookupTM(tm, sourceValue, code, tmKey) === sourceValue,
896
+ config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
897
+ (key, sourceValue) => tmHoldsValue(tm, sourceValue, code, tmKeys, sourceValue),
693
898
  noTranslate.active ? noTranslate.matches : null
694
899
  );
695
900
 
@@ -697,12 +902,38 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
697
902
  return { keys: 0, failedKeys: [] };
698
903
  }
699
904
 
905
+ // Refused before (the lock's records): held back from the method that
906
+ // refused them — the key-value rule (planUIStrings).
907
+ const plan = planUIStrings({
908
+ diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState, pairConfig, redo,
909
+ fallbackPrefix: config.fallbackPrefix,
910
+ });
911
+ const heldSet = new Set(plan.held);
912
+ const lk = (k) => nsPrefix + k;
913
+
700
914
  let keysProcessed = 0;
701
915
  let keysCopied = 0;
916
+ let keysWritten = 0; // translations written (the model, its fallback, the cache)
702
917
  const failedKeys = [];
918
+ const heldKeys = [];
919
+ const fates = { retry: [], held: [] };
703
920
 
704
921
  if (diff.toProcess.length > 0 || diff.noTranslate.length > 0) {
705
- output.info(`${code}/${relPath} — ${diffLabel(diff)}`);
922
+ // The per-file line counts what goes to the pipeline; held keys are
923
+ // said on their own line below.
924
+ const shown = heldSet.size === 0 ? diff : Object.fromEntries(Object.entries(diff).map(([k, v]) => (
925
+ [k, Array.isArray(v) && k !== 'noTranslate' && k !== 'extra' ? v.filter(x => !heldSet.has(x)) : v])));
926
+ const label = diffLabel(shown);
927
+ output.info(`${filename} — ${plan.held.length > 0 ? `${label === 'fully synced' ? '' : `${label} + `}${plan.held.length} held back` : label}`);
928
+ if (plan.held.length > 0) {
929
+ output.warn(describeHeldKeys({ filename, keys: plan.held, pairConfig, command: redoCommand(plan.held, { pair: pairKey }) }));
930
+ }
931
+ if (plan.heldFromPrimary.length > 0) {
932
+ output.info(describeFallbackOnlyKeys({ filename, keys: plan.heldFromPrimary, pairConfig }));
933
+ }
934
+ // Named for a redo but served from the cache (the model is not asked
935
+ // without --fresh): said, with the command that asks the model again.
936
+ const namedQueued = redo.fresh ? [] : diff.toProcess.filter(k => redo.named.has(k) && typeof sourceFlat[k] === 'string');
706
937
 
707
938
  if (!dryRun) {
708
939
  // Merged output starts from what's on disk, then takes the verbatim
@@ -712,6 +943,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
712
943
  const mergedFlat = { ...existingFlat };
713
944
  for (const key of diff.noTranslate) {
714
945
  mergedFlat[key] = sourceFlat[key];
946
+ delete localeState.refused[lk(key)];
715
947
  }
716
948
  keysCopied = diff.noTranslate.length;
717
949
  const writeMerged = () => {
@@ -719,32 +951,77 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
719
951
  fs.mkdirSync(path.dirname(targetFilePath), { recursive: true });
720
952
  fs.writeFileSync(targetFilePath, JSON.stringify(docuOutput, null, 2) + '\n', 'utf-8');
721
953
  };
954
+ // What the gate refused this run is remembered per key (held back
955
+ // by the next plain sync); what got no usable answer is asked again.
956
+ const settleUnfilled = (keys, refusedBy = {}) => {
957
+ for (const k of keys) {
958
+ recordRefusal(localeState, lk(k), sourceFlat[k], refusedBy[k]);
959
+ fates[(refusedBy[k] || []).length > 0 ? 'held' : 'retry'].push(k);
960
+ }
961
+ };
722
962
 
723
963
  if (keysCopied > 0) {
724
964
  const sample = diff.noTranslate.slice(0, 3)
725
965
  .map(k => `${k} (${noTranslate.reason(k, sourceFlat[k])})`)
726
966
  .join(', ');
727
967
  const more = keysCopied > 3 ? `, +${keysCopied - 3} more` : '';
728
- output.info(`${code}/${relPath} — copied ${keysCopied} no-translate key(s) verbatim: ${sample}${more}`);
968
+ output.info(`${filename} — copied ${keysCopied} no-translate key(s) verbatim: ${sample}${more}`);
729
969
  }
730
970
 
731
971
  let translated = null;
972
+ let result = null;
732
973
  const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
733
974
 
734
975
  if (stringKeys.length > 0) {
735
- // Shared pipeline: TM partition → API call → TM store → quality gate
736
- const result = await translateAndValidate(stringKeys, sourceFlat, pairConfig, pairKey, {
737
- apiKey, tm, targetCode: code, descriptions,
976
+ // Shared pipeline: TM partition → API call → quality gate → TM
977
+ // store, then the pair's fallback (if any) for what that left
978
+ // untranslated (lib/translate-pair.js translateWithFallback).
979
+ // Keys held back: the cache is still read, the method not asked.
980
+ result = await translateWithFallback(stringKeys, sourceFlat, pairConfig, pairKey, {
981
+ apiKey, tm, targetCode: code, descriptions, budget: fallbackBudget, cwd,
982
+ // One per locale, as on the standard path: a memorized sentence
983
+ // repeated across UI strings is refused (validate.js).
984
+ sharedOutputs: sharedOutputsFor(code),
985
+ noSendPrimary: new Set([...plan.held, ...plan.heldFromPrimary]),
986
+ noSendFallback: heldSet,
738
987
  });
739
988
  translated = result.translated;
989
+ if (keyFallbackTallies.has(pairKey)) addToTally(keyFallbackTallies.get(pairKey), result.fallback);
990
+ const heldHere = new Set(result.heldKeys || []);
991
+ heldKeys.push(...stringKeys.filter(k => heldHere.has(k) && !(translated && k in translated)));
740
992
 
741
993
  if (translated) {
742
- output.progress(result.failures.length > 0 ? ` [OK] (${result.failures.length} failed gate)\n` : ' [OK]\n');
743
- } else if (result.apiReturnedNull || (result.failures.length > 0 && !translated)) {
744
- output.progress(result.apiReturnedNull ? ' [ERR] translation failed\n' : ' [ERR] all failed quality gate\n');
745
- output.error(`${code}/${relPath}: Translation failed. Check API key and method configuration.`);
994
+ const notDone = stringKeys.filter(k => !(k in translated));
995
+ const refusedHere = notDone.filter(k => (result.refusedBy?.[k] || []).length > 0);
996
+ const heldNow = notDone.filter(k => heldHere.has(k));
997
+ const noAnswer = notDone.length - refusedHere.length - heldNow.length;
998
+ output.progressDone(filename, notDone.length === 0 ? '[OK]' : `[WARN] ${notDone.length} of ${stringKeys.length} key(s) not translated (${[
999
+ refusedHere.length > 0 && `${refusedHere.length} refused by the quality gate`,
1000
+ heldNow.length > 0 && `${heldNow.length} held back`,
1001
+ noAnswer > 0 && `${noAnswer} not returned by the method`,
1002
+ ].filter(Boolean).join(', ')})`);
1003
+ const answered = new Set(result.answeredKeys || []);
1004
+ const fromCache = namedQueued.filter(k => k in translated && !answered.has(k));
1005
+ await reportNamedFromCache({
1006
+ filename, keys: fromCache, served: translated, onDisk: existingFlat, pairConfig, cwd,
1007
+ command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
1008
+ });
1009
+ } else if (heldKeys.length === stringKeys.length) {
1010
+ // Everything queued was held back: nothing was asked, nothing failed anew.
1011
+ output.progressDone(filename, `[WARN] ${heldKeys.length} key(s) held back, not translated`);
1012
+ for (const k of heldKeys) fates.held.push(k);
746
1013
  if (keysCopied > 0) writeMerged();
747
- return { keys: 0, copied: keysCopied, failedKeys: stringKeys };
1014
+ return { keys: 0, copied: keysCopied, failedKeys: [], heldKeys, fates };
1015
+ } else if (result.apiReturnedNull || result.failures.length > 0) {
1016
+ output.progressDone(filename, result.apiReturnedNull ? '[ERR] translation failed' : '[ERR] all failed quality gate');
1017
+ output.error(result.apiReturnedNull
1018
+ ? `${filename}: Translation failed. Check API key and method configuration.`
1019
+ : `${filename}: all translations were rejected by the quality gate${pairConfig.fallback ? ` (and by its fallback, ${pairConfig.fallback.method})` : ''} — see the gate failures above.`);
1020
+ if (keysCopied > 0) writeMerged();
1021
+ const failedNow = stringKeys.filter(k => !heldKeys.includes(k));
1022
+ settleUnfilled(failedNow, result.refusedBy || {});
1023
+ for (const k of heldKeys) fates.held.push(k);
1024
+ return { keys: 0, copied: keysCopied, failedKeys: failedNow, heldKeys, fates };
748
1025
  }
749
1026
  }
750
1027
 
@@ -753,6 +1030,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
753
1030
  // for it, fallbacks first, and a value with unmappable letters
754
1031
  // stays whole in the working script (warned, not failed).
755
1032
  const scriptConverterKey = pairConfig.scriptResolution?.converterKey || null;
1033
+ const heldNow = new Set(heldKeys);
756
1034
  for (const key of diff.toProcess) {
757
1035
  if (translated && key in translated) {
758
1036
  let value = translated[key];
@@ -763,50 +1041,87 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
763
1041
  value = converted;
764
1042
  } else {
765
1043
  output.warn(
766
- `${code}/${relPath}: key "${key}" kept in working script — `
1044
+ `${filename}: key "${key}" kept in working script — `
767
1045
  + `unmapped letter(s): ${unmapped.join(', ')} (see "scriptFallback")`
768
1046
  );
769
1047
  }
770
1048
  }
771
1049
  mergedFlat[key] = value;
1050
+ keysWritten++;
1051
+ // Filled (by the model, its fallback or the cache): no longer refused.
1052
+ delete localeState.refused[lk(key)];
1053
+ } else if (heldNow.has(key)) {
1054
+ // Held back: said above; not sent, not counted as failed.
1055
+ fates.held.push(key);
772
1056
  } else if (typeof sourceFlat[key] === 'string') {
773
- output.warn(`${code}/${relPath}: key "${key}" not translated — skipping`);
1057
+ const fate = (result?.refusedBy?.[key] || []).length > 0 ? 'held' : 'retry';
1058
+ output.warn(`${filename}: key "${key}" ${keyFateNote(fate, pairConfig)}`);
774
1059
  failedKeys.push(key);
1060
+ settleUnfilled([key], result?.refusedBy || {});
775
1061
  } else {
776
1062
  mergedFlat[key] = sourceFlat[key];
777
1063
  }
778
1064
  }
779
1065
 
780
- keysProcessed = diff.toProcess.length;
1066
+ keysProcessed = diff.toProcess.length - heldKeys.length;
781
1067
 
782
1068
  // Inject back into Docusaurus format and write
783
1069
  writeMerged();
784
1070
  } else {
785
- keysProcessed = diff.toProcess.length;
1071
+ // Held keys would not be sent: not counted.
1072
+ keysProcessed = diff.toProcess.length - plan.held.length;
786
1073
  keysCopied = diff.noTranslate.length;
1074
+ if (namedQueued.length > 0) {
1075
+ // What the real run would serve from the cache (the pair's entry,
1076
+ // then its fallback's — the same partition the estimate makes).
1077
+ const served = {};
1078
+ for (const k of namedQueued.filter(x => !heldSet.has(x))) {
1079
+ for (const mk of tmKeys) {
1080
+ const v = peekTM(tm, sourceFlat[k], code, mk);
1081
+ if (v !== null) { served[k] = v; break; }
1082
+ }
1083
+ }
1084
+ const fromCache = Object.keys(served);
1085
+ await reportNamedFromCache({
1086
+ filename, keys: fromCache, served, onDisk: existingFlat, pairConfig, cwd, dryRun: true,
1087
+ command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
1088
+ });
1089
+ }
1090
+ heldKeys.push(...plan.held);
787
1091
  }
788
1092
  }
789
1093
 
790
1094
  if (diff.extra.length > 0) {
791
- output.warn(`${code}/${relPath} — ${diff.extra.length} extra key(s)`);
1095
+ output.warn(`${filename} — ${diff.extra.length} extra key(s)`);
792
1096
  }
793
1097
 
794
- return { keys: keysProcessed, copied: keysCopied, failedKeys };
1098
+ return { keys: keysProcessed, written: keysWritten, copied: keysCopied, failedKeys, heldKeys, fates };
795
1099
  }, { concurrency: jsonConcurrency });
796
1100
 
797
1101
  // Aggregate results for this JSON file
798
1102
  const fileFailedKeys = new Set();
799
- for (const r of pairResults) {
1103
+ // Held keys keep their manifest hash too (an edited source must still
1104
+ // re-fire), but they are not failures.
1105
+ const fileUnfilledKeys = new Set();
1106
+ pairResults.forEach((r, i) => {
800
1107
  totalJSONKeys += r.keys;
1108
+ totalJSONWritten += r.written || 0;
801
1109
  totalJSONCopied += r.copied || 0;
802
- for (const k of r.failedKeys || []) fileFailedKeys.add(k);
803
- }
1110
+ totalJSONHeld += (r.heldKeys || []).length;
1111
+ for (const k of r.failedKeys || []) { fileFailedKeys.add(k); fileUnfilledKeys.add(k); }
1112
+ for (const k of r.heldKeys || []) fileUnfilledKeys.add(k);
1113
+ const pair = pairEntries[i][0];
1114
+ for (const fate of ['retry', 'held']) {
1115
+ for (const key of r.fates?.[fate] || []) jsonFates[fate].push({ pair, key });
1116
+ }
1117
+ });
1118
+ totalJSONFailed += fileFailedKeys.size;
804
1119
 
805
1120
  // Update the manifest for this file: record the current hash ONLY for
806
1121
  // keys that succeeded in every locale that attempted them. A failed
807
- // key keeps its OLD hash (or none) so it is detected as changed and
808
- // RE-FIRES on the next sync — advancing the hash for a failed key
809
- // would silently mark the stale translation as current forever.
1122
+ // (or held) key keeps its OLD hash (or none) so it is detected as
1123
+ // changed and RE-FIRES on the next sync — advancing the hash for a failed
1124
+ // key would silently mark the stale translation as current forever.
810
1125
  if (!dryRun) {
811
1126
  for (const nsKey of Object.keys(updatedLockManifest)) {
812
1127
  // Own-property check: `in` walks the prototype chain, so a source
@@ -818,7 +1133,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
818
1133
  }
819
1134
  for (const [key, value] of Object.entries(sourceFlat)) {
820
1135
  const nsKey = nsPrefix + key;
821
- if (fileFailedKeys.has(key)) {
1136
+ if (fileUnfilledKeys.has(key)) {
822
1137
  // Restore/keep the pre-sync state for failed keys.
823
1138
  if (Object.prototype.hasOwnProperty.call(lockManifest, nsKey)) {
824
1139
  updatedLockManifest[nsKey] = lockManifest[nsKey];
@@ -832,25 +1147,68 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
832
1147
  }
833
1148
  }
834
1149
 
835
- // Persist the Phase 1 source-hash manifest (skip in dry-run — a preview
836
- // must not mark changed keys as resolved).
1150
+ // Persist the Phase 1 source-hash manifest and the refusal records (skip
1151
+ // in dry-run — a preview must not mark changed keys as resolved). Records
1152
+ // of UI strings the source no longer has are dropped for the locales this
1153
+ // run processed.
837
1154
  if (!dryRun && sourceJSONFiles.length > 0) {
838
- writeManifest(cwd, updatedLockManifest);
1155
+ for (const [, pc] of pairEntries) {
1156
+ const refused = lockState.of(pc.target).refused;
1157
+ for (const k of Object.keys(refused)) {
1158
+ if (k.startsWith('docusaurus:') && !liveLockKeys.has(k)) delete refused[k];
1159
+ }
1160
+ }
1161
+ writeManifest(cwd, updatedLockManifest, lockState.toJSON());
1162
+ }
1163
+ for (const [pairKey, tally] of keyFallbackTallies) {
1164
+ printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
1165
+ warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
839
1166
  }
840
1167
 
841
1168
  const copiedNote = totalJSONCopied > 0
842
1169
  ? ` (+${totalJSONCopied} copied verbatim, no-translate)`
843
1170
  : '';
844
- if (totalJSONKeys > 0) {
1171
+ const heldNote = totalJSONHeld > 0
1172
+ ? `; ${totalJSONHeld} held back (refused before; not sent, not billed)`
1173
+ : '';
1174
+ const failedNote = totalJSONFailed > 0 ? `; ${totalJSONFailed} failed (listed above)` : '';
1175
+ // A real run counts what it wrote; a dry run what it would send.
1176
+ const shownKeys = dryRun ? totalJSONKeys : totalJSONWritten;
1177
+ if (shownKeys > 0) {
845
1178
  const action = dryRun ? 'Would process' : 'Synced';
846
- output.ok(`${action} ${totalJSONKeys} JSON key(s)${copiedNote}`);
1179
+ output.ok(`${action} ${shownKeys} JSON key(s)${copiedNote}${failedNote}${heldNote}`);
1180
+ } else if (totalJSONHeld > 0 || totalJSONFailed > 0) {
1181
+ output.warn(`No JSON key ${dryRun ? 'would be' : 'was'} translated${copiedNote}${failedNote}${heldNote}`);
847
1182
  } else if (totalJSONCopied > 0) {
848
1183
  output.ok(`All JSON files fully synced${copiedNote}`);
849
1184
  } else {
850
1185
  output.ok('All JSON files fully synced');
851
1186
  }
1187
+ // What the next sync does with the UI strings this run could not translate
1188
+ // — said per outcome, in the key-value lane's words.
1189
+ if (!dryRun && (jsonFates.retry.length > 0 || jsonFates.held.length > 0)) {
1190
+ if (jsonFates.retry.length > 0) {
1191
+ output.warn(` ${jsonFates.retry.length} UI string(s) got no usable answer (missing from the response, or the method failed) — the next sync asks for them again.`);
1192
+ }
1193
+ // Ids are named bare (a name re-queues that id in every JSON file that has it).
1194
+ const heldNext = [];
1195
+ const seen = new Set();
1196
+ for (const { pair, key } of jsonFates.held) {
1197
+ if (seen.has(`${pair}\u0000${key}`)) continue;
1198
+ seen.add(`${pair}\u0000${key}`);
1199
+ heldNext.push({ pair, key });
1200
+ }
1201
+ for (const line of describeHeldNext(heldNext, (keys, pair) => redoCommand(keys, { pair }))) output.warn(line);
1202
+ }
852
1203
 
853
1204
  // ── Phase 2: Markdown content (docs + blog) ───────────────────
1205
+ let contentFailures = 0; // read after the TM save, outside the content block
1206
+ let contentTranslated = 0;
1207
+ let contentFailedItems = [];
1208
+ // Blocks/fields refused before and held back this run, and refused this run.
1209
+ let contentHeldBack = 0;
1210
+ const contentHeldBackItems = [];
1211
+ let contentRefused = 0;
854
1212
 
855
1213
  if (contentSources.length === 0) {
856
1214
  output.info('No docs/ or blog/ directories found — skipping content sync.');
@@ -872,7 +1230,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
872
1230
  const concurrency = config.contentConcurrency || 48;
873
1231
 
874
1232
  const totalWork = workItems.length;
875
- let contentFailures = 0;
1233
+ // Refusals per translation (lib/content-refusals.js), applied to the lock
1234
+ // with each item's outcome.
1235
+ const refusalStates = new Map();
1236
+ // Every failed (file × locale), with what it was left as on disk — the
1237
+ // end-of-run list an operator needs to tell "failed, previous
1238
+ // translation kept" from "failed, will retry" (dogfood 2026-08-28,
1239
+ // finding 5).
1240
+ const failedItems = [];
876
1241
  // Warn at most once per source file about front matter fields we can't
877
1242
  // translate (arrays / nested blocks like `related:`). The check + add are
878
1243
  // synchronous (no await between), so this is race-free under pMap.
@@ -905,23 +1270,67 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
905
1270
  await pMap(workItems, async (item) => {
906
1271
  const {
907
1272
  parsed, fieldsToTranslate, pageTitle, sourceHash,
908
- relPath, dirName, pairConfig, code,
1273
+ relPath, dirName, pairKey, pairConfig, code,
909
1274
  targetPath, manifestKey, action,
910
1275
  } = item;
911
1276
 
1277
+ let itemFailed = false;
1278
+ // This translation's refusals, recorded with its outcome (held,
1279
+ // failed or written) — in the incremental lock writes too.
1280
+ const settleRefusals = () => {
1281
+ const st = refusalStates.get(manifestKey);
1282
+ if (!st || dryRun) return;
1283
+ storeRefusals(updatedDocuManifest, manifestKey, nextRefusals(st.prior, st));
1284
+ manifestDirty = true;
1285
+ };
912
1286
  try {
913
1287
  if (action === 'changed') {
914
1288
  totalContentRetranslated++;
915
1289
  }
916
1290
 
917
1291
  if (dryRun) {
918
- const targetRel = path.relative(config.localesDir, targetPath);
919
- output.raw(` [DRY] ${dirName}/${relPath} → ${code}`);
920
- totalContent++;
1292
+ // What the quality gate refused before is held back by the real
1293
+ // run (lib/content-refusals.js) — said here.
1294
+ const preview = previewHeld({
1295
+ tm, code, pairConfig, fields: parsed.hasFrontMatter ? fieldsToTranslate : {}, body: parsed.body,
1296
+ segMode: pairConfig.contentSegmentation || config.contentSegmentation || 'block', holds: item.holds,
1297
+ });
1298
+ const held = preview.names.length > 0
1299
+ ? ` — ${preview.pageHeld ? 'would hold the page back:' : 'would hold back'} ${preview.names.join(', ')} (refused before; not sent)`
1300
+ : '';
1301
+ output.raw(` [DRY] ${dirName}/${relPath} → ${code}${held}`);
1302
+ output.event('file', {
1303
+ lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: preview.pageHeld ? 'would-hold' : 'would-translate', action,
1304
+ ...(preview.names.length > 0 && { held: preview.names.length }),
1305
+ });
1306
+ if (!preview.pageHeld) totalContent++;
921
1307
  return;
922
1308
  }
923
1309
 
924
1310
  const { rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parsed;
1311
+ const segModeHere = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
1312
+ const label = `${dirName}/${relPath}`;
1313
+ const refusal = {
1314
+ file: label, locale: code, pageHeld: false,
1315
+ prior: item.refusalsBefore || {},
1316
+ refused: [], filled: new Set(), held: [],
1317
+ fields: hasFrontMatter ? fieldsToTranslate : {},
1318
+ blockSources: segModeHere === 'block' && body.trim() ? translatableBlockSources(body) : [],
1319
+ pageSource: segModeHere === 'page' && body.trim() ? body : null,
1320
+ };
1321
+ refusalStates.set(manifestKey, refusal);
1322
+ const { holds } = item;
1323
+ // A page translated whole that the gate refused before (and that
1324
+ // the cache does not hold): held before anything of it is sent —
1325
+ // its front matter included (a pure look; the body read below is
1326
+ // the one that serves).
1327
+ if (refusal.pageSource && holds.page(body) === 'held'
1328
+ && ![tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])]
1329
+ .some(k => peekTM(tm, body, code, k) !== null)) {
1330
+ const err = new Error('held');
1331
+ err.heldNames = [PAGE_NAME];
1332
+ throw err;
1333
+ }
925
1334
 
926
1335
  // Never silently drop translatable-looking nested/array front matter
927
1336
  // (e.g. `related:` lists). The flat parser can't reach them — surface
@@ -942,10 +1351,32 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
942
1351
  // not re-serve. Mirrors content-sync.js.
943
1352
  const tmKey = tmMethodKey(pairConfig);
944
1353
 
1354
+ // This page's cached content through the repeat check, in one batch
1355
+ // and before any of it is served (lib/fallback.js): a text cached
1356
+ // for several different source strings is evicted, so the lookups
1357
+ // below miss it and it is translated again, checked the same way.
1358
+ const docLabel = `content:${dirName}/${relPath}`;
1359
+ const repeated = refuseRepeatedCachedPage({
1360
+ label: docLabel,
1361
+ fields: hasFrontMatter ? fieldsToTranslate : {},
1362
+ body,
1363
+ tm, code, pairConfig,
1364
+ sharedOutputs: sharedOutputsFor(code),
1365
+ });
1366
+ if (repeated.length > 0) {
1367
+ output.warn(
1368
+ `${dirName}/${relPath} → ${code}: cached ${repeated.join(', ')} held the same text as other, different source strings ` +
1369
+ '(a memorized sentence, not a translation) — removed from the cache and translated again.'
1370
+ );
1371
+ }
1372
+
945
1373
  // Translate front matter fields — TM first, API only for misses.
946
1374
  // Each field is cached on its own source text (exactly like a
947
1375
  // key-value sync key): a title edit re-pays only the title.
948
1376
  const translatedFields = {};
1377
+ // A field left in the source language (refused, or held from an
1378
+ // earlier refusal): the page is written, its lock marked pending.
1379
+ let fieldsLeftInSource = false;
949
1380
  if (hasFrontMatter && Object.keys(fieldsToTranslate).length > 0) {
950
1381
  const { hits: fmHits, misses: fmMisses } = partitionByTM(
951
1382
  tm, fieldsToTranslate, Object.keys(fieldsToTranslate), code, tmKey
@@ -954,61 +1385,94 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
954
1385
  // gateless pipeline (hollowed titles were cached here) must not
955
1386
  // outlive the gate. Failing hits are evicted and re-billed.
956
1387
  for (const [field, cachedValue] of Object.entries(fmHits)) {
957
- if (checkContentPreservation(fieldsToTranslate[field], cachedValue)) {
1388
+ if (contentGateFault(fieldsToTranslate[field], cachedValue, pairConfig)) {
958
1389
  evictTM(tm, fieldsToTranslate[field], code, tmKey);
959
1390
  delete fmHits[field];
960
1391
  fmMisses.push(field);
961
1392
  }
962
1393
  }
963
1394
  Object.assign(translatedFields, fmHits);
1395
+ // Then the fallback's cache (lib/fallback.js ladder).
1396
+ const fbCache = serveFieldsFromFallbackCache(tm, fieldsToTranslate, fmMisses, code, pairConfig);
1397
+ Object.assign(translatedFields, fbCache.hits);
1398
+ if (Object.keys(fbCache.hits).length > 0) notePage(pairKey, label);
1399
+ const fieldsToSend = fbCache.misses;
1400
+ for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
1401
+ // Refused before: a held field is not sent again; it keeps its
1402
+ // source text and the page is written (its lock marked pending).
1403
+ const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
1404
+ const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
1405
+ if (heldFields.length > 0) {
1406
+ fieldsLeftInSource = true;
1407
+ const names = heldFields.map(f => `front matter "${f}"`);
1408
+ refusal.held.push(...names);
1409
+ output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names }));
1410
+ for (const f of heldFields) fieldsToSend.splice(fieldsToSend.indexOf(f), 1);
1411
+ }
964
1412
 
965
- if (fmMisses.length > 0) {
966
- if (!apiKey) {
967
- // No API key — fail loud
968
- throw new Error(
969
- `Docusaurus content sync for ${code}: no API key available.\n` +
970
- ' Set OPENROUTER_API_KEY in .env.local to translate content.'
971
- );
1413
+ if (fieldsToSend.length > 0) {
1414
+ await requirePairReady(pairKey, pairConfig, code, 'content sync');
1415
+ // The pair's method, then its fallback (if any) for every field
1416
+ // it returned nothing for or hollowed. Content-preservation
1417
+ // gate on every value — Phase 2 front matter once reached disk
1418
+ // AND the TM with no validation at all, so a hollowed page
1419
+ // title was written silently and then cached forever. Nothing
1420
+ // is cached until no field is left failing; a failing field
1421
+ // skips the file and leaves its lock entry alone, so the next
1422
+ // sync retries it.
1423
+ const fm = await translateFieldsWithFallback({
1424
+ fields: fieldsToTranslate,
1425
+ misses: fieldsToSend,
1426
+ pairConfig,
1427
+ budget: fallbackBudget,
1428
+ sharedOutputs: sharedOutputsFor(code),
1429
+ label: `${docLabel} front matter`,
1430
+ translate: (keys, cfg, extra = {}) => translateBatch(
1431
+ keys, fieldsToTranslate, cfg,
1432
+ { apiKey, model: cfg.model, batchSize: cfg.batchSize || 30, ...extra },
1433
+ ),
1434
+ fallbackOnly: fallbackOnlyFields,
1435
+ });
1436
+ tallyContent(pairKey, fm.report, label);
1437
+ // Refused fields are remembered, whatever happens to the page.
1438
+ for (const [field, methods] of Object.entries(fm.refusedBy || {})) {
1439
+ refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
972
1440
  }
973
- const fmResult = await translateBatch(
974
- fmMisses, fieldsToTranslate, pairConfig,
975
- { apiKey, model: pairConfig.model, batchSize: pairConfig.batchSize || 30 },
976
- );
977
- if (fmResult) {
978
- // Content-preservation gate — Phase 2 front matter reached
979
- // disk AND the TM with no validation at all, so a hollowed
980
- // page title was written silently and then cached forever.
981
- // Throwing skips the file and leaves its lock entry alone,
982
- // so the next sync retries it.
983
- for (const [field, value] of Object.entries(fmResult)) {
984
- const sourceValue = fieldsToTranslate[field];
985
- if (typeof value !== 'string' || typeof sourceValue !== 'string') continue;
986
- const hollowed = checkContentPreservation(sourceValue, value);
987
- if (hollowed) {
988
- throw new Error(
989
- `Docusaurus content sync for ${code}: front matter "${field}" — ${hollowed.reason}.\n` +
990
- ` source: ${JSON.stringify(sourceValue)}\n` +
991
- ` got: ${JSON.stringify(value)}\n` +
992
- ' Nothing was written or cached. If this is a low-coverage target\n' +
993
- ' language, the model has no vocabulary for this string.'
994
- );
995
- }
996
- }
997
- Object.assign(translatedFields, fmResult);
998
- // Cache only what the API actually returned (the same
999
- // non-null check that gates writing it to the target file).
1000
- for (const [field, value] of Object.entries(fmResult)) {
1001
- if (typeof value === 'string' && typeof fieldsToTranslate[field] === 'string') {
1002
- storeTM(tm, fieldsToTranslate[field], code, tmKey, value);
1003
- }
1441
+ if (fm.hollowed.length > 0) {
1442
+ // Refused, and refused again when asked with the reason: the
1443
+ // field keeps its source text; the page is still written.
1444
+ fieldsLeftInSource = true;
1445
+ for (const h of fm.hollowed) {
1446
+ output.warn(
1447
+ `${label} → ${code}: front matter "${h.field}" kept in the source language — ${h.reason}`
1448
+ + `${h.fallbackReason ? `; the fallback (${pairConfig.fallback.method}): ${h.fallbackReason}` : ''}.`
1449
+ );
1004
1450
  }
1005
- } else {
1451
+ output.warn(describeNewHold({ file: label, pairKey, pairConfig, count: fm.hollowed.length }));
1452
+ }
1453
+ if (fm.noResults) {
1006
1454
  // Front matter translation failed — loud error
1007
1455
  throw new Error(
1008
- `Docusaurus content sync for ${code}: front matter translation returned no results.\n` +
1456
+ `Docusaurus content sync for ${code}: front matter translation returned no results`
1457
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
1009
1458
  ' Check your API key and method configuration.'
1010
1459
  );
1011
1460
  }
1461
+ // Cache only gate-passing values, each under the TM key of the
1462
+ // method that produced it.
1463
+ for (const st of fm.stores) storeTM(tm, st.text, code, st.tmKey, st.value);
1464
+ // A field only the fallback may be asked for that it did not
1465
+ // translate (skipped by --max-cost, or no answer): still refused
1466
+ // by the pair's method — the page is held, as with no fallback.
1467
+ const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
1468
+ if (unfilled.length > 0) {
1469
+ fieldsLeftInSource = true;
1470
+ const names = unfilled.map(f => `front matter "${f}"`);
1471
+ refusal.held.push(...names);
1472
+ output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names }));
1473
+ }
1474
+ Object.assign(translatedFields, fm.translated);
1475
+ for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
1012
1476
  }
1013
1477
  }
1014
1478
 
@@ -1020,40 +1484,92 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1020
1484
  let translatedBody = body;
1021
1485
  let bodyUsedFallback = false;
1022
1486
  if (body.trim()) {
1023
- const wholeBodyCached = lookupTMValidated(tm, body, code, tmKey,
1024
- (src, cached) => !checkContentPreservation(src, cached));
1487
+ // The pair's cache, then its fallback's (lib/fallback.js ladder).
1488
+ const wholeBodyCached = lookupContentTM(tm, body, code, pairConfig);
1025
1489
  if (wholeBodyCached !== null) {
1026
- translatedBody = wholeBodyCached;
1490
+ translatedBody = wholeBodyCached.text;
1491
+ for (const src of refusal.blockSources) refusal.filled.add(blockUnit(src));
1492
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
1027
1493
  } else {
1028
1494
  const segMode = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
1029
1495
  const { protectedBody, blocks } = protectBlocks(body);
1030
1496
  const promptOptions = {
1031
1497
  sourceLanguageName: DEFAULT_REGISTERS[inputLocale]?.name || inputLocale,
1032
1498
  promptContext: pairConfig.promptContext || null,
1499
+ protectedTerms: pairConfig.protectedTerms || [],
1033
1500
  };
1034
1501
 
1035
1502
  // Block stores are deferred until the reassembled body passes
1036
1503
  // the orphaned-placeholder check — the TM must never hold a
1037
- // value that would fail the gate on re-serve.
1504
+ // value that would fail the gate on re-serve. Each carries the
1505
+ // TM key of the method that produced it.
1038
1506
  const pendingBlockStores = [];
1507
+ // The whole body is cached under one method's key — only when
1508
+ // one method produced all of it.
1509
+ let wholeBodyKey = tmKey;
1510
+ let mixedProducers = false;
1511
+ // Page mode: whose whole page the checks below refuse (the
1512
+ // lane remembers it — lib/content-refusals.js "page").
1513
+ let pageRefusedBy = null;
1039
1514
 
1040
1515
  if (segMode === 'page') {
1041
- // Whole-page prompt — today's single-call behavior.
1042
- if (!apiKey) {
1043
- throw new Error(
1044
- `Docusaurus body translation for ${code}: no API key available.\n` +
1045
- ' Set OPENROUTER_API_KEY in .env.local to translate content.'
1046
- );
1516
+ // Whole-page prompt — today's single-call behavior. Refused
1517
+ // before by this method (and the cache does not hold it): not
1518
+ // sent again (lib/content-refusals.js).
1519
+ const pageHold = holds.page(body);
1520
+ if (pageHold === 'held') {
1521
+ const err = new Error('held');
1522
+ err.heldNames = [PAGE_NAME];
1523
+ throw err;
1524
+ }
1525
+ await requirePairReady(pairKey, pairConfig, code, 'body translation');
1526
+ const runPage = (cfg) => translateRawContent(
1527
+ buildContentPrompt(protectedBody, cfg, promptOptions), { apiKey, pairConfig: cfg });
1528
+ let bodyResult;
1529
+ if (pairConfig.fallback) {
1530
+ // When both fail, the primary's page goes to the checks
1531
+ // below, which fail the file exactly as without a fallback.
1532
+ // Refused before by the pair's method: only the fallback is asked.
1533
+ const page = await translatePageWithFallback({
1534
+ body, blocks, pairConfig, runPage, budget: fallbackBudget, fallbackOnly: pageHold === 'fallback-only',
1535
+ });
1536
+ tallyContent(pairKey, page.report, label);
1537
+ if (page.refusedBy.length > 0) {
1538
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: page.refusedBy });
1539
+ }
1540
+ if (pageHold === 'fallback-only' && page.body === null) {
1541
+ // The fallback did not translate it: the pair's method
1542
+ // refused it before, so it is not asked — refused now
1543
+ // by the fallback too, or held (no answer, or skipped
1544
+ // by --max-cost).
1545
+ if (page.refusedBy.length > 0) {
1546
+ const err = new Error(
1547
+ `Docusaurus body for ${code}: the fallback (${pairConfig.fallback.method}) failed the whole-page checks too.\n` +
1548
+ ` Nothing was written or cached. ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`
1549
+ );
1550
+ err.heldNext = true;
1551
+ throw err;
1552
+ }
1553
+ const err = new Error('held');
1554
+ err.heldNames = [PAGE_NAME];
1555
+ throw err;
1556
+ }
1557
+ bodyResult = page.body ?? page.primaryBody;
1558
+ if (page.body !== null) wholeBodyKey = page.tmKey;
1559
+ else if (page.refusedBy.length > 0) pageRefusedBy = page.refusedBy;
1560
+ } else {
1561
+ const raw = await runPage(pairConfig);
1562
+ bodyResult = raw ? restoreBlocks(raw, blocks) : null;
1563
+ if (bodyResult) pageRefusedBy = [tmKey];
1047
1564
  }
1048
- const prompt = buildContentPrompt(protectedBody, pairConfig, promptOptions);
1049
- const bodyResult = await translateRawContent(prompt, { apiKey, pairConfig });
1050
1565
  if (!bodyResult) {
1051
1566
  throw new Error(
1052
- `Docusaurus body for ${code}: translation returned no results.\n` +
1567
+ `Docusaurus body for ${code}: translation returned no results`
1568
+ + `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
1053
1569
  ' Check your API key and method configuration.'
1054
1570
  );
1055
1571
  }
1056
- translatedBody = restoreBlocks(bodyResult, blocks);
1572
+ translatedBody = bodyResult;
1057
1573
  } else {
1058
1574
  // Block mode: segment the PROTECTED body (placeholders are
1059
1575
  // single tokens, so they can never be split), serve blocks
@@ -1073,96 +1589,162 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1073
1589
  }));
1074
1590
 
1075
1591
  const missed = [];
1592
+ const heldBlocks = []; // refused before: not sent, the source text kept
1593
+ let position = 0; // place among the translatable blocks
1076
1594
  for (const r of rendered) {
1077
1595
  if (r.seg.type !== 'translatable') {
1078
1596
  // Separators + passthrough blocks: copied verbatim, never billed.
1079
1597
  r.out = r.source;
1080
1598
  continue;
1081
1599
  }
1082
- const cached = lookupTMValidated(tm, r.source, code, tmKey,
1083
- (src, c) => !checkContentPreservation(src, c));
1600
+ // Its name in the repeat check (content:<file>#<n>).
1601
+ r.pos = position++;
1602
+ const cached = lookupContentTM(tm, r.source, code, pairConfig);
1084
1603
  if (cached !== null) {
1085
- r.out = cached;
1086
- } else {
1087
- missed.push(r);
1604
+ r.out = cached.text;
1605
+ if (cached.fromFallback) { mixedProducers = true; notePage(pairKey, label); }
1606
+ refusal.filled.add(blockUnit(r.source));
1607
+ continue;
1088
1608
  }
1609
+ const hold = holds.block(r.source);
1610
+ if (hold === 'held') {
1611
+ // Refused before by this method (and its fallback): the
1612
+ // source text stays, nothing is sent or billed.
1613
+ r.out = r.source;
1614
+ heldBlocks.push(r);
1615
+ continue;
1616
+ }
1617
+ r.fallbackOnly = hold === 'fallback-only';
1618
+ missed.push(r);
1619
+ }
1620
+ if (heldBlocks.length > 0) {
1621
+ bodyUsedFallback = true;
1622
+ const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
1623
+ refusal.held.push(...names);
1624
+ output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names }));
1089
1625
  }
1090
1626
 
1091
1627
  if (missed.length > 0) {
1092
- if (!apiKey) {
1093
- throw new Error(
1094
- `Docusaurus body translation for ${code}: no API key available.\n` +
1095
- ' Set OPENROUTER_API_KEY in .env.local to translate content.'
1096
- );
1097
- }
1628
+ await requirePairReady(pairKey, pairConfig, code, 'body translation');
1098
1629
  // Self-repair ladder (translateBlockBatchResilient): full
1099
- // batch → one missing-segments-only retry → honest
1100
- // '[EN] '-prefixed source for anything still missing. A
1630
+ // batch → one missing-segments-only retry → (with a
1631
+ // fallback method: one batch through it for every block the
1632
+ // primary dropped or damaged — lib/fallback.js) → honest
1633
+ // the source text, unmarked, for anything still missing. A
1101
1634
  // duplicate/unknown marker or an empty first response
1102
- // still fails the file whole.
1103
- const { blocks: translatedBlocks, fellBack } =
1104
- await translateBlockBatchResilient({
1105
- texts: missed.map(r => r.seg.text),
1106
- buildPrompt: (texts) => buildBlockBatchPrompt(
1107
- texts, pairConfig, { ...promptOptions, pageTitle }),
1108
- callModel: (prompt) => translateRawContent(prompt, { apiKey, pairConfig }),
1109
- fallbackPrefix: config.fallbackPrefix,
1110
- });
1111
- const fellBackSet = new Set(fellBack);
1635
+ // still fails the file whole when no fallback rescues it.
1636
+ const outcome = await translateBlocksWithFallback({
1637
+ missed,
1638
+ blocks,
1639
+ pairConfig,
1640
+ budget: fallbackBudget,
1641
+ sharedOutputs: sharedOutputsFor(code),
1642
+ label: docLabel,
1643
+ runBatch: (texts, cfg) => translateBlockBatchResilient({
1644
+ texts,
1645
+ buildPrompt: (t) => buildBlockBatchPrompt(t, cfg, { ...promptOptions, pageTitle }),
1646
+ callModel: (prompt) => translateRawContent(prompt, { apiKey, pairConfig: cfg }),
1647
+ }),
1648
+ fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
1649
+ });
1650
+ tallyContent(pairKey, outcome.report, label);
1651
+ // Refused blocks are remembered; translated ones drop a record.
1652
+ const fellSet = new Set(outcome.fellBack);
1653
+ const newlyHeld = [];
1654
+ missed.forEach((r, i) => {
1655
+ if (!fellSet.has(i)) { refusal.filled.add(blockUnit(r.source)); return; }
1656
+ const methods = outcome.refusedBy?.[i] || [];
1657
+ if (methods.length > 0) {
1658
+ refusal.refused.push({ unit: blockUnit(r.source), source: r.source, methods });
1659
+ newlyHeld.push(i);
1660
+ }
1661
+ });
1662
+ if (outcome.fromFallback > 0) mixedProducers = true;
1663
+ if ((outcome.sharedOutput || []).length > 0) {
1664
+ output.warn(
1665
+ `Docusaurus body for ${code}: ${outcome.sharedOutput.length} block(s) of ${dirName}/${relPath} came back as the same text ` +
1666
+ 'the model gave for other, different source strings (a memorized sentence, not a translation) — refused.'
1667
+ );
1668
+ }
1669
+ if ((outcome.refused || []).length > 0) {
1670
+ output.warn(
1671
+ `Docusaurus body for ${code}: ${outcome.refused.length} block(s) of ${dirName}/${relPath} refused by the quality gate, also when asked again with the reason — `
1672
+ + outcome.refused.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
1673
+ + `${outcome.refused.length > 3 ? '; …' : ''}.`
1674
+ );
1675
+ }
1676
+ const { fellBack } = outcome;
1112
1677
  if (fellBack.length > 0) {
1113
1678
  bodyUsedFallback = true;
1679
+ const refusedHere = (outcome.sharedOutput || []).length + (outcome.refused || []).length;
1680
+ const why = outcome.report?.attempted > 0
1681
+ ? `neither the primary (${pairConfig.method}) nor its fallback (${pairConfig.fallback.method}) translated safely`
1682
+ : refusedHere >= fellBack.length ? 'refused (above)'
1683
+ : refusedHere > 0 ? `refused (${refusedHere}, above) or missing from the model response after a retry`
1684
+ : 'missing from the model response after a retry';
1114
1685
  output.warn(
1115
1686
  `Docusaurus body for ${code}: ${fellBack.length} of ${missed.length} ` +
1116
- `block(s) missing from the model response after a retry — written as ` +
1117
- `'${config.fallbackPrefix}'-prefixed source. Not cached, lock not ` +
1118
- `advanced: the next sync retries just those block(s).`
1687
+ `block(s) ${why} — left in the source language, unmarked (\`champollion status\` lists them). ` +
1688
+ describeFallenBack({
1689
+ file: label, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: outcome.refusedBy,
1690
+ })
1119
1691
  );
1120
1692
  }
1121
- missed.forEach((r, i) => {
1122
- r.out = restoreBlocks(translatedBlocks[i], blocks);
1123
- // Fallen-back segments are never TM-cached — an error
1124
- // cached is an error forever (they re-bill next sync).
1125
- if (!fellBackSet.has(i)) {
1126
- pendingBlockStores.push({ source: r.source, translation: r.out });
1127
- }
1128
- });
1693
+ // Fallen-back segments are never TM-cached — an error
1694
+ // cached is an error forever (they re-bill next sync).
1695
+ missed.forEach((r, i) => { r.out = outcome.outs[i]; });
1696
+ pendingBlockStores.push(...outcome.stores);
1129
1697
  }
1130
1698
 
1131
1699
  // Reassemble in order with the source's exact separators.
1132
1700
  translatedBody = rendered.map(r => r.out).join('');
1133
1701
  }
1134
1702
 
1703
+ // A page translated whole that fails the checks below is
1704
+ // remembered as refused: the next plain sync does not send it
1705
+ // to the same method again (lib/content-refusals.js "page").
1706
+ const refusePage = (err) => {
1707
+ if (!pageRefusedBy) return err;
1708
+ if (!refusal.refused.some(r => r.unit === PAGE_UNIT)) {
1709
+ refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: pageRefusedBy });
1710
+ }
1711
+ err.message += `\n ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`;
1712
+ err.heldNext = true;
1713
+ return err;
1714
+ };
1715
+
1135
1716
  // Orphaned-placeholder check on the REASSEMBLED body — the
1136
1717
  // same gate for both modes.
1137
1718
  if (hasOrphanedPlaceholders(translatedBody)) {
1138
- throw new Error(
1719
+ throw refusePage(new Error(
1139
1720
  `Docusaurus body for ${code}: placeholder corruption detected.\n` +
1140
1721
  ' Code blocks were corrupted during translation.'
1141
- );
1722
+ ));
1142
1723
  }
1143
1724
 
1144
1725
  // Content-preservation check on the REASSEMBLED body — same
1145
1726
  // lane, same reasoning as the front matter above. Skipped for a
1146
- // fallback body: it deliberately carries '[EN] '-prefixed
1727
+ // fallback body: it deliberately carries untranslated
1147
1728
  // source text and is neither cached nor lock-advanced already.
1148
1729
  const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
1149
1730
  if (bodyHollowed) {
1150
- throw new Error(
1731
+ throw refusePage(new Error(
1151
1732
  `Docusaurus body for ${code}: ${bodyHollowed.reason}.\n` +
1152
1733
  ' Nothing was written or cached. If this is a low-coverage target\n' +
1153
1734
  ' language, the model has no vocabulary for this text.'
1154
- );
1735
+ ));
1155
1736
  }
1737
+ if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
1156
1738
 
1157
1739
  // Store per-block AND whole-body entries only after the check
1158
1740
  // passes (whole-body makes reverts/lock-loss re-runs free). A
1159
1741
  // fallback body is NEVER stored whole — it contains
1160
- // untranslated '[EN] ' text.
1742
+ // untranslated source text.
1161
1743
  for (const s of pendingBlockStores) {
1162
- storeTM(tm, s.source, code, tmKey, s.translation);
1744
+ storeTM(tm, s.source, code, s.tmKey, s.translation);
1163
1745
  }
1164
- if (!bodyUsedFallback) {
1165
- storeTM(tm, body, code, tmKey, translatedBody);
1746
+ if (!bodyUsedFallback && !mixedProducers) {
1747
+ storeTM(tm, body, code, wholeBodyKey, translatedBody);
1166
1748
  }
1167
1749
  }
1168
1750
  }
@@ -1180,16 +1762,48 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1180
1762
  // keeps its old entry (or none), so it re-fires next sync — as
1181
1763
  // does a fallback body (its re-fire is TM-cheap: only the
1182
1764
  // fallen-back segment re-bills).
1183
- if (!bodyUsedFallback) {
1184
- updatedDocuManifest[manifestKey] = sourceHash;
1185
- manifestDirty = true;
1186
- }
1765
+ updatedDocuManifest[manifestKey] = !bodyUsedFallback && !fieldsLeftInSource
1766
+ ? sourceHash : pendingLockValue(sourceHash);
1767
+ manifestDirty = true;
1187
1768
 
1188
1769
  } catch (contentErr) {
1770
+ if (contentErr.heldNames) {
1771
+ // A front-matter field refused before: held back, not failed —
1772
+ // nothing was sent, nothing written, the lock not advanced.
1773
+ const refusal = refusalStates.get(manifestKey);
1774
+ refusal.held.push(...contentErr.heldNames);
1775
+ refusal.pageHeld = true;
1776
+ settleRefusals();
1777
+ output.warn(describeHeld({
1778
+ file: `${dirName}/${relPath}`, code, pairKey, pairConfig, names: contentErr.heldNames, pageHeld: true,
1779
+ }));
1780
+ completed++;
1781
+ output.raw(` [${completed}/${totalWork}] ${dirName}/${relPath} → ${code} [HELD]`);
1782
+ output.event('file', { lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: 'held-refused', held: contentErr.heldNames.length, action });
1783
+ if (completed % MANIFEST_WRITE_INTERVAL === 0) writeManifestIfDirty();
1784
+ return;
1785
+ }
1786
+ itemFailed = true;
1189
1787
  contentFailures++;
1190
1788
  output.error(`${dirName}/${relPath} → ${code} — ${contentErr.message}`);
1789
+ // Nothing was written for this item, and its lock entry was not
1790
+ // advanced. If the lock already matches this source, the file on
1791
+ // disk is the PREVIOUS translation of the same text (a forced
1792
+ // re-translate that failed): it stays, and a plain sync will not
1793
+ // redo it. Otherwise the next sync retries it.
1794
+ const keptPrevious = updatedDocuManifest[manifestKey] === sourceHash && fs.existsSync(targetPath);
1795
+ failedItems.push({
1796
+ file: `${dirName}/${relPath}`,
1797
+ locale: code,
1798
+ error: contentErr.message,
1799
+ // held-back: refused by the quality gate — the next plain sync
1800
+ // does not send it again (lib/content-refusals.js).
1801
+ state: keptPrevious ? 'previous-translation-kept' : contentErr.heldNext ? 'held-back' : 'will-retry',
1802
+ });
1191
1803
  }
1192
1804
 
1805
+ settleRefusals();
1806
+
1193
1807
  // Progress reporting
1194
1808
  completed++;
1195
1809
  const pct = Math.round(100 * completed / totalWork);
@@ -1198,13 +1812,16 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1198
1812
  const remainingMs = msPerItem * (totalWork - completed);
1199
1813
  const remainingSec = Math.ceil(remainingMs / 1000);
1200
1814
  const etaStr = remainingSec > 5 ? ` (~${remainingSec}s left)` : '';
1201
- const tag = contentFailures > 0 && completed === totalWork
1815
+ // The tag describes THIS item. (It used to read the run-wide failure
1816
+ // count, so whichever item finished last was tagged FAIL.)
1817
+ const displayTag = itemFailed
1202
1818
  ? 'FAIL'
1203
1819
  : action === 're-translate' ? 'RE-TRANSLATE' : action === 'changed' ? 'CHANGED' : 'OK';
1204
- // Show [FAIL] for items that just errored (contentErr was caught above)
1205
- const itemFailed = updatedDocuManifest[manifestKey] !== sourceHash;
1206
- const displayTag = itemFailed && !dryRun ? 'FAIL' : tag;
1207
1820
  output.raw(` [${completed}/${totalWork}] (${pct}%) ${dirName}/${relPath} → ${code} [${displayTag}]${etaStr}`);
1821
+ output.event('file', {
1822
+ lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code,
1823
+ status: itemFailed ? 'failed' : dryRun ? 'would-translate' : 'translated', action,
1824
+ });
1208
1825
 
1209
1826
  // Incremental manifest write
1210
1827
  if (completed % MANIFEST_WRITE_INTERVAL === 0) {
@@ -1231,26 +1848,122 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
1231
1848
  output.ok(`${action} ${totalContent} content file(s)${retranslateNote}, ${totalContentSkipped} unchanged`);
1232
1849
  }
1233
1850
 
1234
- // Fail loud if any content translations failed — do NOT exit 0
1235
- if (contentFailures > 0) {
1236
- throw new Error(
1237
- `${contentFailures} content translation(s) failed. ` +
1238
- `Re-run sync to retry failed files (completed files are cached).`
1851
+ contentTranslated = totalContent;
1852
+ contentFailedItems = failedItems;
1853
+ for (const st of refusalStates.values()) {
1854
+ if (st.held.length > 0) {
1855
+ contentHeldBack += st.held.length;
1856
+ contentHeldBackItems.push({ file: st.file, locale: st.locale, units: st.held, ...(st.pageHeld && { pageNotWritten: true }) });
1857
+ }
1858
+ contentRefused += new Set(st.refused.map(r => r.unit)).size;
1859
+ }
1860
+ if (contentHeldBack > 0) {
1861
+ output.warn(
1862
+ `${contentHeldBack} content block(s)/field(s) held back in ${contentHeldBackItems.length} translation(s) — the quality gate refused `
1863
+ + 'the method\'s translation of their current text before; not sent, not billed (listed above). Ask again for all of them: '
1864
+ + '`champollion sync --redo content`; for one page: `champollion sync --redo files:<page>`.'
1239
1865
  );
1240
1866
  }
1867
+ for (const [pairKey, tally] of contentFallbackTallies) {
1868
+ printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)', producedLabel: 'pages' });
1869
+ warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)' });
1870
+ }
1871
+ if (failedItems.length > 0) {
1872
+ output.raw('');
1873
+ output.raw(` Failed content translations (${failedItems.length}):`);
1874
+ for (const f of failedItems.sort((a, b) => (a.file + a.locale).localeCompare(b.file + b.locale))) {
1875
+ const state = f.state === 'previous-translation-kept'
1876
+ ? 'previous translation kept (it matches the current source); re-run with --force-content to try again'
1877
+ : f.state === 'held-back'
1878
+ ? `refused by the quality gate — held back: the next sync does not send it again (\`--redo files:${f.file}\` asks again)`
1879
+ : 'not recorded as done — the next sync retries it';
1880
+ output.raw(` ${f.file} → ${f.locale}: ${state}`);
1881
+ }
1882
+ }
1241
1883
  }
1242
1884
 
1243
1885
  // Save TM if it was mutated during this Docusaurus sync (stores OR
1244
1886
  // evictions — a size check would miss eviction-only runs and same-key
1245
- // replacements). Skip when --no-tm is active.
1887
+ // replacements). Skip when --no-tm is active. Saved BEFORE any failure is
1888
+ // raised: the files that succeeded paid for their translations, and
1889
+ // throwing first discarded every one of them from the cache.
1246
1890
  if (!dryRun && !noTM && isTMDirty(tm)) {
1247
- const tmFinalSize = tmSize(tm);
1248
- const delta = tmFinalSize - tmInitialSize;
1249
1891
  saveTM(cwd, tm);
1250
- output.info(`[TM] Saved ${tmFinalSize} entries (${delta >= 0 ? '+' + delta : delta} this sync)`);
1892
+ output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
1893
+ }
1894
+
1895
+ // Failures are reported, not thrown: a partly failed run still did real
1896
+ // work, and the exit code says so (2 = partial, 1 = nothing succeeded).
1897
+ if (contentFailures > 0) {
1898
+ output.error(
1899
+ `${contentFailures} content translation(s) failed (listed above). ` +
1900
+ 'Completed files and their translations are saved; re-run sync to retry.'
1901
+ );
1251
1902
  }
1252
1903
 
1904
+ // The --max-cost verdict, said once — at the end, before the summary
1905
+ // (lib/cost-report.js reportDryRunMaxCost).
1906
+ if (dryMaxCost) output[dryMaxCost.level](dryMaxCost.message);
1907
+
1908
+ // One machine-readable record of the run — the Docusaurus path never had
1909
+ // one, so --json consumers saw nothing to tell success from failure.
1910
+ output.summary({
1911
+ command: 'sync',
1912
+ format: 'docusaurus',
1913
+ dryRun,
1914
+ totalProcessed: totalJSONKeys,
1915
+ totalFailed: totalJSONFailed,
1916
+ totalCopied: totalJSONCopied,
1917
+ // UI strings held back: the gate refused this method's translation of
1918
+ // their current text before — not sent, not billed (lib/locale-state.js).
1919
+ // A dry run sends nothing either way: said above, not counted (as on
1920
+ // the key-value path).
1921
+ totalHeld: dryRun ? 0 : totalJSONHeld,
1922
+ // What each pair's fallback method did for JSON keys (pairs with a
1923
+ // fallback, real runs); Markdown segments are under content.fallback.
1924
+ ...(keyFallbackTallies.size > 0 && {
1925
+ fallback: [...keyFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
1926
+ }),
1927
+ content: {
1928
+ translated: contentTranslated,
1929
+ skipped: contentScan.totalContentSkipped,
1930
+ failed: contentFailedItems.length,
1931
+ failedItems: contentFailedItems,
1932
+ // Refused before by the method, not sent this run (lib/content-refusals.js),
1933
+ // and refused this run (held back from the next).
1934
+ heldBack: contentHeldBack,
1935
+ heldBackItems: contentHeldBackItems,
1936
+ refused: contentRefused,
1937
+ ...(contentFallbackTallies.size > 0 && {
1938
+ fallback: [...contentFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
1939
+ }),
1940
+ },
1941
+ costEstimate,
1942
+ // Dry runs with --max-cost: whether the real run would stop at the cap.
1943
+ ...(dryMaxCost && { maxCost: {
1944
+ cap: dryMaxCost.cap, estimatedCost: dryMaxCost.estimatedCost, wouldStop: dryMaxCost.wouldStop,
1945
+ ...(dryMaxCost.wouldStop && { exitCode: 2, reason: dryMaxCost.reason }),
1946
+ ...(dryMaxCost.stopsEarlier && { exitCode: 1, stopsEarlier: dryMaxCost.stopsEarlier }),
1947
+ } }),
1948
+ // Keys named for a redo that match no UI string (the run exits 1), each
1949
+ // with the closest ids that exist.
1950
+ ...(unmatchedNamed.length > 0 && { unmatchedKeys: unmatchedKeysSummary(unmatchedNamed) }),
1951
+ });
1253
1952
  output.raw('');
1953
+ // Named keys that matched nothing: said again last, where a reader (or CI)
1954
+ // looks for the verdict — and the run fails (lib/commands/sync.js).
1955
+ reportUnmatchedKeys(unmatchedNamed, cliArgs);
1956
+ return {
1957
+ totalProcessed: totalJSONKeys,
1958
+ totalFailed: totalJSONFailed,
1959
+ totalCopied: totalJSONCopied,
1960
+ totalHeld: dryRun ? 0 : totalJSONHeld,
1961
+ contentTranslated,
1962
+ contentFailed: contentFailedItems.length,
1963
+ contentHeldBack,
1964
+ contentRefused,
1965
+ unmatchedKeys: unmatchedNamed.map(m => m.name),
1966
+ };
1254
1967
  }
1255
1968
 
1256
1969
  export { runDocusaurusSync, discoverDocusaurusJSONFiles };