champollion 0.3.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +82 -37
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1709 -101
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +230 -17
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +284 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1136 -345
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +727 -79
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +975 -262
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +1102 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +575 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +257 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +71 -129
- package/lib/methods/llm.js +87 -33
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +102 -55
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +24 -8
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +996 -21
- package/lib/verify.js +1332 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/licenses.json +99 -3
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/retired-model-aliases.json +9 -0
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/human-services.schema.json +1 -1
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/schemas/retired-model-aliases.schema.json +32 -0
- package/shared/docent/corpus.json +0 -11739
- package/shared/model-aliases.json +0 -7
- package/shared/schemas/model-aliases.schema.json +0 -27
package/lib/docusaurus-sync.js
CHANGED
|
@@ -29,29 +29,54 @@ import {
|
|
|
29
29
|
discoverDocusaurusContentFiles, getDocusaurusTargetPath,
|
|
30
30
|
findUntranslatableNestedFields,
|
|
31
31
|
} from './content.js';
|
|
32
|
-
import { translateRawContent } from './translate.js';
|
|
32
|
+
import { translateRawContent, getMethod } from './translate.js';
|
|
33
33
|
import {
|
|
34
34
|
splitBlocks, buildBlockBatchPrompt, parseBlockBatchResponse,
|
|
35
35
|
translateBlockBatchResilient,
|
|
36
36
|
assertSegmentationMode,
|
|
37
37
|
} from './segment.js';
|
|
38
38
|
import { DEFAULT_REGISTERS } from './registers.js';
|
|
39
|
-
import { DEFAULT_JSON_CONCURRENCY
|
|
39
|
+
import { DEFAULT_JSON_CONCURRENCY } from './config.js';
|
|
40
40
|
import { compileNoTranslate } from './no-translate.js';
|
|
41
|
-
import { checkContentPreservation } from './validate.js';
|
|
41
|
+
import { checkContentPreservation, contentGateFault, SharedOutputIndex } from './validate.js';
|
|
42
42
|
import { convertScript, applyScriptFallback } from './scripts.js';
|
|
43
43
|
import { pMap } from './concurrent.js';
|
|
44
44
|
import {
|
|
45
45
|
loadTM, saveTM, tmSize, isTMDirty,
|
|
46
|
-
lookupTM,
|
|
46
|
+
lookupTM, peekTM, storeTM, evictTM, partitionByTM, tmMethodKey, setModelCarryover, bypassTMFor,
|
|
47
|
+
describeTMChanges, adoptLegacyCoachingKeys,
|
|
47
48
|
} from './tm.js';
|
|
48
49
|
import {
|
|
49
|
-
|
|
50
|
+
readLock, writeManifest, detectChangedKeys, hashValue,
|
|
50
51
|
} from './hash.js';
|
|
52
|
+
import {
|
|
53
|
+
LockState, planQueue, recordRefusal, describeHeldKeys, describeFallbackOnlyKeys, keyFateNote, describeHeldNext,
|
|
54
|
+
shortSourceHash,
|
|
55
|
+
} from './locale-state.js';
|
|
56
|
+
import { redoCommand } from './verify.js';
|
|
51
57
|
import { CONTENT_LOCK_FILENAME } from './content-sync.js';
|
|
52
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
parseMaxCost, abortForMaxCost, maxCostVerdict, reportDryRunMaxCost, printCostTable, warnModelSwitchStrandedTM, priceContentChars,
|
|
60
|
+
preflightStopReason,
|
|
61
|
+
summarizeEstimate,
|
|
62
|
+
} from './cost-report.js';
|
|
63
|
+
import { billableContentChars, translatableBlockSources } from './content-estimate.js';
|
|
64
|
+
import { compileFileScope } from './file-scope.js';
|
|
53
65
|
import { output } from './output.js';
|
|
54
|
-
import {
|
|
66
|
+
import { missingKeyAdvice } from './missing-key.js';
|
|
67
|
+
import { translateWithFallback } from './translate-pair.js';
|
|
68
|
+
import {
|
|
69
|
+
tmKeysForPair, tmHoldsValue, createFallbackBudget, newFallbackReport, addToTally,
|
|
70
|
+
fallbackSummary, printFallbackReport, warnFallbackMajority, lookupContentTM, serveFieldsFromFallbackCache,
|
|
71
|
+
translateFieldsWithFallback, translateBlocksWithFallback, translatePageWithFallback,
|
|
72
|
+
refuseRepeatedCachedPage,
|
|
73
|
+
} from './fallback.js';
|
|
74
|
+
import { walkFiles } from './locale-layout.js';
|
|
75
|
+
import {
|
|
76
|
+
readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
|
|
77
|
+
describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock,
|
|
78
|
+
} from './content-refusals.js';
|
|
79
|
+
import { applyNamedKeyRule, reportUnmatchedKeys, unmatchedKeysSummary, reportNamedFromCache } from './named-keys.js';
|
|
55
80
|
|
|
56
81
|
/**
|
|
57
82
|
* Discover all JSON locale files in a Docusaurus i18n source directory.
|
|
@@ -63,20 +88,64 @@ import { translateAndValidate } from './translate-pair.js';
|
|
|
63
88
|
* @returns {string[]} Absolute paths to JSON files
|
|
64
89
|
*/
|
|
65
90
|
function discoverDocusaurusJSONFiles(sourceLocaleDir) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
91
|
+
// The shared folder-per-locale walk (lib/locale-layout.js). Docusaurus
|
|
92
|
+
// keeps its historical reach: hidden entries are NOT skipped here.
|
|
93
|
+
return walkFiles(sourceLocaleDir, name => name.endsWith('.json'), { skipHidden: false });
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* What one (UI-string file × locale) queue becomes once the lock's refusal
|
|
98
|
+
* records are consulted — the key-value lane's rule (lib/locale-state.js
|
|
99
|
+
* planQueue), shared by the estimate and the sync so the estimate prices
|
|
100
|
+
* exactly what the run sends. A key the gate refused for its current source
|
|
101
|
+
* text is held back from the method that refused it (its fallback, when it
|
|
102
|
+
* has not refused it, is asked instead) unless it is named for a redo
|
|
103
|
+
* (`--redo keys:`), queued by `--redo all`, or the run is `--fresh`.
|
|
104
|
+
*
|
|
105
|
+
* Docusaurus records no written values, so no value counts as a person's
|
|
106
|
+
* edit here (a bulk redo re-translates every UI string, as it always did),
|
|
107
|
+
* and there is no pending retry: a UI string refused under a redo is held
|
|
108
|
+
* back by the next plain sync, as a content block is.
|
|
109
|
+
*
|
|
110
|
+
* @param {object} p
|
|
111
|
+
* @param {object} p.diff - diffLocale result for the file × locale
|
|
112
|
+
* @param {object} p.sourceFlat
|
|
113
|
+
* @param {object} p.targetFlat
|
|
114
|
+
* @param {string} p.nsPrefix - "docusaurus:<relPath>:" (the lock's key space)
|
|
115
|
+
* @param {object} p.localeState - LockState.of(code) / .peek(code)
|
|
116
|
+
* @param {object} p.pairConfig
|
|
117
|
+
* @param {{ named: Set<string>, bulk: boolean, fresh: boolean }} p.redo
|
|
118
|
+
* @param {string} p.fallbackPrefix
|
|
119
|
+
* @returns {{ held: string[], heldFromPrimary: string[] }}
|
|
120
|
+
*/
|
|
121
|
+
function planUIStrings({ diff, sourceFlat, targetFlat, nsPrefix, localeState, pairConfig, redo, fallbackPrefix }) {
|
|
122
|
+
const plan = planQueue({
|
|
123
|
+
diff, sourceFlat, targetFlat, lockKeyOf: (k) => nsPrefix + k, localeState,
|
|
124
|
+
named: redo.named, bulk: redo.bulk, pending: new Set(), fresh: redo.fresh, pairConfig,
|
|
125
|
+
classify: () => 'machine',
|
|
126
|
+
fallbackPrefix,
|
|
127
|
+
});
|
|
128
|
+
return { held: plan.held, heldFromPrimary: plan.heldFromPrimary };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* UI strings refused for an older source text that the file still holds as
|
|
133
|
+
* the English copy of that text (Docusaurus needs every id, so an untranslated
|
|
134
|
+
* one is written as its source message): the source was edited since, which
|
|
135
|
+
* lifts the hold — queued as changed. Without this the edit was invisible: the
|
|
136
|
+
* refused string never had its source hash recorded, and the old English no
|
|
137
|
+
* longer equals the new source, so nothing queued it.
|
|
138
|
+
*
|
|
139
|
+
* @returns {string[]}
|
|
140
|
+
*/
|
|
141
|
+
function editedSinceRefused(sourceFlat, targetFlat, nsPrefix, localeState) {
|
|
142
|
+
const out = [];
|
|
143
|
+
for (const [k, src] of Object.entries(sourceFlat)) {
|
|
144
|
+
const rec = localeState.refused[nsPrefix + k];
|
|
145
|
+
if (!rec || typeof src !== 'string' || typeof targetFlat[k] !== 'string') continue;
|
|
146
|
+
if (rec.source !== shortSourceHash(src) && shortSourceHash(targetFlat[k]) === rec.source) out.push(k);
|
|
77
147
|
}
|
|
78
|
-
|
|
79
|
-
return files.sort();
|
|
148
|
+
return out;
|
|
80
149
|
}
|
|
81
150
|
|
|
82
151
|
/**
|
|
@@ -97,12 +166,13 @@ function discoverDocusaurusJSONFiles(sourceLocaleDir) {
|
|
|
97
166
|
* @param {object} config - Resolved config (localesDir)
|
|
98
167
|
* @param {object} manifest - Content lock manifest (hash per file × locale)
|
|
99
168
|
* @param {boolean} forceContent - --force-content: re-process up-to-date files
|
|
169
|
+
* @param {import('./file-scope.js').FileScope|null} [fileScope] - --files / --retranslate
|
|
100
170
|
* too (action 're-translate'), WITHOUT clearing the manifest — a target
|
|
101
171
|
* with no lock entry and no '[EN] ' markers is a genuine hand-translated
|
|
102
172
|
* file, and force must never overwrite human work with machine output.
|
|
103
173
|
* @returns {{ workItems: Array<object>, totalContentSkipped: number, recordedHashes: object }}
|
|
104
174
|
*/
|
|
105
|
-
function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent) {
|
|
175
|
+
function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent, fileScope = null) {
|
|
106
176
|
const workItems = [];
|
|
107
177
|
let totalContentSkipped = 0;
|
|
108
178
|
const recordedHashes = {};
|
|
@@ -115,6 +185,9 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
115
185
|
|
|
116
186
|
for (const sourcePath of sourceFiles) {
|
|
117
187
|
const relPath = path.relative(sourceContentDir, sourcePath);
|
|
188
|
+
const label = `${dirName}/${relPath}`;
|
|
189
|
+
if (fileScope && !fileScope.includes(label)) continue;
|
|
190
|
+
const retranslate = fileScope ? fileScope.retranslates(label) : false;
|
|
118
191
|
|
|
119
192
|
// Read + parse source file once per source path. The staleness hash
|
|
120
193
|
// covers the RAW file (same as the Hugo twin's hashFileContent): a
|
|
@@ -142,7 +215,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
142
215
|
}
|
|
143
216
|
const { parsed, fieldsToTranslate, pageTitle, sourceHash } = sourceCache.get(sourcePath);
|
|
144
217
|
|
|
145
|
-
for (const [, pairConfig] of pairEntries) {
|
|
218
|
+
for (const [pairKey, pairConfig] of pairEntries) {
|
|
146
219
|
const code = pairConfig.target;
|
|
147
220
|
const targetPath = getDocusaurusTargetPath(
|
|
148
221
|
sourcePath, sourceContentDir, code, config.localesDir, pluginName
|
|
@@ -156,7 +229,11 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
156
229
|
const manifestKey = `docusaurus:${dirName}/${relPath}:${code}`;
|
|
157
230
|
let action = 'new'; // 'new' | 'changed' | 're-translate'
|
|
158
231
|
|
|
159
|
-
if (
|
|
232
|
+
if (retranslate) {
|
|
233
|
+
// --retranslate: named by the operator — no lock skip, no
|
|
234
|
+
// adoption; the caller makes its text miss the TM.
|
|
235
|
+
if (fs.existsSync(targetPath)) action = 're-translate';
|
|
236
|
+
} else if (fs.existsSync(targetPath)) {
|
|
160
237
|
const storedHash = manifest[manifestKey];
|
|
161
238
|
if (storedHash) {
|
|
162
239
|
if (storedHash === sourceHash) {
|
|
@@ -171,7 +248,9 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
171
248
|
action = 'changed';
|
|
172
249
|
}
|
|
173
250
|
} else {
|
|
174
|
-
// No stored hash — check for [EN] fallback markers
|
|
251
|
+
// No stored hash — check for legacy [EN] fallback markers (pages
|
|
252
|
+
// written before 2026-10-05; now such a page has a pending:<hash>
|
|
253
|
+
// lock entry instead). This runs
|
|
175
254
|
// even under --force-content: a target with no lock entry and
|
|
176
255
|
// no [EN] markers is a genuine hand-translated file, and force
|
|
177
256
|
// must never overwrite human work with machine output.
|
|
@@ -190,7 +269,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
190
269
|
workItems.push({
|
|
191
270
|
sourcePath, parsed, fieldsToTranslate, pageTitle, sourceHash,
|
|
192
271
|
relPath, dirName,
|
|
193
|
-
pairConfig, code, targetPath, manifestKey, pluginName, action,
|
|
272
|
+
pairKey, pairConfig, code, targetPath, manifestKey, pluginName, action, retranslate,
|
|
194
273
|
});
|
|
195
274
|
}
|
|
196
275
|
}
|
|
@@ -240,7 +319,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
240
319
|
* Structured estimate, or null when estimation itself failed (callers
|
|
241
320
|
* with --max-cost must fail safe: unknown ≠ free)
|
|
242
321
|
*/
|
|
243
|
-
async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate) {
|
|
322
|
+
async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate, lockState = null, redo = null) {
|
|
244
323
|
try {
|
|
245
324
|
const { estimateCost } = await import('./pairs.js');
|
|
246
325
|
const costEstimates = [];
|
|
@@ -251,6 +330,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
251
330
|
const sourceJSONFiles = discoverDocusaurusJSONFiles(sourceLocaleDir);
|
|
252
331
|
const missesByPair = new Map(); // pairKey → miss count
|
|
253
332
|
const hitsByPair = new Map(); // pairKey → TM hit count
|
|
333
|
+
const heldByPair = new Map(); // pairKey → keys held back (refused before; not sent)
|
|
254
334
|
|
|
255
335
|
for (const sourceFilePath of sourceJSONFiles) {
|
|
256
336
|
const relPath = path.relative(sourceLocaleDir, sourceFilePath);
|
|
@@ -283,26 +363,45 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
283
363
|
|
|
284
364
|
// Same confirmed-echo suppression as the Phase-1 sync below, so the
|
|
285
365
|
// estimate prices exactly the keys the run will actually queue.
|
|
286
|
-
|
|
366
|
+
// With a fallback, its cached values count too (lib/fallback.js).
|
|
367
|
+
const estTmKeys = tmKeysForPair(pairConfig);
|
|
368
|
+
const localeChanged = lockState
|
|
369
|
+
? [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, lockState.peek(code))])]
|
|
370
|
+
: changedKeys;
|
|
287
371
|
const diff = diffLocale(
|
|
288
372
|
sourceFlat, existingFlat, config.fallbackPrefix,
|
|
289
|
-
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys,
|
|
290
|
-
(key, sourceValue) =>
|
|
373
|
+
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
|
|
374
|
+
(key, sourceValue) => tmHoldsValue(tm, sourceValue, code, estTmKeys, sourceValue),
|
|
291
375
|
noTranslate.active ? noTranslate.matches : null
|
|
292
376
|
);
|
|
293
377
|
const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
|
|
294
378
|
if (stringKeys.length === 0) continue;
|
|
295
379
|
|
|
296
|
-
|
|
380
|
+
// Refused before (the lock's records): never sent to the method that
|
|
381
|
+
// refused them — held keys cost nothing; a fallback is not priced.
|
|
382
|
+
const notSent = new Set();
|
|
383
|
+
if (lockState && redo) {
|
|
384
|
+
const plan = planUIStrings({
|
|
385
|
+
diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState: lockState.peek(code), pairConfig, redo,
|
|
386
|
+
fallbackPrefix: config.fallbackPrefix,
|
|
387
|
+
});
|
|
388
|
+
for (const k of [...plan.held, ...plan.heldFromPrimary]) notSent.add(k);
|
|
389
|
+
}
|
|
390
|
+
let { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmMethodKey(pairConfig));
|
|
391
|
+
if (estTmKeys.length > 1) misses = misses.filter(k => lookupTM(tm, sourceFlat[k], code, estTmKeys[1]) === null);
|
|
392
|
+
const held = misses.filter(k => notSent.has(k)).length;
|
|
393
|
+
misses = misses.filter(k => !notSent.has(k));
|
|
297
394
|
missesByPair.set(pairKey, (missesByPair.get(pairKey) || 0) + misses.length);
|
|
298
|
-
hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length));
|
|
395
|
+
hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length - held));
|
|
396
|
+
heldByPair.set(pairKey, (heldByPair.get(pairKey) || 0) + held);
|
|
299
397
|
}
|
|
300
398
|
}
|
|
301
399
|
|
|
302
400
|
for (const [pairKey, pairConfig] of pairEntries) {
|
|
303
401
|
const keysToTranslate = missesByPair.get(pairKey) || 0;
|
|
304
402
|
const tmHits = hitsByPair.get(pairKey) || 0;
|
|
305
|
-
|
|
403
|
+
const heldCount = heldByPair.get(pairKey) || 0;
|
|
404
|
+
if (keysToTranslate === 0 && tmHits === 0 && heldCount === 0) continue;
|
|
306
405
|
|
|
307
406
|
if (keysToTranslate > 0) {
|
|
308
407
|
// eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
|
|
@@ -317,8 +416,15 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
317
416
|
method: pairConfig.method || 'llm',
|
|
318
417
|
keys: keysToTranslate,
|
|
319
418
|
tmHits,
|
|
419
|
+
...(heldCount > 0 && { held: heldCount }),
|
|
320
420
|
estimatedCost: estimate.estimatedCost,
|
|
321
421
|
source: estimate.source,
|
|
422
|
+
...(estimate.local && { local: true }),
|
|
423
|
+
...(estimate.note && { note: estimate.note }),
|
|
424
|
+
// The rate it was priced at, its source and date (cost-report.js rateLine).
|
|
425
|
+
...(estimate.rate && { rate: estimate.rate }),
|
|
426
|
+
// No price: the model it was looked up under, so the table can name it.
|
|
427
|
+
...(estimate.estimatedCost === null && estimate.model && { model: estimate.model }),
|
|
322
428
|
});
|
|
323
429
|
} else {
|
|
324
430
|
// Fully TM-covered: zero API calls → a KNOWN $0, even for
|
|
@@ -328,8 +434,9 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
328
434
|
method: pairConfig.method || 'llm',
|
|
329
435
|
keys: 0,
|
|
330
436
|
tmHits,
|
|
437
|
+
...(heldCount > 0 && { held: heldCount }),
|
|
331
438
|
estimatedCost: 0,
|
|
332
|
-
source: 'translation-memory',
|
|
439
|
+
source: tmHits > 0 ? 'translation-memory' : 'nothing-to-send',
|
|
333
440
|
});
|
|
334
441
|
}
|
|
335
442
|
}
|
|
@@ -342,72 +449,51 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
342
449
|
// block sources once (only needed for 'block' mode misses).
|
|
343
450
|
const blockSourcesCache = new Map(); // sourcePath → string[]
|
|
344
451
|
const charsByPair = new Map(); // pairConfig (by reference) → billed source chars
|
|
452
|
+
const roughCharsByPair = new Map(); // pairConfig → all source chars, as if the TM were empty
|
|
345
453
|
const pendingSourceFiles = new Set();
|
|
346
454
|
|
|
347
455
|
for (const item of contentWorkItems) {
|
|
348
456
|
pendingSourceFiles.add(item.sourcePath);
|
|
349
457
|
const code = item.code;
|
|
350
|
-
const
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
// free), then per-block in 'block' mode — the same ladder the
|
|
361
|
-
// sync runs below.
|
|
362
|
-
const body = item.parsed.body;
|
|
363
|
-
if (body.trim() && lookupTM(tm, body, code, tmKey) === null) {
|
|
364
|
-
const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
365
|
-
if (segMode === 'page') {
|
|
366
|
-
chars += body.length;
|
|
367
|
-
} else {
|
|
458
|
+
const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
459
|
+
const { billedChars: chars, totalChars } = billableContentChars({
|
|
460
|
+
tm,
|
|
461
|
+
code,
|
|
462
|
+
tmKey: tmMethodKey(item.pairConfig),
|
|
463
|
+
fallbackTmKey: item.pairConfig.fallback ? tmMethodKey(item.pairConfig.fallback) : null,
|
|
464
|
+
fields: item.fieldsToTranslate,
|
|
465
|
+
body: item.parsed.body,
|
|
466
|
+
segMode,
|
|
467
|
+
blockSources: () => {
|
|
368
468
|
if (!blockSourcesCache.has(item.sourcePath)) {
|
|
369
|
-
|
|
370
|
-
blockSourcesCache.set(
|
|
371
|
-
item.sourcePath,
|
|
372
|
-
splitBlocks(protectedBody)
|
|
373
|
-
.filter(seg => seg.type === 'translatable')
|
|
374
|
-
.map(seg => restoreBlocks(seg.text, blocks))
|
|
375
|
-
);
|
|
376
|
-
}
|
|
377
|
-
for (const source of blockSourcesCache.get(item.sourcePath)) {
|
|
378
|
-
if (lookupTM(tm, source, code, tmKey) === null) chars += source.length;
|
|
469
|
+
blockSourcesCache.set(item.sourcePath, translatableBlockSources(item.parsed.body));
|
|
379
470
|
}
|
|
380
|
-
|
|
381
|
-
|
|
471
|
+
return blockSourcesCache.get(item.sourcePath);
|
|
472
|
+
},
|
|
473
|
+
holds: item.holds || null,
|
|
474
|
+
});
|
|
475
|
+
roughCharsByPair.set(item.pairConfig, (roughCharsByPair.get(item.pairConfig) || 0) + totalChars);
|
|
382
476
|
|
|
383
477
|
if (chars > 0) {
|
|
384
478
|
charsByPair.set(item.pairConfig, (charsByPair.get(item.pairConfig) || 0) + chars);
|
|
385
479
|
}
|
|
386
480
|
}
|
|
387
481
|
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
// Fully TM-covered pairs are a KNOWN $0 (zero API calls) — they
|
|
393
|
-
// must not consult estimateCost, whose unknown-pricing null would
|
|
394
|
-
// wrongly trip hasUnknownCosts and abort a free run under a cap.
|
|
395
|
-
if (!chars) continue;
|
|
396
|
-
const keyEquivalents = Math.ceil(chars / EST_CHARS_PER_KEY);
|
|
397
|
-
// eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
|
|
398
|
-
const estimate = await estimateCost(keyEquivalents, pairConfig);
|
|
399
|
-
if (estimate.estimatedCost !== null) {
|
|
400
|
-
contentCost += estimate.estimatedCost;
|
|
401
|
-
} else {
|
|
402
|
-
contentUnknown = true;
|
|
403
|
-
}
|
|
404
|
-
}
|
|
482
|
+
const billed = await priceContentChars(charsByPair, pairEntries, estimateCost);
|
|
483
|
+
const rough = await priceContentChars(roughCharsByPair, pairEntries, estimateCost);
|
|
484
|
+
const contentCost = billed.cost;
|
|
485
|
+
const contentUnknown = billed.unknown;
|
|
405
486
|
|
|
406
487
|
content = {
|
|
407
488
|
files: pendingSourceFiles.size,
|
|
408
489
|
pendingTranslations: contentWorkItems.length,
|
|
409
490
|
estimatedCost: contentUnknown ? null : contentCost,
|
|
491
|
+
costWithoutTM: rough.unknown ? null : rough.cost,
|
|
410
492
|
rough: true,
|
|
493
|
+
// Which pairs have no price, and why (printCostTable names them).
|
|
494
|
+
...(contentUnknown && { unpriced: billed.unpriced }),
|
|
495
|
+
// The rates the billed part was priced at.
|
|
496
|
+
...(billed.rates.length > 0 && { rates: billed.rates }),
|
|
411
497
|
};
|
|
412
498
|
if (contentUnknown) {
|
|
413
499
|
hasUnknownCosts = true;
|
|
@@ -418,16 +504,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
418
504
|
|
|
419
505
|
printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
|
|
420
506
|
|
|
421
|
-
return
|
|
422
|
-
currency: 'USD',
|
|
423
|
-
pairs: costEstimates,
|
|
424
|
-
keyCost: content && content.estimatedCost !== null
|
|
425
|
-
? totalEstimatedCost - content.estimatedCost
|
|
426
|
-
: totalEstimatedCost,
|
|
427
|
-
content,
|
|
428
|
-
totalEstimatedCost,
|
|
429
|
-
hasUnknownCosts,
|
|
430
|
-
};
|
|
507
|
+
return summarizeEstimate(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
|
|
431
508
|
} catch (costError) {
|
|
432
509
|
// Non-blocking without a cap — warn and continue. Callers enforcing
|
|
433
510
|
// --max-cost must treat the null return as unknown (abort).
|
|
@@ -508,9 +585,48 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
508
585
|
);
|
|
509
586
|
}
|
|
510
587
|
|
|
588
|
+
// Keys named for a redo (--redo keys: / --force-keys) must exist — the
|
|
589
|
+
// rule the key-value path applies (lib/named-keys.js). A name matching no
|
|
590
|
+
// UI string used to re-queue nothing in silence and exit 0. Checked before
|
|
591
|
+
// the preflight, the estimate or any spend: with no name matching nothing
|
|
592
|
+
// runs; with some, those are redone and the run then fails naming the rest.
|
|
593
|
+
// Docusaurus message ids are per file but named bare: a name re-queues
|
|
594
|
+
// that id in every JSON file that has it (as the diff below applies it).
|
|
595
|
+
const unmatchedNamed = applyNamedKeyRule({
|
|
596
|
+
cliArgs, config,
|
|
597
|
+
known: () => {
|
|
598
|
+
const ids = new Set();
|
|
599
|
+
for (const file of discoverDocusaurusJSONFiles(sourceLocaleDir)) {
|
|
600
|
+
for (const key of Object.keys(extractDocusaurusMessages(JSON.parse(fs.readFileSync(file, 'utf-8'))))) {
|
|
601
|
+
if (!isUnsafeKey(key)) ids.add(key);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
return ids;
|
|
605
|
+
},
|
|
606
|
+
});
|
|
607
|
+
|
|
511
608
|
// Thread dryRun/audit into cliArgs so the preflight check can skip
|
|
512
609
|
// when appropriate — same pattern as the main sync path in sync.js
|
|
513
|
-
const { apiKey, pairEntries } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
|
|
610
|
+
const { apiKey, pairEntries, preflightFailures = [] } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
|
|
611
|
+
|
|
612
|
+
// Content needs what the pair's own method needs — a local model needs no
|
|
613
|
+
// OPENROUTER_API_KEY (the lane used to demand it for every pair). Same
|
|
614
|
+
// lazy, per-pair readiness check as the content lane (content-sync.js).
|
|
615
|
+
const readinessCache = new Map();
|
|
616
|
+
const requirePairReady = async (pairKey, pairConfig, code, what) => {
|
|
617
|
+
if (!readinessCache.has(pairKey)) {
|
|
618
|
+
const method = getMethod(pairConfig.method || 'llm', pairConfig);
|
|
619
|
+
readinessCache.set(pairKey, Promise.resolve(method.checkReadiness({ apiKey, cwd })));
|
|
620
|
+
}
|
|
621
|
+
const readiness = await readinessCache.get(pairKey);
|
|
622
|
+
if (!readiness.ready) {
|
|
623
|
+
throw new Error(
|
|
624
|
+
`Docusaurus ${what} for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
|
|
625
|
+
` ${readiness.reason}\n` +
|
|
626
|
+
missingKeyAdvice({ reasons: [readiness.reason], setupHelp: [' Set the key it names in .env.local (or your environment) to translate content.'] }).join('\n')
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
};
|
|
514
630
|
|
|
515
631
|
if (pairEntries.length === 0) {
|
|
516
632
|
output.info('No target languages configured. Add pairs to champollion.config.json.');
|
|
@@ -542,6 +658,13 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
542
658
|
} else if (tmInitialSize > 0) {
|
|
543
659
|
output.info(`[TM] ${tmInitialSize} cached entries loaded`);
|
|
544
660
|
}
|
|
661
|
+
// Model carry-over (lib/tm.js) is on unless --fresh-on-model-change.
|
|
662
|
+
const freshOnModelChange = !!cliArgs['fresh-on-model-change'];
|
|
663
|
+
if (freshOnModelChange) setModelCarryover(tm, false);
|
|
664
|
+
// Entries made before coaching was keyed, with today's coaching: still
|
|
665
|
+
// served (lib/tm.js adoptLegacyCoachingKeys).
|
|
666
|
+
if (!noTM) adoptLegacyCoachingKeys(tm, pairEntries.map(([, pc]) => pc));
|
|
667
|
+
if (!noTM) warnModelSwitchStrandedTM(tm, pairEntries.map(([, pc]) => pc), { fresh: freshOnModelChange });
|
|
545
668
|
|
|
546
669
|
// Phase-1 source-hash manifest (.champollion.lock) — read up front
|
|
547
670
|
// because the cost estimator mirrors Phase 1's changed-key detection.
|
|
@@ -551,8 +674,19 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
551
674
|
// manifest, EDITING an English UI string never re-translated it: the
|
|
552
675
|
// diff only sees missing keys and [EN] fallbacks, so a changed source
|
|
553
676
|
// value looked "fully synced" forever.
|
|
554
|
-
const
|
|
677
|
+
const lock = readLock(cwd);
|
|
678
|
+
const lockManifest = lock.source;
|
|
555
679
|
const updatedLockManifest = { ...lockManifest };
|
|
680
|
+
// The lock's per-locale record (lib/locale-state.js): which UI strings the
|
|
681
|
+
// quality gate refused, per (file × key × locale), for their current source
|
|
682
|
+
// text and the method key that produced the refused answer. A plain sync
|
|
683
|
+
// holds them back from that method — the key-value rule. Keyed like the
|
|
684
|
+
// manifest ("docusaurus:<relPath>:<id>").
|
|
685
|
+
const lockState = new LockState(lock.locales);
|
|
686
|
+
// What counts as an explicit redo (always sent, never held back): ids named
|
|
687
|
+
// by --redo keys: / --force-keys, every id under --redo all (--force), and
|
|
688
|
+
// anything under --fresh.
|
|
689
|
+
const redo = { named: new Set(config.forceKeys || []), bulk: !!config._forceAllKeys, fresh: noTM };
|
|
556
690
|
|
|
557
691
|
// ── Content discovery + pending-work scan ─────────────────────
|
|
558
692
|
// Done up front (not in Phase 2) because the cost estimate needs the
|
|
@@ -586,9 +720,29 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
586
720
|
if (fs.existsSync(contentLockPath)) {
|
|
587
721
|
try { docuContentManifest = JSON.parse(fs.readFileSync(contentLockPath, 'utf-8')); } catch { /* first run */ }
|
|
588
722
|
}
|
|
723
|
+
// --files / --retranslate (lib/file-scope.js). Resolved at the scan, so a
|
|
724
|
+
// pattern that matches nothing fails before the estimate or any spend.
|
|
725
|
+
const fileScope = compileFileScope(cliArgs);
|
|
589
726
|
const contentScan = scanDocusaurusContentWork(
|
|
590
|
-
contentSources, pairEntries, config, docuContentManifest, forceContent
|
|
727
|
+
contentSources, pairEntries, config, docuContentManifest, forceContent, fileScope
|
|
591
728
|
);
|
|
729
|
+
if (fileScope) fileScope.assertAllMatched();
|
|
730
|
+
// --retranslate files miss the TM for this run (estimate and sync alike),
|
|
731
|
+
// so they are priced and translated fresh; new results replace the cache.
|
|
732
|
+
for (const item of contentScan.workItems) {
|
|
733
|
+
if (!item.retranslate) continue;
|
|
734
|
+
bypassTMFor(tm, item.code, [
|
|
735
|
+
...Object.values(item.fieldsToTranslate), item.parsed.body, ...translatableBlockSources(item.parsed.body),
|
|
736
|
+
]);
|
|
737
|
+
}
|
|
738
|
+
// Blocks and front-matter fields the quality gate refused before
|
|
739
|
+
// (lib/content-refusals.js): not sent to the method that refused them again
|
|
740
|
+
// unless the page is named for a redo (--redo files: / --redo content /
|
|
741
|
+
// --retranslate / --fresh). The estimate and the sync read the same holds.
|
|
742
|
+
for (const item of contentScan.workItems) {
|
|
743
|
+
item.refusalsBefore = readRefusals(docuContentManifest, item.manifestKey);
|
|
744
|
+
item.holds = contentHolds(item.refusalsBefore, item.pairConfig, { redo: forceContent || item.retranslate || noTM });
|
|
745
|
+
}
|
|
592
746
|
|
|
593
747
|
// ── Pre-sync cost estimation + --max-cost gate ────────────────
|
|
594
748
|
// Mirrors sync.js: without a cap the estimate is informational and
|
|
@@ -598,26 +752,63 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
598
752
|
// exact thing a capped user needs to see.
|
|
599
753
|
const maxCost = parseMaxCost(cliArgs['max-cost']);
|
|
600
754
|
const costEstimate = await printDocusaurusCostEstimate(
|
|
601
|
-
pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate
|
|
755
|
+
pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate, lockState, redo
|
|
602
756
|
);
|
|
757
|
+
// Machine-readable estimate BEFORE the gate (see sync.js).
|
|
758
|
+
if (costEstimate) output.event('cost', costEstimate);
|
|
759
|
+
// A dry run never stops at the cap, but says what the real run would do.
|
|
760
|
+
let dryMaxCost = null;
|
|
603
761
|
if (maxCost !== null && !dryRun) {
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
762
|
+
const verdict = maxCostVerdict(maxCost, costEstimate);
|
|
763
|
+
if (verdict.wouldStop) return abortForMaxCost(maxCost, verdict.estimatedCost, verdict.reason);
|
|
764
|
+
} else if (maxCost !== null) {
|
|
765
|
+
dryMaxCost = reportDryRunMaxCost(maxCost, costEstimate, { stopsEarlier: preflightStopReason(preflightFailures) });
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// Fallback methods are outside the estimate (they translate what the
|
|
769
|
+
// primary fails, unknown in advance). Under --max-cost each fallback batch
|
|
770
|
+
// must fit in what the estimate left of the cap (lib/fallback.js).
|
|
771
|
+
if (pairEntries.some(([, p]) => p.fallback)) {
|
|
772
|
+
output.info(
|
|
773
|
+
'Fallback methods are not in this estimate — they only translate what the primary method fails, '
|
|
774
|
+
+ 'which is not known in advance.'
|
|
775
|
+
+ (maxCost !== null ? ' Under --max-cost each fallback batch is priced before it runs and skipped if it would pass the cap.' : '')
|
|
776
|
+
);
|
|
777
|
+
}
|
|
778
|
+
const fallbackBudget = createFallbackBudget({
|
|
779
|
+
maxCost: dryRun ? null : maxCost,
|
|
780
|
+
committed: costEstimate?.knownEstimatedCost ?? 0,
|
|
781
|
+
cwd,
|
|
782
|
+
});
|
|
783
|
+
// Different inputs, same output (lib/validate.js SharedOutputIndex): one
|
|
784
|
+
// index per locale for this run's UI strings and Markdown pages — cached
|
|
785
|
+
// and fresh alike, as in the content lane (lib/content-sync.js).
|
|
786
|
+
const sharedOutputIndexes = new Map();
|
|
787
|
+
const sharedOutputsFor = (code) => {
|
|
788
|
+
if (!sharedOutputIndexes.has(code)) {
|
|
789
|
+
sharedOutputIndexes.set(code, new SharedOutputIndex({ protectedTerms: config.protectedTerms || [] }));
|
|
615
790
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
791
|
+
return sharedOutputIndexes.get(code);
|
|
792
|
+
};
|
|
793
|
+
// Per-pair tallies of what each fallback did: JSON keys, Markdown segments.
|
|
794
|
+
const keyFallbackTallies = new Map();
|
|
795
|
+
const contentFallbackTallies = new Map();
|
|
796
|
+
// A page some of whose text the fallback produced (its answer or its
|
|
797
|
+
// cache): named in the [FALLBACK] line.
|
|
798
|
+
const notePage = (pairKey, page) => {
|
|
799
|
+
const tally = contentFallbackTallies.get(pairKey);
|
|
800
|
+
if (tally && !tally.produced.includes(page)) tally.produced.push(page);
|
|
801
|
+
};
|
|
802
|
+
const tallyContent = (pairKey, report, page) => {
|
|
803
|
+
if (!contentFallbackTallies.has(pairKey)) return;
|
|
804
|
+
addToTally(contentFallbackTallies.get(pairKey), report);
|
|
805
|
+
if (report && (report.accepted > 0 || report.cached > 0)) notePage(pairKey, page);
|
|
806
|
+
};
|
|
807
|
+
if (!dryRun) {
|
|
808
|
+
for (const [pairKey, pc] of pairEntries) {
|
|
809
|
+
if (!pc.fallback) continue;
|
|
810
|
+
keyFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
|
|
811
|
+
contentFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
|
|
621
812
|
}
|
|
622
813
|
}
|
|
623
814
|
|
|
@@ -628,6 +819,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
628
819
|
|
|
629
820
|
let totalJSONKeys = 0;
|
|
630
821
|
let totalJSONCopied = 0;
|
|
822
|
+
let totalJSONFailed = 0;
|
|
823
|
+
// UI strings held back (refused before; not sent) and what the next sync
|
|
824
|
+
// does with each one this run could not translate (lib/locale-state.js).
|
|
825
|
+
let totalJSONHeld = 0;
|
|
826
|
+
let totalJSONWritten = 0;
|
|
827
|
+
const jsonFates = { retry: [], held: [] }; // { pair, key }
|
|
828
|
+
// Every lock key the source still has (refused records for others are dropped).
|
|
829
|
+
const liveLockKeys = new Set();
|
|
631
830
|
|
|
632
831
|
for (const sourceFilePath of sourceJSONFiles) {
|
|
633
832
|
const relPath = path.relative(sourceLocaleDir, sourceFilePath);
|
|
@@ -649,6 +848,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
649
848
|
// The manifest stores namespaced keys; detectChangedKeys expects the
|
|
650
849
|
// same key space as sourceFlat, so build a per-file un-namespaced view.
|
|
651
850
|
const nsPrefix = `docusaurus:${relPath}:`;
|
|
851
|
+
for (const key of Object.keys(sourceFlat)) liveLockKeys.add(nsPrefix + key);
|
|
652
852
|
const fileOldManifest = {};
|
|
653
853
|
for (const [nsKey, storedHash] of Object.entries(lockManifest)) {
|
|
654
854
|
if (nsKey.startsWith(nsPrefix)) {
|
|
@@ -664,10 +864,11 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
664
864
|
const pairResults = await pMap(pairEntries, async ([pairKey, pairConfig]) => {
|
|
665
865
|
const code = pairConfig.target;
|
|
666
866
|
const targetFilePath = path.join(config.localesDir, code, relPath);
|
|
867
|
+
const filename = `${code}/${relPath}`;
|
|
667
868
|
|
|
668
869
|
// Security: verify target path stays within i18n directory
|
|
669
870
|
if (!isPathContained(targetFilePath, config.localesDir)) {
|
|
670
|
-
output.error(`${
|
|
871
|
+
output.error(`${filename} — refusing to write outside i18n directory`);
|
|
671
872
|
// Nothing was attempted, but nothing succeeded either: keep every
|
|
672
873
|
// to-be-processed key out of the "succeeded" set so its manifest
|
|
673
874
|
// hash is not advanced (an unwritable locale must re-fire).
|
|
@@ -685,11 +886,15 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
685
886
|
// re-translate (they are otherwise invisible to the diff). Echo keys
|
|
686
887
|
// (target === source) the TM confirms as pipeline-produced are NOT
|
|
687
888
|
// requeued — see lib/diff.js isConfirmedEcho.
|
|
688
|
-
|
|
889
|
+
// A value the pair's fallback produced is cached under the fallback.
|
|
890
|
+
const tmKeys = tmKeysForPair(pairConfig);
|
|
891
|
+
const localeState = dryRun ? lockState.peek(code) : lockState.of(code);
|
|
892
|
+
// A refused string whose source was edited since: queued (the hold lifts).
|
|
893
|
+
const localeChanged = [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, localeState)])];
|
|
689
894
|
const diff = diffLocale(
|
|
690
895
|
sourceFlat, existingFlat, config.fallbackPrefix,
|
|
691
|
-
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys,
|
|
692
|
-
(key, sourceValue) =>
|
|
896
|
+
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
|
|
897
|
+
(key, sourceValue) => tmHoldsValue(tm, sourceValue, code, tmKeys, sourceValue),
|
|
693
898
|
noTranslate.active ? noTranslate.matches : null
|
|
694
899
|
);
|
|
695
900
|
|
|
@@ -697,12 +902,38 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
697
902
|
return { keys: 0, failedKeys: [] };
|
|
698
903
|
}
|
|
699
904
|
|
|
905
|
+
// Refused before (the lock's records): held back from the method that
|
|
906
|
+
// refused them — the key-value rule (planUIStrings).
|
|
907
|
+
const plan = planUIStrings({
|
|
908
|
+
diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState, pairConfig, redo,
|
|
909
|
+
fallbackPrefix: config.fallbackPrefix,
|
|
910
|
+
});
|
|
911
|
+
const heldSet = new Set(plan.held);
|
|
912
|
+
const lk = (k) => nsPrefix + k;
|
|
913
|
+
|
|
700
914
|
let keysProcessed = 0;
|
|
701
915
|
let keysCopied = 0;
|
|
916
|
+
let keysWritten = 0; // translations written (the model, its fallback, the cache)
|
|
702
917
|
const failedKeys = [];
|
|
918
|
+
const heldKeys = [];
|
|
919
|
+
const fates = { retry: [], held: [] };
|
|
703
920
|
|
|
704
921
|
if (diff.toProcess.length > 0 || diff.noTranslate.length > 0) {
|
|
705
|
-
|
|
922
|
+
// The per-file line counts what goes to the pipeline; held keys are
|
|
923
|
+
// said on their own line below.
|
|
924
|
+
const shown = heldSet.size === 0 ? diff : Object.fromEntries(Object.entries(diff).map(([k, v]) => (
|
|
925
|
+
[k, Array.isArray(v) && k !== 'noTranslate' && k !== 'extra' ? v.filter(x => !heldSet.has(x)) : v])));
|
|
926
|
+
const label = diffLabel(shown);
|
|
927
|
+
output.info(`${filename} — ${plan.held.length > 0 ? `${label === 'fully synced' ? '' : `${label} + `}${plan.held.length} held back` : label}`);
|
|
928
|
+
if (plan.held.length > 0) {
|
|
929
|
+
output.warn(describeHeldKeys({ filename, keys: plan.held, pairConfig, command: redoCommand(plan.held, { pair: pairKey }) }));
|
|
930
|
+
}
|
|
931
|
+
if (plan.heldFromPrimary.length > 0) {
|
|
932
|
+
output.info(describeFallbackOnlyKeys({ filename, keys: plan.heldFromPrimary, pairConfig }));
|
|
933
|
+
}
|
|
934
|
+
// Named for a redo but served from the cache (the model is not asked
|
|
935
|
+
// without --fresh): said, with the command that asks the model again.
|
|
936
|
+
const namedQueued = redo.fresh ? [] : diff.toProcess.filter(k => redo.named.has(k) && typeof sourceFlat[k] === 'string');
|
|
706
937
|
|
|
707
938
|
if (!dryRun) {
|
|
708
939
|
// Merged output starts from what's on disk, then takes the verbatim
|
|
@@ -712,6 +943,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
712
943
|
const mergedFlat = { ...existingFlat };
|
|
713
944
|
for (const key of diff.noTranslate) {
|
|
714
945
|
mergedFlat[key] = sourceFlat[key];
|
|
946
|
+
delete localeState.refused[lk(key)];
|
|
715
947
|
}
|
|
716
948
|
keysCopied = diff.noTranslate.length;
|
|
717
949
|
const writeMerged = () => {
|
|
@@ -719,32 +951,77 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
719
951
|
fs.mkdirSync(path.dirname(targetFilePath), { recursive: true });
|
|
720
952
|
fs.writeFileSync(targetFilePath, JSON.stringify(docuOutput, null, 2) + '\n', 'utf-8');
|
|
721
953
|
};
|
|
954
|
+
// What the gate refused this run is remembered per key (held back
|
|
955
|
+
// by the next plain sync); what got no usable answer is asked again.
|
|
956
|
+
const settleUnfilled = (keys, refusedBy = {}) => {
|
|
957
|
+
for (const k of keys) {
|
|
958
|
+
recordRefusal(localeState, lk(k), sourceFlat[k], refusedBy[k]);
|
|
959
|
+
fates[(refusedBy[k] || []).length > 0 ? 'held' : 'retry'].push(k);
|
|
960
|
+
}
|
|
961
|
+
};
|
|
722
962
|
|
|
723
963
|
if (keysCopied > 0) {
|
|
724
964
|
const sample = diff.noTranslate.slice(0, 3)
|
|
725
965
|
.map(k => `${k} (${noTranslate.reason(k, sourceFlat[k])})`)
|
|
726
966
|
.join(', ');
|
|
727
967
|
const more = keysCopied > 3 ? `, +${keysCopied - 3} more` : '';
|
|
728
|
-
output.info(`${
|
|
968
|
+
output.info(`${filename} — copied ${keysCopied} no-translate key(s) verbatim: ${sample}${more}`);
|
|
729
969
|
}
|
|
730
970
|
|
|
731
971
|
let translated = null;
|
|
972
|
+
let result = null;
|
|
732
973
|
const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
|
|
733
974
|
|
|
734
975
|
if (stringKeys.length > 0) {
|
|
735
|
-
// Shared pipeline: TM partition → API call →
|
|
736
|
-
|
|
737
|
-
|
|
976
|
+
// Shared pipeline: TM partition → API call → quality gate → TM
|
|
977
|
+
// store, then the pair's fallback (if any) for what that left
|
|
978
|
+
// untranslated (lib/translate-pair.js translateWithFallback).
|
|
979
|
+
// Keys held back: the cache is still read, the method not asked.
|
|
980
|
+
result = await translateWithFallback(stringKeys, sourceFlat, pairConfig, pairKey, {
|
|
981
|
+
apiKey, tm, targetCode: code, descriptions, budget: fallbackBudget, cwd,
|
|
982
|
+
// One per locale, as on the standard path: a memorized sentence
|
|
983
|
+
// repeated across UI strings is refused (validate.js).
|
|
984
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
985
|
+
noSendPrimary: new Set([...plan.held, ...plan.heldFromPrimary]),
|
|
986
|
+
noSendFallback: heldSet,
|
|
738
987
|
});
|
|
739
988
|
translated = result.translated;
|
|
989
|
+
if (keyFallbackTallies.has(pairKey)) addToTally(keyFallbackTallies.get(pairKey), result.fallback);
|
|
990
|
+
const heldHere = new Set(result.heldKeys || []);
|
|
991
|
+
heldKeys.push(...stringKeys.filter(k => heldHere.has(k) && !(translated && k in translated)));
|
|
740
992
|
|
|
741
993
|
if (translated) {
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
994
|
+
const notDone = stringKeys.filter(k => !(k in translated));
|
|
995
|
+
const refusedHere = notDone.filter(k => (result.refusedBy?.[k] || []).length > 0);
|
|
996
|
+
const heldNow = notDone.filter(k => heldHere.has(k));
|
|
997
|
+
const noAnswer = notDone.length - refusedHere.length - heldNow.length;
|
|
998
|
+
output.progressDone(filename, notDone.length === 0 ? '[OK]' : `[WARN] ${notDone.length} of ${stringKeys.length} key(s) not translated (${[
|
|
999
|
+
refusedHere.length > 0 && `${refusedHere.length} refused by the quality gate`,
|
|
1000
|
+
heldNow.length > 0 && `${heldNow.length} held back`,
|
|
1001
|
+
noAnswer > 0 && `${noAnswer} not returned by the method`,
|
|
1002
|
+
].filter(Boolean).join(', ')})`);
|
|
1003
|
+
const answered = new Set(result.answeredKeys || []);
|
|
1004
|
+
const fromCache = namedQueued.filter(k => k in translated && !answered.has(k));
|
|
1005
|
+
await reportNamedFromCache({
|
|
1006
|
+
filename, keys: fromCache, served: translated, onDisk: existingFlat, pairConfig, cwd,
|
|
1007
|
+
command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
|
|
1008
|
+
});
|
|
1009
|
+
} else if (heldKeys.length === stringKeys.length) {
|
|
1010
|
+
// Everything queued was held back: nothing was asked, nothing failed anew.
|
|
1011
|
+
output.progressDone(filename, `[WARN] ${heldKeys.length} key(s) held back, not translated`);
|
|
1012
|
+
for (const k of heldKeys) fates.held.push(k);
|
|
746
1013
|
if (keysCopied > 0) writeMerged();
|
|
747
|
-
return { keys: 0, copied: keysCopied, failedKeys:
|
|
1014
|
+
return { keys: 0, copied: keysCopied, failedKeys: [], heldKeys, fates };
|
|
1015
|
+
} else if (result.apiReturnedNull || result.failures.length > 0) {
|
|
1016
|
+
output.progressDone(filename, result.apiReturnedNull ? '[ERR] translation failed' : '[ERR] all failed quality gate');
|
|
1017
|
+
output.error(result.apiReturnedNull
|
|
1018
|
+
? `${filename}: Translation failed. Check API key and method configuration.`
|
|
1019
|
+
: `${filename}: all translations were rejected by the quality gate${pairConfig.fallback ? ` (and by its fallback, ${pairConfig.fallback.method})` : ''} — see the gate failures above.`);
|
|
1020
|
+
if (keysCopied > 0) writeMerged();
|
|
1021
|
+
const failedNow = stringKeys.filter(k => !heldKeys.includes(k));
|
|
1022
|
+
settleUnfilled(failedNow, result.refusedBy || {});
|
|
1023
|
+
for (const k of heldKeys) fates.held.push(k);
|
|
1024
|
+
return { keys: 0, copied: keysCopied, failedKeys: failedNow, heldKeys, fates };
|
|
748
1025
|
}
|
|
749
1026
|
}
|
|
750
1027
|
|
|
@@ -753,6 +1030,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
753
1030
|
// for it, fallbacks first, and a value with unmappable letters
|
|
754
1031
|
// stays whole in the working script (warned, not failed).
|
|
755
1032
|
const scriptConverterKey = pairConfig.scriptResolution?.converterKey || null;
|
|
1033
|
+
const heldNow = new Set(heldKeys);
|
|
756
1034
|
for (const key of diff.toProcess) {
|
|
757
1035
|
if (translated && key in translated) {
|
|
758
1036
|
let value = translated[key];
|
|
@@ -763,50 +1041,87 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
763
1041
|
value = converted;
|
|
764
1042
|
} else {
|
|
765
1043
|
output.warn(
|
|
766
|
-
`${
|
|
1044
|
+
`${filename}: key "${key}" kept in working script — `
|
|
767
1045
|
+ `unmapped letter(s): ${unmapped.join(', ')} (see "scriptFallback")`
|
|
768
1046
|
);
|
|
769
1047
|
}
|
|
770
1048
|
}
|
|
771
1049
|
mergedFlat[key] = value;
|
|
1050
|
+
keysWritten++;
|
|
1051
|
+
// Filled (by the model, its fallback or the cache): no longer refused.
|
|
1052
|
+
delete localeState.refused[lk(key)];
|
|
1053
|
+
} else if (heldNow.has(key)) {
|
|
1054
|
+
// Held back: said above; not sent, not counted as failed.
|
|
1055
|
+
fates.held.push(key);
|
|
772
1056
|
} else if (typeof sourceFlat[key] === 'string') {
|
|
773
|
-
|
|
1057
|
+
const fate = (result?.refusedBy?.[key] || []).length > 0 ? 'held' : 'retry';
|
|
1058
|
+
output.warn(`${filename}: key "${key}" ${keyFateNote(fate, pairConfig)}`);
|
|
774
1059
|
failedKeys.push(key);
|
|
1060
|
+
settleUnfilled([key], result?.refusedBy || {});
|
|
775
1061
|
} else {
|
|
776
1062
|
mergedFlat[key] = sourceFlat[key];
|
|
777
1063
|
}
|
|
778
1064
|
}
|
|
779
1065
|
|
|
780
|
-
keysProcessed = diff.toProcess.length;
|
|
1066
|
+
keysProcessed = diff.toProcess.length - heldKeys.length;
|
|
781
1067
|
|
|
782
1068
|
// Inject back into Docusaurus format and write
|
|
783
1069
|
writeMerged();
|
|
784
1070
|
} else {
|
|
785
|
-
|
|
1071
|
+
// Held keys would not be sent: not counted.
|
|
1072
|
+
keysProcessed = diff.toProcess.length - plan.held.length;
|
|
786
1073
|
keysCopied = diff.noTranslate.length;
|
|
1074
|
+
if (namedQueued.length > 0) {
|
|
1075
|
+
// What the real run would serve from the cache (the pair's entry,
|
|
1076
|
+
// then its fallback's — the same partition the estimate makes).
|
|
1077
|
+
const served = {};
|
|
1078
|
+
for (const k of namedQueued.filter(x => !heldSet.has(x))) {
|
|
1079
|
+
for (const mk of tmKeys) {
|
|
1080
|
+
const v = peekTM(tm, sourceFlat[k], code, mk);
|
|
1081
|
+
if (v !== null) { served[k] = v; break; }
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
const fromCache = Object.keys(served);
|
|
1085
|
+
await reportNamedFromCache({
|
|
1086
|
+
filename, keys: fromCache, served, onDisk: existingFlat, pairConfig, cwd, dryRun: true,
|
|
1087
|
+
command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
|
|
1088
|
+
});
|
|
1089
|
+
}
|
|
1090
|
+
heldKeys.push(...plan.held);
|
|
787
1091
|
}
|
|
788
1092
|
}
|
|
789
1093
|
|
|
790
1094
|
if (diff.extra.length > 0) {
|
|
791
|
-
output.warn(`${
|
|
1095
|
+
output.warn(`${filename} — ${diff.extra.length} extra key(s)`);
|
|
792
1096
|
}
|
|
793
1097
|
|
|
794
|
-
return { keys: keysProcessed, copied: keysCopied, failedKeys };
|
|
1098
|
+
return { keys: keysProcessed, written: keysWritten, copied: keysCopied, failedKeys, heldKeys, fates };
|
|
795
1099
|
}, { concurrency: jsonConcurrency });
|
|
796
1100
|
|
|
797
1101
|
// Aggregate results for this JSON file
|
|
798
1102
|
const fileFailedKeys = new Set();
|
|
799
|
-
|
|
1103
|
+
// Held keys keep their manifest hash too (an edited source must still
|
|
1104
|
+
// re-fire), but they are not failures.
|
|
1105
|
+
const fileUnfilledKeys = new Set();
|
|
1106
|
+
pairResults.forEach((r, i) => {
|
|
800
1107
|
totalJSONKeys += r.keys;
|
|
1108
|
+
totalJSONWritten += r.written || 0;
|
|
801
1109
|
totalJSONCopied += r.copied || 0;
|
|
802
|
-
|
|
803
|
-
|
|
1110
|
+
totalJSONHeld += (r.heldKeys || []).length;
|
|
1111
|
+
for (const k of r.failedKeys || []) { fileFailedKeys.add(k); fileUnfilledKeys.add(k); }
|
|
1112
|
+
for (const k of r.heldKeys || []) fileUnfilledKeys.add(k);
|
|
1113
|
+
const pair = pairEntries[i][0];
|
|
1114
|
+
for (const fate of ['retry', 'held']) {
|
|
1115
|
+
for (const key of r.fates?.[fate] || []) jsonFates[fate].push({ pair, key });
|
|
1116
|
+
}
|
|
1117
|
+
});
|
|
1118
|
+
totalJSONFailed += fileFailedKeys.size;
|
|
804
1119
|
|
|
805
1120
|
// Update the manifest for this file: record the current hash ONLY for
|
|
806
1121
|
// keys that succeeded in every locale that attempted them. A failed
|
|
807
|
-
// key keeps its OLD hash (or none) so it is detected as
|
|
808
|
-
// RE-FIRES on the next sync — advancing the hash for a failed
|
|
809
|
-
// would silently mark the stale translation as current forever.
|
|
1122
|
+
// (or held) key keeps its OLD hash (or none) so it is detected as
|
|
1123
|
+
// changed and RE-FIRES on the next sync — advancing the hash for a failed
|
|
1124
|
+
// key would silently mark the stale translation as current forever.
|
|
810
1125
|
if (!dryRun) {
|
|
811
1126
|
for (const nsKey of Object.keys(updatedLockManifest)) {
|
|
812
1127
|
// Own-property check: `in` walks the prototype chain, so a source
|
|
@@ -818,7 +1133,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
818
1133
|
}
|
|
819
1134
|
for (const [key, value] of Object.entries(sourceFlat)) {
|
|
820
1135
|
const nsKey = nsPrefix + key;
|
|
821
|
-
if (
|
|
1136
|
+
if (fileUnfilledKeys.has(key)) {
|
|
822
1137
|
// Restore/keep the pre-sync state for failed keys.
|
|
823
1138
|
if (Object.prototype.hasOwnProperty.call(lockManifest, nsKey)) {
|
|
824
1139
|
updatedLockManifest[nsKey] = lockManifest[nsKey];
|
|
@@ -832,25 +1147,68 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
832
1147
|
}
|
|
833
1148
|
}
|
|
834
1149
|
|
|
835
|
-
// Persist the Phase 1 source-hash manifest
|
|
836
|
-
// must not mark changed keys as resolved).
|
|
1150
|
+
// Persist the Phase 1 source-hash manifest and the refusal records (skip
|
|
1151
|
+
// in dry-run — a preview must not mark changed keys as resolved). Records
|
|
1152
|
+
// of UI strings the source no longer has are dropped for the locales this
|
|
1153
|
+
// run processed.
|
|
837
1154
|
if (!dryRun && sourceJSONFiles.length > 0) {
|
|
838
|
-
|
|
1155
|
+
for (const [, pc] of pairEntries) {
|
|
1156
|
+
const refused = lockState.of(pc.target).refused;
|
|
1157
|
+
for (const k of Object.keys(refused)) {
|
|
1158
|
+
if (k.startsWith('docusaurus:') && !liveLockKeys.has(k)) delete refused[k];
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
writeManifest(cwd, updatedLockManifest, lockState.toJSON());
|
|
1162
|
+
}
|
|
1163
|
+
for (const [pairKey, tally] of keyFallbackTallies) {
|
|
1164
|
+
printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
|
|
1165
|
+
warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
|
|
839
1166
|
}
|
|
840
1167
|
|
|
841
1168
|
const copiedNote = totalJSONCopied > 0
|
|
842
1169
|
? ` (+${totalJSONCopied} copied verbatim, no-translate)`
|
|
843
1170
|
: '';
|
|
844
|
-
|
|
1171
|
+
const heldNote = totalJSONHeld > 0
|
|
1172
|
+
? `; ${totalJSONHeld} held back (refused before; not sent, not billed)`
|
|
1173
|
+
: '';
|
|
1174
|
+
const failedNote = totalJSONFailed > 0 ? `; ${totalJSONFailed} failed (listed above)` : '';
|
|
1175
|
+
// A real run counts what it wrote; a dry run what it would send.
|
|
1176
|
+
const shownKeys = dryRun ? totalJSONKeys : totalJSONWritten;
|
|
1177
|
+
if (shownKeys > 0) {
|
|
845
1178
|
const action = dryRun ? 'Would process' : 'Synced';
|
|
846
|
-
output.ok(`${action} ${
|
|
1179
|
+
output.ok(`${action} ${shownKeys} JSON key(s)${copiedNote}${failedNote}${heldNote}`);
|
|
1180
|
+
} else if (totalJSONHeld > 0 || totalJSONFailed > 0) {
|
|
1181
|
+
output.warn(`No JSON key ${dryRun ? 'would be' : 'was'} translated${copiedNote}${failedNote}${heldNote}`);
|
|
847
1182
|
} else if (totalJSONCopied > 0) {
|
|
848
1183
|
output.ok(`All JSON files fully synced${copiedNote}`);
|
|
849
1184
|
} else {
|
|
850
1185
|
output.ok('All JSON files fully synced');
|
|
851
1186
|
}
|
|
1187
|
+
// What the next sync does with the UI strings this run could not translate
|
|
1188
|
+
// — said per outcome, in the key-value lane's words.
|
|
1189
|
+
if (!dryRun && (jsonFates.retry.length > 0 || jsonFates.held.length > 0)) {
|
|
1190
|
+
if (jsonFates.retry.length > 0) {
|
|
1191
|
+
output.warn(` ${jsonFates.retry.length} UI string(s) got no usable answer (missing from the response, or the method failed) — the next sync asks for them again.`);
|
|
1192
|
+
}
|
|
1193
|
+
// Ids are named bare (a name re-queues that id in every JSON file that has it).
|
|
1194
|
+
const heldNext = [];
|
|
1195
|
+
const seen = new Set();
|
|
1196
|
+
for (const { pair, key } of jsonFates.held) {
|
|
1197
|
+
if (seen.has(`${pair}\u0000${key}`)) continue;
|
|
1198
|
+
seen.add(`${pair}\u0000${key}`);
|
|
1199
|
+
heldNext.push({ pair, key });
|
|
1200
|
+
}
|
|
1201
|
+
for (const line of describeHeldNext(heldNext, (keys, pair) => redoCommand(keys, { pair }))) output.warn(line);
|
|
1202
|
+
}
|
|
852
1203
|
|
|
853
1204
|
// ── Phase 2: Markdown content (docs + blog) ───────────────────
|
|
1205
|
+
let contentFailures = 0; // read after the TM save, outside the content block
|
|
1206
|
+
let contentTranslated = 0;
|
|
1207
|
+
let contentFailedItems = [];
|
|
1208
|
+
// Blocks/fields refused before and held back this run, and refused this run.
|
|
1209
|
+
let contentHeldBack = 0;
|
|
1210
|
+
const contentHeldBackItems = [];
|
|
1211
|
+
let contentRefused = 0;
|
|
854
1212
|
|
|
855
1213
|
if (contentSources.length === 0) {
|
|
856
1214
|
output.info('No docs/ or blog/ directories found — skipping content sync.');
|
|
@@ -872,7 +1230,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
872
1230
|
const concurrency = config.contentConcurrency || 48;
|
|
873
1231
|
|
|
874
1232
|
const totalWork = workItems.length;
|
|
875
|
-
|
|
1233
|
+
// Refusals per translation (lib/content-refusals.js), applied to the lock
|
|
1234
|
+
// with each item's outcome.
|
|
1235
|
+
const refusalStates = new Map();
|
|
1236
|
+
// Every failed (file × locale), with what it was left as on disk — the
|
|
1237
|
+
// end-of-run list an operator needs to tell "failed, previous
|
|
1238
|
+
// translation kept" from "failed, will retry" (dogfood 2026-08-28,
|
|
1239
|
+
// finding 5).
|
|
1240
|
+
const failedItems = [];
|
|
876
1241
|
// Warn at most once per source file about front matter fields we can't
|
|
877
1242
|
// translate (arrays / nested blocks like `related:`). The check + add are
|
|
878
1243
|
// synchronous (no await between), so this is race-free under pMap.
|
|
@@ -905,23 +1270,67 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
905
1270
|
await pMap(workItems, async (item) => {
|
|
906
1271
|
const {
|
|
907
1272
|
parsed, fieldsToTranslate, pageTitle, sourceHash,
|
|
908
|
-
relPath, dirName, pairConfig, code,
|
|
1273
|
+
relPath, dirName, pairKey, pairConfig, code,
|
|
909
1274
|
targetPath, manifestKey, action,
|
|
910
1275
|
} = item;
|
|
911
1276
|
|
|
1277
|
+
let itemFailed = false;
|
|
1278
|
+
// This translation's refusals, recorded with its outcome (held,
|
|
1279
|
+
// failed or written) — in the incremental lock writes too.
|
|
1280
|
+
const settleRefusals = () => {
|
|
1281
|
+
const st = refusalStates.get(manifestKey);
|
|
1282
|
+
if (!st || dryRun) return;
|
|
1283
|
+
storeRefusals(updatedDocuManifest, manifestKey, nextRefusals(st.prior, st));
|
|
1284
|
+
manifestDirty = true;
|
|
1285
|
+
};
|
|
912
1286
|
try {
|
|
913
1287
|
if (action === 'changed') {
|
|
914
1288
|
totalContentRetranslated++;
|
|
915
1289
|
}
|
|
916
1290
|
|
|
917
1291
|
if (dryRun) {
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1292
|
+
// What the quality gate refused before is held back by the real
|
|
1293
|
+
// run (lib/content-refusals.js) — said here.
|
|
1294
|
+
const preview = previewHeld({
|
|
1295
|
+
tm, code, pairConfig, fields: parsed.hasFrontMatter ? fieldsToTranslate : {}, body: parsed.body,
|
|
1296
|
+
segMode: pairConfig.contentSegmentation || config.contentSegmentation || 'block', holds: item.holds,
|
|
1297
|
+
});
|
|
1298
|
+
const held = preview.names.length > 0
|
|
1299
|
+
? ` — ${preview.pageHeld ? 'would hold the page back:' : 'would hold back'} ${preview.names.join(', ')} (refused before; not sent)`
|
|
1300
|
+
: '';
|
|
1301
|
+
output.raw(` [DRY] ${dirName}/${relPath} → ${code}${held}`);
|
|
1302
|
+
output.event('file', {
|
|
1303
|
+
lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: preview.pageHeld ? 'would-hold' : 'would-translate', action,
|
|
1304
|
+
...(preview.names.length > 0 && { held: preview.names.length }),
|
|
1305
|
+
});
|
|
1306
|
+
if (!preview.pageHeld) totalContent++;
|
|
921
1307
|
return;
|
|
922
1308
|
}
|
|
923
1309
|
|
|
924
1310
|
const { rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parsed;
|
|
1311
|
+
const segModeHere = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
1312
|
+
const label = `${dirName}/${relPath}`;
|
|
1313
|
+
const refusal = {
|
|
1314
|
+
file: label, locale: code, pageHeld: false,
|
|
1315
|
+
prior: item.refusalsBefore || {},
|
|
1316
|
+
refused: [], filled: new Set(), held: [],
|
|
1317
|
+
fields: hasFrontMatter ? fieldsToTranslate : {},
|
|
1318
|
+
blockSources: segModeHere === 'block' && body.trim() ? translatableBlockSources(body) : [],
|
|
1319
|
+
pageSource: segModeHere === 'page' && body.trim() ? body : null,
|
|
1320
|
+
};
|
|
1321
|
+
refusalStates.set(manifestKey, refusal);
|
|
1322
|
+
const { holds } = item;
|
|
1323
|
+
// A page translated whole that the gate refused before (and that
|
|
1324
|
+
// the cache does not hold): held before anything of it is sent —
|
|
1325
|
+
// its front matter included (a pure look; the body read below is
|
|
1326
|
+
// the one that serves).
|
|
1327
|
+
if (refusal.pageSource && holds.page(body) === 'held'
|
|
1328
|
+
&& ![tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])]
|
|
1329
|
+
.some(k => peekTM(tm, body, code, k) !== null)) {
|
|
1330
|
+
const err = new Error('held');
|
|
1331
|
+
err.heldNames = [PAGE_NAME];
|
|
1332
|
+
throw err;
|
|
1333
|
+
}
|
|
925
1334
|
|
|
926
1335
|
// Never silently drop translatable-looking nested/array front matter
|
|
927
1336
|
// (e.g. `related:` lists). The flat parser can't reach them — surface
|
|
@@ -942,10 +1351,32 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
942
1351
|
// not re-serve. Mirrors content-sync.js.
|
|
943
1352
|
const tmKey = tmMethodKey(pairConfig);
|
|
944
1353
|
|
|
1354
|
+
// This page's cached content through the repeat check, in one batch
|
|
1355
|
+
// and before any of it is served (lib/fallback.js): a text cached
|
|
1356
|
+
// for several different source strings is evicted, so the lookups
|
|
1357
|
+
// below miss it and it is translated again, checked the same way.
|
|
1358
|
+
const docLabel = `content:${dirName}/${relPath}`;
|
|
1359
|
+
const repeated = refuseRepeatedCachedPage({
|
|
1360
|
+
label: docLabel,
|
|
1361
|
+
fields: hasFrontMatter ? fieldsToTranslate : {},
|
|
1362
|
+
body,
|
|
1363
|
+
tm, code, pairConfig,
|
|
1364
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
1365
|
+
});
|
|
1366
|
+
if (repeated.length > 0) {
|
|
1367
|
+
output.warn(
|
|
1368
|
+
`${dirName}/${relPath} → ${code}: cached ${repeated.join(', ')} held the same text as other, different source strings ` +
|
|
1369
|
+
'(a memorized sentence, not a translation) — removed from the cache and translated again.'
|
|
1370
|
+
);
|
|
1371
|
+
}
|
|
1372
|
+
|
|
945
1373
|
// Translate front matter fields — TM first, API only for misses.
|
|
946
1374
|
// Each field is cached on its own source text (exactly like a
|
|
947
1375
|
// key-value sync key): a title edit re-pays only the title.
|
|
948
1376
|
const translatedFields = {};
|
|
1377
|
+
// A field left in the source language (refused, or held from an
|
|
1378
|
+
// earlier refusal): the page is written, its lock marked pending.
|
|
1379
|
+
let fieldsLeftInSource = false;
|
|
949
1380
|
if (hasFrontMatter && Object.keys(fieldsToTranslate).length > 0) {
|
|
950
1381
|
const { hits: fmHits, misses: fmMisses } = partitionByTM(
|
|
951
1382
|
tm, fieldsToTranslate, Object.keys(fieldsToTranslate), code, tmKey
|
|
@@ -954,61 +1385,94 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
954
1385
|
// gateless pipeline (hollowed titles were cached here) must not
|
|
955
1386
|
// outlive the gate. Failing hits are evicted and re-billed.
|
|
956
1387
|
for (const [field, cachedValue] of Object.entries(fmHits)) {
|
|
957
|
-
if (
|
|
1388
|
+
if (contentGateFault(fieldsToTranslate[field], cachedValue, pairConfig)) {
|
|
958
1389
|
evictTM(tm, fieldsToTranslate[field], code, tmKey);
|
|
959
1390
|
delete fmHits[field];
|
|
960
1391
|
fmMisses.push(field);
|
|
961
1392
|
}
|
|
962
1393
|
}
|
|
963
1394
|
Object.assign(translatedFields, fmHits);
|
|
1395
|
+
// Then the fallback's cache (lib/fallback.js ladder).
|
|
1396
|
+
const fbCache = serveFieldsFromFallbackCache(tm, fieldsToTranslate, fmMisses, code, pairConfig);
|
|
1397
|
+
Object.assign(translatedFields, fbCache.hits);
|
|
1398
|
+
if (Object.keys(fbCache.hits).length > 0) notePage(pairKey, label);
|
|
1399
|
+
const fieldsToSend = fbCache.misses;
|
|
1400
|
+
for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
|
|
1401
|
+
// Refused before: a held field is not sent again; it keeps its
|
|
1402
|
+
// source text and the page is written (its lock marked pending).
|
|
1403
|
+
const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
|
|
1404
|
+
const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
|
|
1405
|
+
if (heldFields.length > 0) {
|
|
1406
|
+
fieldsLeftInSource = true;
|
|
1407
|
+
const names = heldFields.map(f => `front matter "${f}"`);
|
|
1408
|
+
refusal.held.push(...names);
|
|
1409
|
+
output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names }));
|
|
1410
|
+
for (const f of heldFields) fieldsToSend.splice(fieldsToSend.indexOf(f), 1);
|
|
1411
|
+
}
|
|
964
1412
|
|
|
965
|
-
if (
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
1413
|
+
if (fieldsToSend.length > 0) {
|
|
1414
|
+
await requirePairReady(pairKey, pairConfig, code, 'content sync');
|
|
1415
|
+
// The pair's method, then its fallback (if any) for every field
|
|
1416
|
+
// it returned nothing for or hollowed. Content-preservation
|
|
1417
|
+
// gate on every value — Phase 2 front matter once reached disk
|
|
1418
|
+
// AND the TM with no validation at all, so a hollowed page
|
|
1419
|
+
// title was written silently and then cached forever. Nothing
|
|
1420
|
+
// is cached until no field is left failing; a failing field
|
|
1421
|
+
// skips the file and leaves its lock entry alone, so the next
|
|
1422
|
+
// sync retries it.
|
|
1423
|
+
const fm = await translateFieldsWithFallback({
|
|
1424
|
+
fields: fieldsToTranslate,
|
|
1425
|
+
misses: fieldsToSend,
|
|
1426
|
+
pairConfig,
|
|
1427
|
+
budget: fallbackBudget,
|
|
1428
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
1429
|
+
label: `${docLabel} front matter`,
|
|
1430
|
+
translate: (keys, cfg, extra = {}) => translateBatch(
|
|
1431
|
+
keys, fieldsToTranslate, cfg,
|
|
1432
|
+
{ apiKey, model: cfg.model, batchSize: cfg.batchSize || 30, ...extra },
|
|
1433
|
+
),
|
|
1434
|
+
fallbackOnly: fallbackOnlyFields,
|
|
1435
|
+
});
|
|
1436
|
+
tallyContent(pairKey, fm.report, label);
|
|
1437
|
+
// Refused fields are remembered, whatever happens to the page.
|
|
1438
|
+
for (const [field, methods] of Object.entries(fm.refusedBy || {})) {
|
|
1439
|
+
refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
|
|
972
1440
|
}
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
// so the next sync retries it.
|
|
983
|
-
for (const [field, value] of Object.entries(fmResult)) {
|
|
984
|
-
const sourceValue = fieldsToTranslate[field];
|
|
985
|
-
if (typeof value !== 'string' || typeof sourceValue !== 'string') continue;
|
|
986
|
-
const hollowed = checkContentPreservation(sourceValue, value);
|
|
987
|
-
if (hollowed) {
|
|
988
|
-
throw new Error(
|
|
989
|
-
`Docusaurus content sync for ${code}: front matter "${field}" — ${hollowed.reason}.\n` +
|
|
990
|
-
` source: ${JSON.stringify(sourceValue)}\n` +
|
|
991
|
-
` got: ${JSON.stringify(value)}\n` +
|
|
992
|
-
' Nothing was written or cached. If this is a low-coverage target\n' +
|
|
993
|
-
' language, the model has no vocabulary for this string.'
|
|
994
|
-
);
|
|
995
|
-
}
|
|
996
|
-
}
|
|
997
|
-
Object.assign(translatedFields, fmResult);
|
|
998
|
-
// Cache only what the API actually returned (the same
|
|
999
|
-
// non-null check that gates writing it to the target file).
|
|
1000
|
-
for (const [field, value] of Object.entries(fmResult)) {
|
|
1001
|
-
if (typeof value === 'string' && typeof fieldsToTranslate[field] === 'string') {
|
|
1002
|
-
storeTM(tm, fieldsToTranslate[field], code, tmKey, value);
|
|
1003
|
-
}
|
|
1441
|
+
if (fm.hollowed.length > 0) {
|
|
1442
|
+
// Refused, and refused again when asked with the reason: the
|
|
1443
|
+
// field keeps its source text; the page is still written.
|
|
1444
|
+
fieldsLeftInSource = true;
|
|
1445
|
+
for (const h of fm.hollowed) {
|
|
1446
|
+
output.warn(
|
|
1447
|
+
`${label} → ${code}: front matter "${h.field}" kept in the source language — ${h.reason}`
|
|
1448
|
+
+ `${h.fallbackReason ? `; the fallback (${pairConfig.fallback.method}): ${h.fallbackReason}` : ''}.`
|
|
1449
|
+
);
|
|
1004
1450
|
}
|
|
1005
|
-
|
|
1451
|
+
output.warn(describeNewHold({ file: label, pairKey, pairConfig, count: fm.hollowed.length }));
|
|
1452
|
+
}
|
|
1453
|
+
if (fm.noResults) {
|
|
1006
1454
|
// Front matter translation failed — loud error
|
|
1007
1455
|
throw new Error(
|
|
1008
|
-
`Docusaurus content sync for ${code}: front matter translation returned no results
|
|
1456
|
+
`Docusaurus content sync for ${code}: front matter translation returned no results`
|
|
1457
|
+
+ `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
|
|
1009
1458
|
' Check your API key and method configuration.'
|
|
1010
1459
|
);
|
|
1011
1460
|
}
|
|
1461
|
+
// Cache only gate-passing values, each under the TM key of the
|
|
1462
|
+
// method that produced it.
|
|
1463
|
+
for (const st of fm.stores) storeTM(tm, st.text, code, st.tmKey, st.value);
|
|
1464
|
+
// A field only the fallback may be asked for that it did not
|
|
1465
|
+
// translate (skipped by --max-cost, or no answer): still refused
|
|
1466
|
+
// by the pair's method — the page is held, as with no fallback.
|
|
1467
|
+
const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
|
|
1468
|
+
if (unfilled.length > 0) {
|
|
1469
|
+
fieldsLeftInSource = true;
|
|
1470
|
+
const names = unfilled.map(f => `front matter "${f}"`);
|
|
1471
|
+
refusal.held.push(...names);
|
|
1472
|
+
output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names }));
|
|
1473
|
+
}
|
|
1474
|
+
Object.assign(translatedFields, fm.translated);
|
|
1475
|
+
for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
|
|
1012
1476
|
}
|
|
1013
1477
|
}
|
|
1014
1478
|
|
|
@@ -1020,40 +1484,92 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1020
1484
|
let translatedBody = body;
|
|
1021
1485
|
let bodyUsedFallback = false;
|
|
1022
1486
|
if (body.trim()) {
|
|
1023
|
-
|
|
1024
|
-
|
|
1487
|
+
// The pair's cache, then its fallback's (lib/fallback.js ladder).
|
|
1488
|
+
const wholeBodyCached = lookupContentTM(tm, body, code, pairConfig);
|
|
1025
1489
|
if (wholeBodyCached !== null) {
|
|
1026
|
-
translatedBody = wholeBodyCached;
|
|
1490
|
+
translatedBody = wholeBodyCached.text;
|
|
1491
|
+
for (const src of refusal.blockSources) refusal.filled.add(blockUnit(src));
|
|
1492
|
+
if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
|
|
1027
1493
|
} else {
|
|
1028
1494
|
const segMode = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
1029
1495
|
const { protectedBody, blocks } = protectBlocks(body);
|
|
1030
1496
|
const promptOptions = {
|
|
1031
1497
|
sourceLanguageName: DEFAULT_REGISTERS[inputLocale]?.name || inputLocale,
|
|
1032
1498
|
promptContext: pairConfig.promptContext || null,
|
|
1499
|
+
protectedTerms: pairConfig.protectedTerms || [],
|
|
1033
1500
|
};
|
|
1034
1501
|
|
|
1035
1502
|
// Block stores are deferred until the reassembled body passes
|
|
1036
1503
|
// the orphaned-placeholder check — the TM must never hold a
|
|
1037
|
-
// value that would fail the gate on re-serve.
|
|
1504
|
+
// value that would fail the gate on re-serve. Each carries the
|
|
1505
|
+
// TM key of the method that produced it.
|
|
1038
1506
|
const pendingBlockStores = [];
|
|
1507
|
+
// The whole body is cached under one method's key — only when
|
|
1508
|
+
// one method produced all of it.
|
|
1509
|
+
let wholeBodyKey = tmKey;
|
|
1510
|
+
let mixedProducers = false;
|
|
1511
|
+
// Page mode: whose whole page the checks below refuse (the
|
|
1512
|
+
// lane remembers it — lib/content-refusals.js "page").
|
|
1513
|
+
let pageRefusedBy = null;
|
|
1039
1514
|
|
|
1040
1515
|
if (segMode === 'page') {
|
|
1041
|
-
// Whole-page prompt — today's single-call behavior.
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
);
|
|
1516
|
+
// Whole-page prompt — today's single-call behavior. Refused
|
|
1517
|
+
// before by this method (and the cache does not hold it): not
|
|
1518
|
+
// sent again (lib/content-refusals.js).
|
|
1519
|
+
const pageHold = holds.page(body);
|
|
1520
|
+
if (pageHold === 'held') {
|
|
1521
|
+
const err = new Error('held');
|
|
1522
|
+
err.heldNames = [PAGE_NAME];
|
|
1523
|
+
throw err;
|
|
1524
|
+
}
|
|
1525
|
+
await requirePairReady(pairKey, pairConfig, code, 'body translation');
|
|
1526
|
+
const runPage = (cfg) => translateRawContent(
|
|
1527
|
+
buildContentPrompt(protectedBody, cfg, promptOptions), { apiKey, pairConfig: cfg });
|
|
1528
|
+
let bodyResult;
|
|
1529
|
+
if (pairConfig.fallback) {
|
|
1530
|
+
// When both fail, the primary's page goes to the checks
|
|
1531
|
+
// below, which fail the file exactly as without a fallback.
|
|
1532
|
+
// Refused before by the pair's method: only the fallback is asked.
|
|
1533
|
+
const page = await translatePageWithFallback({
|
|
1534
|
+
body, blocks, pairConfig, runPage, budget: fallbackBudget, fallbackOnly: pageHold === 'fallback-only',
|
|
1535
|
+
});
|
|
1536
|
+
tallyContent(pairKey, page.report, label);
|
|
1537
|
+
if (page.refusedBy.length > 0) {
|
|
1538
|
+
refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: page.refusedBy });
|
|
1539
|
+
}
|
|
1540
|
+
if (pageHold === 'fallback-only' && page.body === null) {
|
|
1541
|
+
// The fallback did not translate it: the pair's method
|
|
1542
|
+
// refused it before, so it is not asked — refused now
|
|
1543
|
+
// by the fallback too, or held (no answer, or skipped
|
|
1544
|
+
// by --max-cost).
|
|
1545
|
+
if (page.refusedBy.length > 0) {
|
|
1546
|
+
const err = new Error(
|
|
1547
|
+
`Docusaurus body for ${code}: the fallback (${pairConfig.fallback.method}) failed the whole-page checks too.\n` +
|
|
1548
|
+
` Nothing was written or cached. ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`
|
|
1549
|
+
);
|
|
1550
|
+
err.heldNext = true;
|
|
1551
|
+
throw err;
|
|
1552
|
+
}
|
|
1553
|
+
const err = new Error('held');
|
|
1554
|
+
err.heldNames = [PAGE_NAME];
|
|
1555
|
+
throw err;
|
|
1556
|
+
}
|
|
1557
|
+
bodyResult = page.body ?? page.primaryBody;
|
|
1558
|
+
if (page.body !== null) wholeBodyKey = page.tmKey;
|
|
1559
|
+
else if (page.refusedBy.length > 0) pageRefusedBy = page.refusedBy;
|
|
1560
|
+
} else {
|
|
1561
|
+
const raw = await runPage(pairConfig);
|
|
1562
|
+
bodyResult = raw ? restoreBlocks(raw, blocks) : null;
|
|
1563
|
+
if (bodyResult) pageRefusedBy = [tmKey];
|
|
1047
1564
|
}
|
|
1048
|
-
const prompt = buildContentPrompt(protectedBody, pairConfig, promptOptions);
|
|
1049
|
-
const bodyResult = await translateRawContent(prompt, { apiKey, pairConfig });
|
|
1050
1565
|
if (!bodyResult) {
|
|
1051
1566
|
throw new Error(
|
|
1052
|
-
`Docusaurus body for ${code}: translation returned no results
|
|
1567
|
+
`Docusaurus body for ${code}: translation returned no results`
|
|
1568
|
+
+ `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
|
|
1053
1569
|
' Check your API key and method configuration.'
|
|
1054
1570
|
);
|
|
1055
1571
|
}
|
|
1056
|
-
translatedBody =
|
|
1572
|
+
translatedBody = bodyResult;
|
|
1057
1573
|
} else {
|
|
1058
1574
|
// Block mode: segment the PROTECTED body (placeholders are
|
|
1059
1575
|
// single tokens, so they can never be split), serve blocks
|
|
@@ -1073,96 +1589,162 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1073
1589
|
}));
|
|
1074
1590
|
|
|
1075
1591
|
const missed = [];
|
|
1592
|
+
const heldBlocks = []; // refused before: not sent, the source text kept
|
|
1593
|
+
let position = 0; // place among the translatable blocks
|
|
1076
1594
|
for (const r of rendered) {
|
|
1077
1595
|
if (r.seg.type !== 'translatable') {
|
|
1078
1596
|
// Separators + passthrough blocks: copied verbatim, never billed.
|
|
1079
1597
|
r.out = r.source;
|
|
1080
1598
|
continue;
|
|
1081
1599
|
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1600
|
+
// Its name in the repeat check (content:<file>#<n>).
|
|
1601
|
+
r.pos = position++;
|
|
1602
|
+
const cached = lookupContentTM(tm, r.source, code, pairConfig);
|
|
1084
1603
|
if (cached !== null) {
|
|
1085
|
-
r.out = cached;
|
|
1086
|
-
|
|
1087
|
-
|
|
1604
|
+
r.out = cached.text;
|
|
1605
|
+
if (cached.fromFallback) { mixedProducers = true; notePage(pairKey, label); }
|
|
1606
|
+
refusal.filled.add(blockUnit(r.source));
|
|
1607
|
+
continue;
|
|
1088
1608
|
}
|
|
1609
|
+
const hold = holds.block(r.source);
|
|
1610
|
+
if (hold === 'held') {
|
|
1611
|
+
// Refused before by this method (and its fallback): the
|
|
1612
|
+
// source text stays, nothing is sent or billed.
|
|
1613
|
+
r.out = r.source;
|
|
1614
|
+
heldBlocks.push(r);
|
|
1615
|
+
continue;
|
|
1616
|
+
}
|
|
1617
|
+
r.fallbackOnly = hold === 'fallback-only';
|
|
1618
|
+
missed.push(r);
|
|
1619
|
+
}
|
|
1620
|
+
if (heldBlocks.length > 0) {
|
|
1621
|
+
bodyUsedFallback = true;
|
|
1622
|
+
const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
|
|
1623
|
+
refusal.held.push(...names);
|
|
1624
|
+
output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names }));
|
|
1089
1625
|
}
|
|
1090
1626
|
|
|
1091
1627
|
if (missed.length > 0) {
|
|
1092
|
-
|
|
1093
|
-
throw new Error(
|
|
1094
|
-
`Docusaurus body translation for ${code}: no API key available.\n` +
|
|
1095
|
-
' Set OPENROUTER_API_KEY in .env.local to translate content.'
|
|
1096
|
-
);
|
|
1097
|
-
}
|
|
1628
|
+
await requirePairReady(pairKey, pairConfig, code, 'body translation');
|
|
1098
1629
|
// Self-repair ladder (translateBlockBatchResilient): full
|
|
1099
|
-
// batch → one missing-segments-only retry →
|
|
1100
|
-
//
|
|
1630
|
+
// batch → one missing-segments-only retry → (with a
|
|
1631
|
+
// fallback method: one batch through it for every block the
|
|
1632
|
+
// primary dropped or damaged — lib/fallback.js) → honest
|
|
1633
|
+
// the source text, unmarked, for anything still missing. A
|
|
1101
1634
|
// duplicate/unknown marker or an empty first response
|
|
1102
|
-
// still fails the file whole.
|
|
1103
|
-
const
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1635
|
+
// still fails the file whole when no fallback rescues it.
|
|
1636
|
+
const outcome = await translateBlocksWithFallback({
|
|
1637
|
+
missed,
|
|
1638
|
+
blocks,
|
|
1639
|
+
pairConfig,
|
|
1640
|
+
budget: fallbackBudget,
|
|
1641
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
1642
|
+
label: docLabel,
|
|
1643
|
+
runBatch: (texts, cfg) => translateBlockBatchResilient({
|
|
1644
|
+
texts,
|
|
1645
|
+
buildPrompt: (t) => buildBlockBatchPrompt(t, cfg, { ...promptOptions, pageTitle }),
|
|
1646
|
+
callModel: (prompt) => translateRawContent(prompt, { apiKey, pairConfig: cfg }),
|
|
1647
|
+
}),
|
|
1648
|
+
fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
|
|
1649
|
+
});
|
|
1650
|
+
tallyContent(pairKey, outcome.report, label);
|
|
1651
|
+
// Refused blocks are remembered; translated ones drop a record.
|
|
1652
|
+
const fellSet = new Set(outcome.fellBack);
|
|
1653
|
+
const newlyHeld = [];
|
|
1654
|
+
missed.forEach((r, i) => {
|
|
1655
|
+
if (!fellSet.has(i)) { refusal.filled.add(blockUnit(r.source)); return; }
|
|
1656
|
+
const methods = outcome.refusedBy?.[i] || [];
|
|
1657
|
+
if (methods.length > 0) {
|
|
1658
|
+
refusal.refused.push({ unit: blockUnit(r.source), source: r.source, methods });
|
|
1659
|
+
newlyHeld.push(i);
|
|
1660
|
+
}
|
|
1661
|
+
});
|
|
1662
|
+
if (outcome.fromFallback > 0) mixedProducers = true;
|
|
1663
|
+
if ((outcome.sharedOutput || []).length > 0) {
|
|
1664
|
+
output.warn(
|
|
1665
|
+
`Docusaurus body for ${code}: ${outcome.sharedOutput.length} block(s) of ${dirName}/${relPath} came back as the same text ` +
|
|
1666
|
+
'the model gave for other, different source strings (a memorized sentence, not a translation) — refused.'
|
|
1667
|
+
);
|
|
1668
|
+
}
|
|
1669
|
+
if ((outcome.refused || []).length > 0) {
|
|
1670
|
+
output.warn(
|
|
1671
|
+
`Docusaurus body for ${code}: ${outcome.refused.length} block(s) of ${dirName}/${relPath} refused by the quality gate, also when asked again with the reason — `
|
|
1672
|
+
+ outcome.refused.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
|
|
1673
|
+
+ `${outcome.refused.length > 3 ? '; …' : ''}.`
|
|
1674
|
+
);
|
|
1675
|
+
}
|
|
1676
|
+
const { fellBack } = outcome;
|
|
1112
1677
|
if (fellBack.length > 0) {
|
|
1113
1678
|
bodyUsedFallback = true;
|
|
1679
|
+
const refusedHere = (outcome.sharedOutput || []).length + (outcome.refused || []).length;
|
|
1680
|
+
const why = outcome.report?.attempted > 0
|
|
1681
|
+
? `neither the primary (${pairConfig.method}) nor its fallback (${pairConfig.fallback.method}) translated safely`
|
|
1682
|
+
: refusedHere >= fellBack.length ? 'refused (above)'
|
|
1683
|
+
: refusedHere > 0 ? `refused (${refusedHere}, above) or missing from the model response after a retry`
|
|
1684
|
+
: 'missing from the model response after a retry';
|
|
1114
1685
|
output.warn(
|
|
1115
1686
|
`Docusaurus body for ${code}: ${fellBack.length} of ${missed.length} ` +
|
|
1116
|
-
`block(s)
|
|
1117
|
-
|
|
1118
|
-
|
|
1687
|
+
`block(s) ${why} — left in the source language, unmarked (\`champollion status\` lists them). ` +
|
|
1688
|
+
describeFallenBack({
|
|
1689
|
+
file: label, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: outcome.refusedBy,
|
|
1690
|
+
})
|
|
1119
1691
|
);
|
|
1120
1692
|
}
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
if (!fellBackSet.has(i)) {
|
|
1126
|
-
pendingBlockStores.push({ source: r.source, translation: r.out });
|
|
1127
|
-
}
|
|
1128
|
-
});
|
|
1693
|
+
// Fallen-back segments are never TM-cached — an error
|
|
1694
|
+
// cached is an error forever (they re-bill next sync).
|
|
1695
|
+
missed.forEach((r, i) => { r.out = outcome.outs[i]; });
|
|
1696
|
+
pendingBlockStores.push(...outcome.stores);
|
|
1129
1697
|
}
|
|
1130
1698
|
|
|
1131
1699
|
// Reassemble in order with the source's exact separators.
|
|
1132
1700
|
translatedBody = rendered.map(r => r.out).join('');
|
|
1133
1701
|
}
|
|
1134
1702
|
|
|
1703
|
+
// A page translated whole that fails the checks below is
|
|
1704
|
+
// remembered as refused: the next plain sync does not send it
|
|
1705
|
+
// to the same method again (lib/content-refusals.js "page").
|
|
1706
|
+
const refusePage = (err) => {
|
|
1707
|
+
if (!pageRefusedBy) return err;
|
|
1708
|
+
if (!refusal.refused.some(r => r.unit === PAGE_UNIT)) {
|
|
1709
|
+
refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: pageRefusedBy });
|
|
1710
|
+
}
|
|
1711
|
+
err.message += `\n ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`;
|
|
1712
|
+
err.heldNext = true;
|
|
1713
|
+
return err;
|
|
1714
|
+
};
|
|
1715
|
+
|
|
1135
1716
|
// Orphaned-placeholder check on the REASSEMBLED body — the
|
|
1136
1717
|
// same gate for both modes.
|
|
1137
1718
|
if (hasOrphanedPlaceholders(translatedBody)) {
|
|
1138
|
-
throw new Error(
|
|
1719
|
+
throw refusePage(new Error(
|
|
1139
1720
|
`Docusaurus body for ${code}: placeholder corruption detected.\n` +
|
|
1140
1721
|
' Code blocks were corrupted during translation.'
|
|
1141
|
-
);
|
|
1722
|
+
));
|
|
1142
1723
|
}
|
|
1143
1724
|
|
|
1144
1725
|
// Content-preservation check on the REASSEMBLED body — same
|
|
1145
1726
|
// lane, same reasoning as the front matter above. Skipped for a
|
|
1146
|
-
// fallback body: it deliberately carries
|
|
1727
|
+
// fallback body: it deliberately carries untranslated
|
|
1147
1728
|
// source text and is neither cached nor lock-advanced already.
|
|
1148
1729
|
const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
|
|
1149
1730
|
if (bodyHollowed) {
|
|
1150
|
-
throw new Error(
|
|
1731
|
+
throw refusePage(new Error(
|
|
1151
1732
|
`Docusaurus body for ${code}: ${bodyHollowed.reason}.\n` +
|
|
1152
1733
|
' Nothing was written or cached. If this is a low-coverage target\n' +
|
|
1153
1734
|
' language, the model has no vocabulary for this text.'
|
|
1154
|
-
);
|
|
1735
|
+
));
|
|
1155
1736
|
}
|
|
1737
|
+
if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
|
|
1156
1738
|
|
|
1157
1739
|
// Store per-block AND whole-body entries only after the check
|
|
1158
1740
|
// passes (whole-body makes reverts/lock-loss re-runs free). A
|
|
1159
1741
|
// fallback body is NEVER stored whole — it contains
|
|
1160
|
-
// untranslated
|
|
1742
|
+
// untranslated source text.
|
|
1161
1743
|
for (const s of pendingBlockStores) {
|
|
1162
|
-
storeTM(tm, s.source, code, tmKey, s.translation);
|
|
1744
|
+
storeTM(tm, s.source, code, s.tmKey, s.translation);
|
|
1163
1745
|
}
|
|
1164
|
-
if (!bodyUsedFallback) {
|
|
1165
|
-
storeTM(tm, body, code,
|
|
1746
|
+
if (!bodyUsedFallback && !mixedProducers) {
|
|
1747
|
+
storeTM(tm, body, code, wholeBodyKey, translatedBody);
|
|
1166
1748
|
}
|
|
1167
1749
|
}
|
|
1168
1750
|
}
|
|
@@ -1180,16 +1762,48 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1180
1762
|
// keeps its old entry (or none), so it re-fires next sync — as
|
|
1181
1763
|
// does a fallback body (its re-fire is TM-cheap: only the
|
|
1182
1764
|
// fallen-back segment re-bills).
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
}
|
|
1765
|
+
updatedDocuManifest[manifestKey] = !bodyUsedFallback && !fieldsLeftInSource
|
|
1766
|
+
? sourceHash : pendingLockValue(sourceHash);
|
|
1767
|
+
manifestDirty = true;
|
|
1187
1768
|
|
|
1188
1769
|
} catch (contentErr) {
|
|
1770
|
+
if (contentErr.heldNames) {
|
|
1771
|
+
// A front-matter field refused before: held back, not failed —
|
|
1772
|
+
// nothing was sent, nothing written, the lock not advanced.
|
|
1773
|
+
const refusal = refusalStates.get(manifestKey);
|
|
1774
|
+
refusal.held.push(...contentErr.heldNames);
|
|
1775
|
+
refusal.pageHeld = true;
|
|
1776
|
+
settleRefusals();
|
|
1777
|
+
output.warn(describeHeld({
|
|
1778
|
+
file: `${dirName}/${relPath}`, code, pairKey, pairConfig, names: contentErr.heldNames, pageHeld: true,
|
|
1779
|
+
}));
|
|
1780
|
+
completed++;
|
|
1781
|
+
output.raw(` [${completed}/${totalWork}] ${dirName}/${relPath} → ${code} [HELD]`);
|
|
1782
|
+
output.event('file', { lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: 'held-refused', held: contentErr.heldNames.length, action });
|
|
1783
|
+
if (completed % MANIFEST_WRITE_INTERVAL === 0) writeManifestIfDirty();
|
|
1784
|
+
return;
|
|
1785
|
+
}
|
|
1786
|
+
itemFailed = true;
|
|
1189
1787
|
contentFailures++;
|
|
1190
1788
|
output.error(`${dirName}/${relPath} → ${code} — ${contentErr.message}`);
|
|
1789
|
+
// Nothing was written for this item, and its lock entry was not
|
|
1790
|
+
// advanced. If the lock already matches this source, the file on
|
|
1791
|
+
// disk is the PREVIOUS translation of the same text (a forced
|
|
1792
|
+
// re-translate that failed): it stays, and a plain sync will not
|
|
1793
|
+
// redo it. Otherwise the next sync retries it.
|
|
1794
|
+
const keptPrevious = updatedDocuManifest[manifestKey] === sourceHash && fs.existsSync(targetPath);
|
|
1795
|
+
failedItems.push({
|
|
1796
|
+
file: `${dirName}/${relPath}`,
|
|
1797
|
+
locale: code,
|
|
1798
|
+
error: contentErr.message,
|
|
1799
|
+
// held-back: refused by the quality gate — the next plain sync
|
|
1800
|
+
// does not send it again (lib/content-refusals.js).
|
|
1801
|
+
state: keptPrevious ? 'previous-translation-kept' : contentErr.heldNext ? 'held-back' : 'will-retry',
|
|
1802
|
+
});
|
|
1191
1803
|
}
|
|
1192
1804
|
|
|
1805
|
+
settleRefusals();
|
|
1806
|
+
|
|
1193
1807
|
// Progress reporting
|
|
1194
1808
|
completed++;
|
|
1195
1809
|
const pct = Math.round(100 * completed / totalWork);
|
|
@@ -1198,13 +1812,16 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1198
1812
|
const remainingMs = msPerItem * (totalWork - completed);
|
|
1199
1813
|
const remainingSec = Math.ceil(remainingMs / 1000);
|
|
1200
1814
|
const etaStr = remainingSec > 5 ? ` (~${remainingSec}s left)` : '';
|
|
1201
|
-
|
|
1815
|
+
// The tag describes THIS item. (It used to read the run-wide failure
|
|
1816
|
+
// count, so whichever item finished last was tagged FAIL.)
|
|
1817
|
+
const displayTag = itemFailed
|
|
1202
1818
|
? 'FAIL'
|
|
1203
1819
|
: action === 're-translate' ? 'RE-TRANSLATE' : action === 'changed' ? 'CHANGED' : 'OK';
|
|
1204
|
-
// Show [FAIL] for items that just errored (contentErr was caught above)
|
|
1205
|
-
const itemFailed = updatedDocuManifest[manifestKey] !== sourceHash;
|
|
1206
|
-
const displayTag = itemFailed && !dryRun ? 'FAIL' : tag;
|
|
1207
1820
|
output.raw(` [${completed}/${totalWork}] (${pct}%) ${dirName}/${relPath} → ${code} [${displayTag}]${etaStr}`);
|
|
1821
|
+
output.event('file', {
|
|
1822
|
+
lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code,
|
|
1823
|
+
status: itemFailed ? 'failed' : dryRun ? 'would-translate' : 'translated', action,
|
|
1824
|
+
});
|
|
1208
1825
|
|
|
1209
1826
|
// Incremental manifest write
|
|
1210
1827
|
if (completed % MANIFEST_WRITE_INTERVAL === 0) {
|
|
@@ -1231,26 +1848,122 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1231
1848
|
output.ok(`${action} ${totalContent} content file(s)${retranslateNote}, ${totalContentSkipped} unchanged`);
|
|
1232
1849
|
}
|
|
1233
1850
|
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1851
|
+
contentTranslated = totalContent;
|
|
1852
|
+
contentFailedItems = failedItems;
|
|
1853
|
+
for (const st of refusalStates.values()) {
|
|
1854
|
+
if (st.held.length > 0) {
|
|
1855
|
+
contentHeldBack += st.held.length;
|
|
1856
|
+
contentHeldBackItems.push({ file: st.file, locale: st.locale, units: st.held, ...(st.pageHeld && { pageNotWritten: true }) });
|
|
1857
|
+
}
|
|
1858
|
+
contentRefused += new Set(st.refused.map(r => r.unit)).size;
|
|
1859
|
+
}
|
|
1860
|
+
if (contentHeldBack > 0) {
|
|
1861
|
+
output.warn(
|
|
1862
|
+
`${contentHeldBack} content block(s)/field(s) held back in ${contentHeldBackItems.length} translation(s) — the quality gate refused `
|
|
1863
|
+
+ 'the method\'s translation of their current text before; not sent, not billed (listed above). Ask again for all of them: '
|
|
1864
|
+
+ '`champollion sync --redo content`; for one page: `champollion sync --redo files:<page>`.'
|
|
1239
1865
|
);
|
|
1240
1866
|
}
|
|
1867
|
+
for (const [pairKey, tally] of contentFallbackTallies) {
|
|
1868
|
+
printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)', producedLabel: 'pages' });
|
|
1869
|
+
warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)' });
|
|
1870
|
+
}
|
|
1871
|
+
if (failedItems.length > 0) {
|
|
1872
|
+
output.raw('');
|
|
1873
|
+
output.raw(` Failed content translations (${failedItems.length}):`);
|
|
1874
|
+
for (const f of failedItems.sort((a, b) => (a.file + a.locale).localeCompare(b.file + b.locale))) {
|
|
1875
|
+
const state = f.state === 'previous-translation-kept'
|
|
1876
|
+
? 'previous translation kept (it matches the current source); re-run with --force-content to try again'
|
|
1877
|
+
: f.state === 'held-back'
|
|
1878
|
+
? `refused by the quality gate — held back: the next sync does not send it again (\`--redo files:${f.file}\` asks again)`
|
|
1879
|
+
: 'not recorded as done — the next sync retries it';
|
|
1880
|
+
output.raw(` ${f.file} → ${f.locale}: ${state}`);
|
|
1881
|
+
}
|
|
1882
|
+
}
|
|
1241
1883
|
}
|
|
1242
1884
|
|
|
1243
1885
|
// Save TM if it was mutated during this Docusaurus sync (stores OR
|
|
1244
1886
|
// evictions — a size check would miss eviction-only runs and same-key
|
|
1245
|
-
// replacements). Skip when --no-tm is active.
|
|
1887
|
+
// replacements). Skip when --no-tm is active. Saved BEFORE any failure is
|
|
1888
|
+
// raised: the files that succeeded paid for their translations, and
|
|
1889
|
+
// throwing first discarded every one of them from the cache.
|
|
1246
1890
|
if (!dryRun && !noTM && isTMDirty(tm)) {
|
|
1247
|
-
const tmFinalSize = tmSize(tm);
|
|
1248
|
-
const delta = tmFinalSize - tmInitialSize;
|
|
1249
1891
|
saveTM(cwd, tm);
|
|
1250
|
-
output.info(`[TM] Saved ${
|
|
1892
|
+
output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
|
|
1893
|
+
}
|
|
1894
|
+
|
|
1895
|
+
// Failures are reported, not thrown: a partly failed run still did real
|
|
1896
|
+
// work, and the exit code says so (2 = partial, 1 = nothing succeeded).
|
|
1897
|
+
if (contentFailures > 0) {
|
|
1898
|
+
output.error(
|
|
1899
|
+
`${contentFailures} content translation(s) failed (listed above). ` +
|
|
1900
|
+
'Completed files and their translations are saved; re-run sync to retry.'
|
|
1901
|
+
);
|
|
1251
1902
|
}
|
|
1252
1903
|
|
|
1904
|
+
// The --max-cost verdict, said once — at the end, before the summary
|
|
1905
|
+
// (lib/cost-report.js reportDryRunMaxCost).
|
|
1906
|
+
if (dryMaxCost) output[dryMaxCost.level](dryMaxCost.message);
|
|
1907
|
+
|
|
1908
|
+
// One machine-readable record of the run — the Docusaurus path never had
|
|
1909
|
+
// one, so --json consumers saw nothing to tell success from failure.
|
|
1910
|
+
output.summary({
|
|
1911
|
+
command: 'sync',
|
|
1912
|
+
format: 'docusaurus',
|
|
1913
|
+
dryRun,
|
|
1914
|
+
totalProcessed: totalJSONKeys,
|
|
1915
|
+
totalFailed: totalJSONFailed,
|
|
1916
|
+
totalCopied: totalJSONCopied,
|
|
1917
|
+
// UI strings held back: the gate refused this method's translation of
|
|
1918
|
+
// their current text before — not sent, not billed (lib/locale-state.js).
|
|
1919
|
+
// A dry run sends nothing either way: said above, not counted (as on
|
|
1920
|
+
// the key-value path).
|
|
1921
|
+
totalHeld: dryRun ? 0 : totalJSONHeld,
|
|
1922
|
+
// What each pair's fallback method did for JSON keys (pairs with a
|
|
1923
|
+
// fallback, real runs); Markdown segments are under content.fallback.
|
|
1924
|
+
...(keyFallbackTallies.size > 0 && {
|
|
1925
|
+
fallback: [...keyFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
|
|
1926
|
+
}),
|
|
1927
|
+
content: {
|
|
1928
|
+
translated: contentTranslated,
|
|
1929
|
+
skipped: contentScan.totalContentSkipped,
|
|
1930
|
+
failed: contentFailedItems.length,
|
|
1931
|
+
failedItems: contentFailedItems,
|
|
1932
|
+
// Refused before by the method, not sent this run (lib/content-refusals.js),
|
|
1933
|
+
// and refused this run (held back from the next).
|
|
1934
|
+
heldBack: contentHeldBack,
|
|
1935
|
+
heldBackItems: contentHeldBackItems,
|
|
1936
|
+
refused: contentRefused,
|
|
1937
|
+
...(contentFallbackTallies.size > 0 && {
|
|
1938
|
+
fallback: [...contentFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
|
|
1939
|
+
}),
|
|
1940
|
+
},
|
|
1941
|
+
costEstimate,
|
|
1942
|
+
// Dry runs with --max-cost: whether the real run would stop at the cap.
|
|
1943
|
+
...(dryMaxCost && { maxCost: {
|
|
1944
|
+
cap: dryMaxCost.cap, estimatedCost: dryMaxCost.estimatedCost, wouldStop: dryMaxCost.wouldStop,
|
|
1945
|
+
...(dryMaxCost.wouldStop && { exitCode: 2, reason: dryMaxCost.reason }),
|
|
1946
|
+
...(dryMaxCost.stopsEarlier && { exitCode: 1, stopsEarlier: dryMaxCost.stopsEarlier }),
|
|
1947
|
+
} }),
|
|
1948
|
+
// Keys named for a redo that match no UI string (the run exits 1), each
|
|
1949
|
+
// with the closest ids that exist.
|
|
1950
|
+
...(unmatchedNamed.length > 0 && { unmatchedKeys: unmatchedKeysSummary(unmatchedNamed) }),
|
|
1951
|
+
});
|
|
1253
1952
|
output.raw('');
|
|
1953
|
+
// Named keys that matched nothing: said again last, where a reader (or CI)
|
|
1954
|
+
// looks for the verdict — and the run fails (lib/commands/sync.js).
|
|
1955
|
+
reportUnmatchedKeys(unmatchedNamed, cliArgs);
|
|
1956
|
+
return {
|
|
1957
|
+
totalProcessed: totalJSONKeys,
|
|
1958
|
+
totalFailed: totalJSONFailed,
|
|
1959
|
+
totalCopied: totalJSONCopied,
|
|
1960
|
+
totalHeld: dryRun ? 0 : totalJSONHeld,
|
|
1961
|
+
contentTranslated,
|
|
1962
|
+
contentFailed: contentFailedItems.length,
|
|
1963
|
+
contentHeldBack,
|
|
1964
|
+
contentRefused,
|
|
1965
|
+
unmatchedKeys: unmatchedNamed.map(m => m.name),
|
|
1966
|
+
};
|
|
1254
1967
|
}
|
|
1255
1968
|
|
|
1256
1969
|
export { runDocusaurusSync, discoverDocusaurusJSONFiles };
|