champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
package/lib/docusaurus-sync.js
CHANGED
|
@@ -29,29 +29,54 @@ import {
|
|
|
29
29
|
discoverDocusaurusContentFiles, getDocusaurusTargetPath,
|
|
30
30
|
findUntranslatableNestedFields,
|
|
31
31
|
} from './content.js';
|
|
32
|
-
import { translateRawContent } from './translate.js';
|
|
32
|
+
import { translateRawContent, getMethod } from './translate.js';
|
|
33
33
|
import {
|
|
34
34
|
splitBlocks, buildBlockBatchPrompt, parseBlockBatchResponse,
|
|
35
35
|
translateBlockBatchResilient,
|
|
36
36
|
assertSegmentationMode,
|
|
37
37
|
} from './segment.js';
|
|
38
38
|
import { DEFAULT_REGISTERS } from './registers.js';
|
|
39
|
-
import { DEFAULT_JSON_CONCURRENCY
|
|
39
|
+
import { DEFAULT_JSON_CONCURRENCY } from './config.js';
|
|
40
40
|
import { compileNoTranslate } from './no-translate.js';
|
|
41
|
-
import { checkContentPreservation } from './validate.js';
|
|
41
|
+
import { checkContentPreservation, contentGateFault, SharedOutputIndex } from './validate.js';
|
|
42
42
|
import { convertScript, applyScriptFallback } from './scripts.js';
|
|
43
43
|
import { pMap } from './concurrent.js';
|
|
44
44
|
import {
|
|
45
45
|
loadTM, saveTM, tmSize, isTMDirty,
|
|
46
|
-
lookupTM,
|
|
46
|
+
lookupTM, peekTM, storeTM, evictTM, partitionByTM, tmMethodKey, setModelCarryover, bypassTMFor,
|
|
47
|
+
describeTMChanges, adoptLegacyCoachingKeys,
|
|
47
48
|
} from './tm.js';
|
|
48
49
|
import {
|
|
49
|
-
|
|
50
|
+
readLock, writeManifest, detectChangedKeys, hashValue,
|
|
50
51
|
} from './hash.js';
|
|
52
|
+
import {
|
|
53
|
+
LockState, planQueue, recordRefusal, describeHeldKeys, describeFallbackOnlyKeys, keyFateNote, describeHeldNext,
|
|
54
|
+
shortSourceHash,
|
|
55
|
+
} from './locale-state.js';
|
|
56
|
+
import { redoCommand } from './verify.js';
|
|
51
57
|
import { CONTENT_LOCK_FILENAME } from './content-sync.js';
|
|
52
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
parseMaxCost, abortForMaxCost, maxCostVerdict, reportDryRunMaxCost, printCostTable, warnModelSwitchStrandedTM, priceContentChars,
|
|
60
|
+
preflightStopReason,
|
|
61
|
+
summarizeEstimate,
|
|
62
|
+
} from './cost-report.js';
|
|
63
|
+
import { billableContentChars, translatableBlockSources } from './content-estimate.js';
|
|
64
|
+
import { compileFileScope } from './file-scope.js';
|
|
53
65
|
import { output } from './output.js';
|
|
54
|
-
import {
|
|
66
|
+
import { missingKeyAdvice } from './missing-key.js';
|
|
67
|
+
import { translateWithFallback } from './translate-pair.js';
|
|
68
|
+
import {
|
|
69
|
+
tmKeysForPair, tmHoldsValue, createFallbackBudget, newFallbackReport, addToTally,
|
|
70
|
+
fallbackSummary, printFallbackReport, warnFallbackMajority, lookupContentTM, serveFieldsFromFallbackCache,
|
|
71
|
+
translateFieldsWithFallback, translateBlocksWithFallback, translatePageWithFallback,
|
|
72
|
+
refuseRepeatedCachedPage,
|
|
73
|
+
} from './fallback.js';
|
|
74
|
+
import { walkFiles } from './locale-layout.js';
|
|
75
|
+
import {
|
|
76
|
+
readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
|
|
77
|
+
describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME,
|
|
78
|
+
} from './content-refusals.js';
|
|
79
|
+
import { applyNamedKeyRule, reportUnmatchedKeys, unmatchedKeysSummary, reportNamedFromCache } from './named-keys.js';
|
|
55
80
|
|
|
56
81
|
/**
|
|
57
82
|
* Discover all JSON locale files in a Docusaurus i18n source directory.
|
|
@@ -63,20 +88,64 @@ import { translateAndValidate } from './translate-pair.js';
|
|
|
63
88
|
* @returns {string[]} Absolute paths to JSON files
|
|
64
89
|
*/
|
|
65
90
|
function discoverDocusaurusJSONFiles(sourceLocaleDir) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
91
|
+
// The shared folder-per-locale walk (lib/locale-layout.js). Docusaurus
|
|
92
|
+
// keeps its historical reach: hidden entries are NOT skipped here.
|
|
93
|
+
return walkFiles(sourceLocaleDir, name => name.endsWith('.json'), { skipHidden: false });
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* What one (UI-string file × locale) queue becomes once the lock's refusal
|
|
98
|
+
* records are consulted — the key-value lane's rule (lib/locale-state.js
|
|
99
|
+
* planQueue), shared by the estimate and the sync so the estimate prices
|
|
100
|
+
* exactly what the run sends. A key the gate refused for its current source
|
|
101
|
+
* text is held back from the method that refused it (its fallback, when it
|
|
102
|
+
* has not refused it, is asked instead) unless it is named for a redo
|
|
103
|
+
* (`--redo keys:`), queued by `--redo all`, or the run is `--fresh`.
|
|
104
|
+
*
|
|
105
|
+
* Docusaurus records no written values, so no value counts as a person's
|
|
106
|
+
* edit here (a bulk redo re-translates every UI string, as it always did),
|
|
107
|
+
* and there is no pending retry: a UI string refused under a redo is held
|
|
108
|
+
* back by the next plain sync, as a content block is.
|
|
109
|
+
*
|
|
110
|
+
* @param {object} p
|
|
111
|
+
* @param {object} p.diff - diffLocale result for the file × locale
|
|
112
|
+
* @param {object} p.sourceFlat
|
|
113
|
+
* @param {object} p.targetFlat
|
|
114
|
+
* @param {string} p.nsPrefix - "docusaurus:<relPath>:" (the lock's key space)
|
|
115
|
+
* @param {object} p.localeState - LockState.of(code) / .peek(code)
|
|
116
|
+
* @param {object} p.pairConfig
|
|
117
|
+
* @param {{ named: Set<string>, bulk: boolean, fresh: boolean }} p.redo
|
|
118
|
+
* @param {string} p.fallbackPrefix
|
|
119
|
+
* @returns {{ held: string[], heldFromPrimary: string[] }}
|
|
120
|
+
*/
|
|
121
|
+
function planUIStrings({ diff, sourceFlat, targetFlat, nsPrefix, localeState, pairConfig, redo, fallbackPrefix }) {
|
|
122
|
+
const plan = planQueue({
|
|
123
|
+
diff, sourceFlat, targetFlat, lockKeyOf: (k) => nsPrefix + k, localeState,
|
|
124
|
+
named: redo.named, bulk: redo.bulk, pending: new Set(), fresh: redo.fresh, pairConfig,
|
|
125
|
+
classify: () => 'machine',
|
|
126
|
+
fallbackPrefix,
|
|
127
|
+
});
|
|
128
|
+
return { held: plan.held, heldFromPrimary: plan.heldFromPrimary };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* UI strings refused for an older source text that the file still holds as
|
|
133
|
+
* the English copy of that text (Docusaurus needs every id, so an untranslated
|
|
134
|
+
* one is written as its source message): the source was edited since, which
|
|
135
|
+
* lifts the hold — queued as changed. Without this the edit was invisible: the
|
|
136
|
+
* refused string never had its source hash recorded, and the old English no
|
|
137
|
+
* longer equals the new source, so nothing queued it.
|
|
138
|
+
*
|
|
139
|
+
* @returns {string[]}
|
|
140
|
+
*/
|
|
141
|
+
function editedSinceRefused(sourceFlat, targetFlat, nsPrefix, localeState) {
|
|
142
|
+
const out = [];
|
|
143
|
+
for (const [k, src] of Object.entries(sourceFlat)) {
|
|
144
|
+
const rec = localeState.refused[nsPrefix + k];
|
|
145
|
+
if (!rec || typeof src !== 'string' || typeof targetFlat[k] !== 'string') continue;
|
|
146
|
+
if (rec.source !== shortSourceHash(src) && shortSourceHash(targetFlat[k]) === rec.source) out.push(k);
|
|
77
147
|
}
|
|
78
|
-
|
|
79
|
-
return files.sort();
|
|
148
|
+
return out;
|
|
80
149
|
}
|
|
81
150
|
|
|
82
151
|
/**
|
|
@@ -97,12 +166,13 @@ function discoverDocusaurusJSONFiles(sourceLocaleDir) {
|
|
|
97
166
|
* @param {object} config - Resolved config (localesDir)
|
|
98
167
|
* @param {object} manifest - Content lock manifest (hash per file × locale)
|
|
99
168
|
* @param {boolean} forceContent - --force-content: re-process up-to-date files
|
|
169
|
+
* @param {import('./file-scope.js').FileScope|null} [fileScope] - --files / --retranslate
|
|
100
170
|
* too (action 're-translate'), WITHOUT clearing the manifest — a target
|
|
101
171
|
* with no lock entry and no '[EN] ' markers is a genuine hand-translated
|
|
102
172
|
* file, and force must never overwrite human work with machine output.
|
|
103
173
|
* @returns {{ workItems: Array<object>, totalContentSkipped: number, recordedHashes: object }}
|
|
104
174
|
*/
|
|
105
|
-
function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent) {
|
|
175
|
+
function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest, forceContent, fileScope = null) {
|
|
106
176
|
const workItems = [];
|
|
107
177
|
let totalContentSkipped = 0;
|
|
108
178
|
const recordedHashes = {};
|
|
@@ -115,6 +185,9 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
115
185
|
|
|
116
186
|
for (const sourcePath of sourceFiles) {
|
|
117
187
|
const relPath = path.relative(sourceContentDir, sourcePath);
|
|
188
|
+
const label = `${dirName}/${relPath}`;
|
|
189
|
+
if (fileScope && !fileScope.includes(label)) continue;
|
|
190
|
+
const retranslate = fileScope ? fileScope.retranslates(label) : false;
|
|
118
191
|
|
|
119
192
|
// Read + parse source file once per source path. The staleness hash
|
|
120
193
|
// covers the RAW file (same as the Hugo twin's hashFileContent): a
|
|
@@ -142,7 +215,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
142
215
|
}
|
|
143
216
|
const { parsed, fieldsToTranslate, pageTitle, sourceHash } = sourceCache.get(sourcePath);
|
|
144
217
|
|
|
145
|
-
for (const [, pairConfig] of pairEntries) {
|
|
218
|
+
for (const [pairKey, pairConfig] of pairEntries) {
|
|
146
219
|
const code = pairConfig.target;
|
|
147
220
|
const targetPath = getDocusaurusTargetPath(
|
|
148
221
|
sourcePath, sourceContentDir, code, config.localesDir, pluginName
|
|
@@ -156,7 +229,11 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
156
229
|
const manifestKey = `docusaurus:${dirName}/${relPath}:${code}`;
|
|
157
230
|
let action = 'new'; // 'new' | 'changed' | 're-translate'
|
|
158
231
|
|
|
159
|
-
if (
|
|
232
|
+
if (retranslate) {
|
|
233
|
+
// --retranslate: named by the operator — no lock skip, no
|
|
234
|
+
// adoption; the caller makes its text miss the TM.
|
|
235
|
+
if (fs.existsSync(targetPath)) action = 're-translate';
|
|
236
|
+
} else if (fs.existsSync(targetPath)) {
|
|
160
237
|
const storedHash = manifest[manifestKey];
|
|
161
238
|
if (storedHash) {
|
|
162
239
|
if (storedHash === sourceHash) {
|
|
@@ -190,7 +267,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
190
267
|
workItems.push({
|
|
191
268
|
sourcePath, parsed, fieldsToTranslate, pageTitle, sourceHash,
|
|
192
269
|
relPath, dirName,
|
|
193
|
-
pairConfig, code, targetPath, manifestKey, pluginName, action,
|
|
270
|
+
pairKey, pairConfig, code, targetPath, manifestKey, pluginName, action, retranslate,
|
|
194
271
|
});
|
|
195
272
|
}
|
|
196
273
|
}
|
|
@@ -240,7 +317,7 @@ function scanDocusaurusContentWork(contentSources, pairEntries, config, manifest
|
|
|
240
317
|
* Structured estimate, or null when estimation itself failed (callers
|
|
241
318
|
* with --max-cost must fail safe: unknown ≠ free)
|
|
242
319
|
*/
|
|
243
|
-
async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate) {
|
|
320
|
+
async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir, tm, contentWorkItems, lockManifest, noTranslate, lockState = null, redo = null) {
|
|
244
321
|
try {
|
|
245
322
|
const { estimateCost } = await import('./pairs.js');
|
|
246
323
|
const costEstimates = [];
|
|
@@ -251,6 +328,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
251
328
|
const sourceJSONFiles = discoverDocusaurusJSONFiles(sourceLocaleDir);
|
|
252
329
|
const missesByPair = new Map(); // pairKey → miss count
|
|
253
330
|
const hitsByPair = new Map(); // pairKey → TM hit count
|
|
331
|
+
const heldByPair = new Map(); // pairKey → keys held back (refused before; not sent)
|
|
254
332
|
|
|
255
333
|
for (const sourceFilePath of sourceJSONFiles) {
|
|
256
334
|
const relPath = path.relative(sourceLocaleDir, sourceFilePath);
|
|
@@ -283,26 +361,45 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
283
361
|
|
|
284
362
|
// Same confirmed-echo suppression as the Phase-1 sync below, so the
|
|
285
363
|
// estimate prices exactly the keys the run will actually queue.
|
|
286
|
-
|
|
364
|
+
// With a fallback, its cached values count too (lib/fallback.js).
|
|
365
|
+
const estTmKeys = tmKeysForPair(pairConfig);
|
|
366
|
+
const localeChanged = lockState
|
|
367
|
+
? [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, lockState.peek(code))])]
|
|
368
|
+
: changedKeys;
|
|
287
369
|
const diff = diffLocale(
|
|
288
370
|
sourceFlat, existingFlat, config.fallbackPrefix,
|
|
289
|
-
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys,
|
|
290
|
-
(key, sourceValue) =>
|
|
371
|
+
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
|
|
372
|
+
(key, sourceValue) => tmHoldsValue(tm, sourceValue, code, estTmKeys, sourceValue),
|
|
291
373
|
noTranslate.active ? noTranslate.matches : null
|
|
292
374
|
);
|
|
293
375
|
const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
|
|
294
376
|
if (stringKeys.length === 0) continue;
|
|
295
377
|
|
|
296
|
-
|
|
378
|
+
// Refused before (the lock's records): never sent to the method that
|
|
379
|
+
// refused them — held keys cost nothing; a fallback is not priced.
|
|
380
|
+
const notSent = new Set();
|
|
381
|
+
if (lockState && redo) {
|
|
382
|
+
const plan = planUIStrings({
|
|
383
|
+
diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState: lockState.peek(code), pairConfig, redo,
|
|
384
|
+
fallbackPrefix: config.fallbackPrefix,
|
|
385
|
+
});
|
|
386
|
+
for (const k of [...plan.held, ...plan.heldFromPrimary]) notSent.add(k);
|
|
387
|
+
}
|
|
388
|
+
let { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmMethodKey(pairConfig));
|
|
389
|
+
if (estTmKeys.length > 1) misses = misses.filter(k => lookupTM(tm, sourceFlat[k], code, estTmKeys[1]) === null);
|
|
390
|
+
const held = misses.filter(k => notSent.has(k)).length;
|
|
391
|
+
misses = misses.filter(k => !notSent.has(k));
|
|
297
392
|
missesByPair.set(pairKey, (missesByPair.get(pairKey) || 0) + misses.length);
|
|
298
|
-
hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length));
|
|
393
|
+
hitsByPair.set(pairKey, (hitsByPair.get(pairKey) || 0) + (stringKeys.length - misses.length - held));
|
|
394
|
+
heldByPair.set(pairKey, (heldByPair.get(pairKey) || 0) + held);
|
|
299
395
|
}
|
|
300
396
|
}
|
|
301
397
|
|
|
302
398
|
for (const [pairKey, pairConfig] of pairEntries) {
|
|
303
399
|
const keysToTranslate = missesByPair.get(pairKey) || 0;
|
|
304
400
|
const tmHits = hitsByPair.get(pairKey) || 0;
|
|
305
|
-
|
|
401
|
+
const heldCount = heldByPair.get(pairKey) || 0;
|
|
402
|
+
if (keysToTranslate === 0 && tmHits === 0 && heldCount === 0) continue;
|
|
306
403
|
|
|
307
404
|
if (keysToTranslate > 0) {
|
|
308
405
|
// eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
|
|
@@ -317,8 +414,15 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
317
414
|
method: pairConfig.method || 'llm',
|
|
318
415
|
keys: keysToTranslate,
|
|
319
416
|
tmHits,
|
|
417
|
+
...(heldCount > 0 && { held: heldCount }),
|
|
320
418
|
estimatedCost: estimate.estimatedCost,
|
|
321
419
|
source: estimate.source,
|
|
420
|
+
...(estimate.local && { local: true }),
|
|
421
|
+
...(estimate.note && { note: estimate.note }),
|
|
422
|
+
// The rate it was priced at, its source and date (cost-report.js rateLine).
|
|
423
|
+
...(estimate.rate && { rate: estimate.rate }),
|
|
424
|
+
// No price: the model it was looked up under, so the table can name it.
|
|
425
|
+
...(estimate.estimatedCost === null && estimate.model && { model: estimate.model }),
|
|
322
426
|
});
|
|
323
427
|
} else {
|
|
324
428
|
// Fully TM-covered: zero API calls → a KNOWN $0, even for
|
|
@@ -328,8 +432,9 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
328
432
|
method: pairConfig.method || 'llm',
|
|
329
433
|
keys: 0,
|
|
330
434
|
tmHits,
|
|
435
|
+
...(heldCount > 0 && { held: heldCount }),
|
|
331
436
|
estimatedCost: 0,
|
|
332
|
-
source: 'translation-memory',
|
|
437
|
+
source: tmHits > 0 ? 'translation-memory' : 'nothing-to-send',
|
|
333
438
|
});
|
|
334
439
|
}
|
|
335
440
|
}
|
|
@@ -342,72 +447,51 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
342
447
|
// block sources once (only needed for 'block' mode misses).
|
|
343
448
|
const blockSourcesCache = new Map(); // sourcePath → string[]
|
|
344
449
|
const charsByPair = new Map(); // pairConfig (by reference) → billed source chars
|
|
450
|
+
const roughCharsByPair = new Map(); // pairConfig → all source chars, as if the TM were empty
|
|
345
451
|
const pendingSourceFiles = new Set();
|
|
346
452
|
|
|
347
453
|
for (const item of contentWorkItems) {
|
|
348
454
|
pendingSourceFiles.add(item.sourcePath);
|
|
349
455
|
const code = item.code;
|
|
350
|
-
const
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
// free), then per-block in 'block' mode — the same ladder the
|
|
361
|
-
// sync runs below.
|
|
362
|
-
const body = item.parsed.body;
|
|
363
|
-
if (body.trim() && lookupTM(tm, body, code, tmKey) === null) {
|
|
364
|
-
const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
365
|
-
if (segMode === 'page') {
|
|
366
|
-
chars += body.length;
|
|
367
|
-
} else {
|
|
456
|
+
const segMode = item.pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
457
|
+
const { billedChars: chars, totalChars } = billableContentChars({
|
|
458
|
+
tm,
|
|
459
|
+
code,
|
|
460
|
+
tmKey: tmMethodKey(item.pairConfig),
|
|
461
|
+
fallbackTmKey: item.pairConfig.fallback ? tmMethodKey(item.pairConfig.fallback) : null,
|
|
462
|
+
fields: item.fieldsToTranslate,
|
|
463
|
+
body: item.parsed.body,
|
|
464
|
+
segMode,
|
|
465
|
+
blockSources: () => {
|
|
368
466
|
if (!blockSourcesCache.has(item.sourcePath)) {
|
|
369
|
-
|
|
370
|
-
blockSourcesCache.set(
|
|
371
|
-
item.sourcePath,
|
|
372
|
-
splitBlocks(protectedBody)
|
|
373
|
-
.filter(seg => seg.type === 'translatable')
|
|
374
|
-
.map(seg => restoreBlocks(seg.text, blocks))
|
|
375
|
-
);
|
|
376
|
-
}
|
|
377
|
-
for (const source of blockSourcesCache.get(item.sourcePath)) {
|
|
378
|
-
if (lookupTM(tm, source, code, tmKey) === null) chars += source.length;
|
|
467
|
+
blockSourcesCache.set(item.sourcePath, translatableBlockSources(item.parsed.body));
|
|
379
468
|
}
|
|
380
|
-
|
|
381
|
-
|
|
469
|
+
return blockSourcesCache.get(item.sourcePath);
|
|
470
|
+
},
|
|
471
|
+
holds: item.holds || null,
|
|
472
|
+
});
|
|
473
|
+
roughCharsByPair.set(item.pairConfig, (roughCharsByPair.get(item.pairConfig) || 0) + totalChars);
|
|
382
474
|
|
|
383
475
|
if (chars > 0) {
|
|
384
476
|
charsByPair.set(item.pairConfig, (charsByPair.get(item.pairConfig) || 0) + chars);
|
|
385
477
|
}
|
|
386
478
|
}
|
|
387
479
|
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
// Fully TM-covered pairs are a KNOWN $0 (zero API calls) — they
|
|
393
|
-
// must not consult estimateCost, whose unknown-pricing null would
|
|
394
|
-
// wrongly trip hasUnknownCosts and abort a free run under a cap.
|
|
395
|
-
if (!chars) continue;
|
|
396
|
-
const keyEquivalents = Math.ceil(chars / EST_CHARS_PER_KEY);
|
|
397
|
-
// eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
|
|
398
|
-
const estimate = await estimateCost(keyEquivalents, pairConfig);
|
|
399
|
-
if (estimate.estimatedCost !== null) {
|
|
400
|
-
contentCost += estimate.estimatedCost;
|
|
401
|
-
} else {
|
|
402
|
-
contentUnknown = true;
|
|
403
|
-
}
|
|
404
|
-
}
|
|
480
|
+
const billed = await priceContentChars(charsByPair, pairEntries, estimateCost);
|
|
481
|
+
const rough = await priceContentChars(roughCharsByPair, pairEntries, estimateCost);
|
|
482
|
+
const contentCost = billed.cost;
|
|
483
|
+
const contentUnknown = billed.unknown;
|
|
405
484
|
|
|
406
485
|
content = {
|
|
407
486
|
files: pendingSourceFiles.size,
|
|
408
487
|
pendingTranslations: contentWorkItems.length,
|
|
409
488
|
estimatedCost: contentUnknown ? null : contentCost,
|
|
489
|
+
costWithoutTM: rough.unknown ? null : rough.cost,
|
|
410
490
|
rough: true,
|
|
491
|
+
// Which pairs have no price, and why (printCostTable names them).
|
|
492
|
+
...(contentUnknown && { unpriced: billed.unpriced }),
|
|
493
|
+
// The rates the billed part was priced at.
|
|
494
|
+
...(billed.rates.length > 0 && { rates: billed.rates }),
|
|
411
495
|
};
|
|
412
496
|
if (contentUnknown) {
|
|
413
497
|
hasUnknownCosts = true;
|
|
@@ -418,16 +502,7 @@ async function printDocusaurusCostEstimate(pairEntries, config, sourceLocaleDir,
|
|
|
418
502
|
|
|
419
503
|
printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
|
|
420
504
|
|
|
421
|
-
return
|
|
422
|
-
currency: 'USD',
|
|
423
|
-
pairs: costEstimates,
|
|
424
|
-
keyCost: content && content.estimatedCost !== null
|
|
425
|
-
? totalEstimatedCost - content.estimatedCost
|
|
426
|
-
: totalEstimatedCost,
|
|
427
|
-
content,
|
|
428
|
-
totalEstimatedCost,
|
|
429
|
-
hasUnknownCosts,
|
|
430
|
-
};
|
|
505
|
+
return summarizeEstimate(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
|
|
431
506
|
} catch (costError) {
|
|
432
507
|
// Non-blocking without a cap — warn and continue. Callers enforcing
|
|
433
508
|
// --max-cost must treat the null return as unknown (abort).
|
|
@@ -508,9 +583,48 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
508
583
|
);
|
|
509
584
|
}
|
|
510
585
|
|
|
586
|
+
// Keys named for a redo (--redo keys: / --force-keys) must exist — the
|
|
587
|
+
// rule the key-value path applies (lib/named-keys.js). A name matching no
|
|
588
|
+
// UI string used to re-queue nothing in silence and exit 0. Checked before
|
|
589
|
+
// the preflight, the estimate or any spend: with no name matching nothing
|
|
590
|
+
// runs; with some, those are redone and the run then fails naming the rest.
|
|
591
|
+
// Docusaurus message ids are per file but named bare: a name re-queues
|
|
592
|
+
// that id in every JSON file that has it (as the diff below applies it).
|
|
593
|
+
const unmatchedNamed = applyNamedKeyRule({
|
|
594
|
+
cliArgs, config,
|
|
595
|
+
known: () => {
|
|
596
|
+
const ids = new Set();
|
|
597
|
+
for (const file of discoverDocusaurusJSONFiles(sourceLocaleDir)) {
|
|
598
|
+
for (const key of Object.keys(extractDocusaurusMessages(JSON.parse(fs.readFileSync(file, 'utf-8'))))) {
|
|
599
|
+
if (!isUnsafeKey(key)) ids.add(key);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
return ids;
|
|
603
|
+
},
|
|
604
|
+
});
|
|
605
|
+
|
|
511
606
|
// Thread dryRun/audit into cliArgs so the preflight check can skip
|
|
512
607
|
// when appropriate — same pattern as the main sync path in sync.js
|
|
513
|
-
const { apiKey, pairEntries } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
|
|
608
|
+
const { apiKey, pairEntries, preflightFailures = [] } = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit });
|
|
609
|
+
|
|
610
|
+
// Content needs what the pair's own method needs — a local model needs no
|
|
611
|
+
// OPENROUTER_API_KEY (the lane used to demand it for every pair). Same
|
|
612
|
+
// lazy, per-pair readiness check as the content lane (content-sync.js).
|
|
613
|
+
const readinessCache = new Map();
|
|
614
|
+
const requirePairReady = async (pairKey, pairConfig, code, what) => {
|
|
615
|
+
if (!readinessCache.has(pairKey)) {
|
|
616
|
+
const method = getMethod(pairConfig.method || 'llm', pairConfig);
|
|
617
|
+
readinessCache.set(pairKey, Promise.resolve(method.checkReadiness({ apiKey, cwd })));
|
|
618
|
+
}
|
|
619
|
+
const readiness = await readinessCache.get(pairKey);
|
|
620
|
+
if (!readiness.ready) {
|
|
621
|
+
throw new Error(
|
|
622
|
+
`Docusaurus ${what} for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
|
|
623
|
+
` ${readiness.reason}\n` +
|
|
624
|
+
missingKeyAdvice({ reasons: [readiness.reason], setupHelp: [' Set the key it names in .env.local (or your environment) to translate content.'] }).join('\n')
|
|
625
|
+
);
|
|
626
|
+
}
|
|
627
|
+
};
|
|
514
628
|
|
|
515
629
|
if (pairEntries.length === 0) {
|
|
516
630
|
output.info('No target languages configured. Add pairs to champollion.config.json.');
|
|
@@ -542,6 +656,13 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
542
656
|
} else if (tmInitialSize > 0) {
|
|
543
657
|
output.info(`[TM] ${tmInitialSize} cached entries loaded`);
|
|
544
658
|
}
|
|
659
|
+
// Model carry-over (lib/tm.js) is on unless --fresh-on-model-change.
|
|
660
|
+
const freshOnModelChange = !!cliArgs['fresh-on-model-change'];
|
|
661
|
+
if (freshOnModelChange) setModelCarryover(tm, false);
|
|
662
|
+
// Entries made before coaching was keyed, with today's coaching: still
|
|
663
|
+
// served (lib/tm.js adoptLegacyCoachingKeys).
|
|
664
|
+
if (!noTM) adoptLegacyCoachingKeys(tm, pairEntries.map(([, pc]) => pc));
|
|
665
|
+
if (!noTM) warnModelSwitchStrandedTM(tm, pairEntries.map(([, pc]) => pc), { fresh: freshOnModelChange });
|
|
545
666
|
|
|
546
667
|
// Phase-1 source-hash manifest (.champollion.lock) — read up front
|
|
547
668
|
// because the cost estimator mirrors Phase 1's changed-key detection.
|
|
@@ -551,8 +672,19 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
551
672
|
// manifest, EDITING an English UI string never re-translated it: the
|
|
552
673
|
// diff only sees missing keys and [EN] fallbacks, so a changed source
|
|
553
674
|
// value looked "fully synced" forever.
|
|
554
|
-
const
|
|
675
|
+
const lock = readLock(cwd);
|
|
676
|
+
const lockManifest = lock.source;
|
|
555
677
|
const updatedLockManifest = { ...lockManifest };
|
|
678
|
+
// The lock's per-locale record (lib/locale-state.js): which UI strings the
|
|
679
|
+
// quality gate refused, per (file × key × locale), for their current source
|
|
680
|
+
// text and the method key that produced the refused answer. A plain sync
|
|
681
|
+
// holds them back from that method — the key-value rule. Keyed like the
|
|
682
|
+
// manifest ("docusaurus:<relPath>:<id>").
|
|
683
|
+
const lockState = new LockState(lock.locales);
|
|
684
|
+
// What counts as an explicit redo (always sent, never held back): ids named
|
|
685
|
+
// by --redo keys: / --force-keys, every id under --redo all (--force), and
|
|
686
|
+
// anything under --fresh.
|
|
687
|
+
const redo = { named: new Set(config.forceKeys || []), bulk: !!config._forceAllKeys, fresh: noTM };
|
|
556
688
|
|
|
557
689
|
// ── Content discovery + pending-work scan ─────────────────────
|
|
558
690
|
// Done up front (not in Phase 2) because the cost estimate needs the
|
|
@@ -586,9 +718,29 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
586
718
|
if (fs.existsSync(contentLockPath)) {
|
|
587
719
|
try { docuContentManifest = JSON.parse(fs.readFileSync(contentLockPath, 'utf-8')); } catch { /* first run */ }
|
|
588
720
|
}
|
|
721
|
+
// --files / --retranslate (lib/file-scope.js). Resolved at the scan, so a
|
|
722
|
+
// pattern that matches nothing fails before the estimate or any spend.
|
|
723
|
+
const fileScope = compileFileScope(cliArgs);
|
|
589
724
|
const contentScan = scanDocusaurusContentWork(
|
|
590
|
-
contentSources, pairEntries, config, docuContentManifest, forceContent
|
|
725
|
+
contentSources, pairEntries, config, docuContentManifest, forceContent, fileScope
|
|
591
726
|
);
|
|
727
|
+
if (fileScope) fileScope.assertAllMatched();
|
|
728
|
+
// --retranslate files miss the TM for this run (estimate and sync alike),
|
|
729
|
+
// so they are priced and translated fresh; new results replace the cache.
|
|
730
|
+
for (const item of contentScan.workItems) {
|
|
731
|
+
if (!item.retranslate) continue;
|
|
732
|
+
bypassTMFor(tm, item.code, [
|
|
733
|
+
...Object.values(item.fieldsToTranslate), item.parsed.body, ...translatableBlockSources(item.parsed.body),
|
|
734
|
+
]);
|
|
735
|
+
}
|
|
736
|
+
// Blocks and front-matter fields the quality gate refused before
|
|
737
|
+
// (lib/content-refusals.js): not sent to the method that refused them again
|
|
738
|
+
// unless the page is named for a redo (--redo files: / --redo content /
|
|
739
|
+
// --retranslate / --fresh). The estimate and the sync read the same holds.
|
|
740
|
+
for (const item of contentScan.workItems) {
|
|
741
|
+
item.refusalsBefore = readRefusals(docuContentManifest, item.manifestKey);
|
|
742
|
+
item.holds = contentHolds(item.refusalsBefore, item.pairConfig, { redo: forceContent || item.retranslate || noTM });
|
|
743
|
+
}
|
|
592
744
|
|
|
593
745
|
// ── Pre-sync cost estimation + --max-cost gate ────────────────
|
|
594
746
|
// Mirrors sync.js: without a cap the estimate is informational and
|
|
@@ -598,26 +750,63 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
598
750
|
// exact thing a capped user needs to see.
|
|
599
751
|
const maxCost = parseMaxCost(cliArgs['max-cost']);
|
|
600
752
|
const costEstimate = await printDocusaurusCostEstimate(
|
|
601
|
-
pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate
|
|
753
|
+
pairEntries, config, sourceLocaleDir, tm, contentScan.workItems, lockManifest, noTranslate, lockState, redo
|
|
602
754
|
);
|
|
755
|
+
// Machine-readable estimate BEFORE the gate (see sync.js).
|
|
756
|
+
if (costEstimate) output.event('cost', costEstimate);
|
|
757
|
+
// A dry run never stops at the cap, but says what the real run would do.
|
|
758
|
+
let dryMaxCost = null;
|
|
603
759
|
if (maxCost !== null && !dryRun) {
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
760
|
+
const verdict = maxCostVerdict(maxCost, costEstimate);
|
|
761
|
+
if (verdict.wouldStop) return abortForMaxCost(maxCost, verdict.estimatedCost, verdict.reason);
|
|
762
|
+
} else if (maxCost !== null) {
|
|
763
|
+
dryMaxCost = reportDryRunMaxCost(maxCost, costEstimate, { stopsEarlier: preflightStopReason(preflightFailures) });
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
// Fallback methods are outside the estimate (they translate what the
|
|
767
|
+
// primary fails, unknown in advance). Under --max-cost each fallback batch
|
|
768
|
+
// must fit in what the estimate left of the cap (lib/fallback.js).
|
|
769
|
+
if (pairEntries.some(([, p]) => p.fallback)) {
|
|
770
|
+
output.info(
|
|
771
|
+
'Fallback methods are not in this estimate — they only translate what the primary method fails, '
|
|
772
|
+
+ 'which is not known in advance.'
|
|
773
|
+
+ (maxCost !== null ? ' Under --max-cost each fallback batch is priced before it runs and skipped if it would pass the cap.' : '')
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
const fallbackBudget = createFallbackBudget({
|
|
777
|
+
maxCost: dryRun ? null : maxCost,
|
|
778
|
+
committed: costEstimate?.knownEstimatedCost ?? 0,
|
|
779
|
+
cwd,
|
|
780
|
+
});
|
|
781
|
+
// Different inputs, same output (lib/validate.js SharedOutputIndex): one
|
|
782
|
+
// index per locale for this run's UI strings and Markdown pages — cached
|
|
783
|
+
// and fresh alike, as in the content lane (lib/content-sync.js).
|
|
784
|
+
const sharedOutputIndexes = new Map();
|
|
785
|
+
const sharedOutputsFor = (code) => {
|
|
786
|
+
if (!sharedOutputIndexes.has(code)) {
|
|
787
|
+
sharedOutputIndexes.set(code, new SharedOutputIndex({ protectedTerms: config.protectedTerms || [] }));
|
|
615
788
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
789
|
+
return sharedOutputIndexes.get(code);
|
|
790
|
+
};
|
|
791
|
+
// Per-pair tallies of what each fallback did: JSON keys, Markdown segments.
|
|
792
|
+
const keyFallbackTallies = new Map();
|
|
793
|
+
const contentFallbackTallies = new Map();
|
|
794
|
+
// A page some of whose text the fallback produced (its answer or its
|
|
795
|
+
// cache): named in the [FALLBACK] line.
|
|
796
|
+
const notePage = (pairKey, page) => {
|
|
797
|
+
const tally = contentFallbackTallies.get(pairKey);
|
|
798
|
+
if (tally && !tally.produced.includes(page)) tally.produced.push(page);
|
|
799
|
+
};
|
|
800
|
+
const tallyContent = (pairKey, report, page) => {
|
|
801
|
+
if (!contentFallbackTallies.has(pairKey)) return;
|
|
802
|
+
addToTally(contentFallbackTallies.get(pairKey), report);
|
|
803
|
+
if (report && (report.accepted > 0 || report.cached > 0)) notePage(pairKey, page);
|
|
804
|
+
};
|
|
805
|
+
if (!dryRun) {
|
|
806
|
+
for (const [pairKey, pc] of pairEntries) {
|
|
807
|
+
if (!pc.fallback) continue;
|
|
808
|
+
keyFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
|
|
809
|
+
contentFallbackTallies.set(pairKey, newFallbackReport(pc.fallback));
|
|
621
810
|
}
|
|
622
811
|
}
|
|
623
812
|
|
|
@@ -628,6 +817,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
628
817
|
|
|
629
818
|
let totalJSONKeys = 0;
|
|
630
819
|
let totalJSONCopied = 0;
|
|
820
|
+
let totalJSONFailed = 0;
|
|
821
|
+
// UI strings held back (refused before; not sent) and what the next sync
|
|
822
|
+
// does with each one this run could not translate (lib/locale-state.js).
|
|
823
|
+
let totalJSONHeld = 0;
|
|
824
|
+
let totalJSONWritten = 0;
|
|
825
|
+
const jsonFates = { retry: [], held: [] }; // { pair, key }
|
|
826
|
+
// Every lock key the source still has (refused records for others are dropped).
|
|
827
|
+
const liveLockKeys = new Set();
|
|
631
828
|
|
|
632
829
|
for (const sourceFilePath of sourceJSONFiles) {
|
|
633
830
|
const relPath = path.relative(sourceLocaleDir, sourceFilePath);
|
|
@@ -649,6 +846,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
649
846
|
// The manifest stores namespaced keys; detectChangedKeys expects the
|
|
650
847
|
// same key space as sourceFlat, so build a per-file un-namespaced view.
|
|
651
848
|
const nsPrefix = `docusaurus:${relPath}:`;
|
|
849
|
+
for (const key of Object.keys(sourceFlat)) liveLockKeys.add(nsPrefix + key);
|
|
652
850
|
const fileOldManifest = {};
|
|
653
851
|
for (const [nsKey, storedHash] of Object.entries(lockManifest)) {
|
|
654
852
|
if (nsKey.startsWith(nsPrefix)) {
|
|
@@ -664,10 +862,11 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
664
862
|
const pairResults = await pMap(pairEntries, async ([pairKey, pairConfig]) => {
|
|
665
863
|
const code = pairConfig.target;
|
|
666
864
|
const targetFilePath = path.join(config.localesDir, code, relPath);
|
|
865
|
+
const filename = `${code}/${relPath}`;
|
|
667
866
|
|
|
668
867
|
// Security: verify target path stays within i18n directory
|
|
669
868
|
if (!isPathContained(targetFilePath, config.localesDir)) {
|
|
670
|
-
output.error(`${
|
|
869
|
+
output.error(`${filename} — refusing to write outside i18n directory`);
|
|
671
870
|
// Nothing was attempted, but nothing succeeded either: keep every
|
|
672
871
|
// to-be-processed key out of the "succeeded" set so its manifest
|
|
673
872
|
// hash is not advanced (an unwritable locale must re-fire).
|
|
@@ -685,11 +884,15 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
685
884
|
// re-translate (they are otherwise invisible to the diff). Echo keys
|
|
686
885
|
// (target === source) the TM confirms as pipeline-produced are NOT
|
|
687
886
|
// requeued — see lib/diff.js isConfirmedEcho.
|
|
688
|
-
|
|
887
|
+
// A value the pair's fallback produced is cached under the fallback.
|
|
888
|
+
const tmKeys = tmKeysForPair(pairConfig);
|
|
889
|
+
const localeState = dryRun ? lockState.peek(code) : lockState.of(code);
|
|
890
|
+
// A refused string whose source was edited since: queued (the hold lifts).
|
|
891
|
+
const localeChanged = [...new Set([...changedKeys, ...editedSinceRefused(sourceFlat, existingFlat, nsPrefix, localeState)])];
|
|
689
892
|
const diff = diffLocale(
|
|
690
893
|
sourceFlat, existingFlat, config.fallbackPrefix,
|
|
691
|
-
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys,
|
|
692
|
-
(key, sourceValue) =>
|
|
894
|
+
config._forceAllKeys ? Object.keys(sourceFlat) : config.forceKeys, localeChanged,
|
|
895
|
+
(key, sourceValue) => tmHoldsValue(tm, sourceValue, code, tmKeys, sourceValue),
|
|
693
896
|
noTranslate.active ? noTranslate.matches : null
|
|
694
897
|
);
|
|
695
898
|
|
|
@@ -697,12 +900,38 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
697
900
|
return { keys: 0, failedKeys: [] };
|
|
698
901
|
}
|
|
699
902
|
|
|
903
|
+
// Refused before (the lock's records): held back from the method that
|
|
904
|
+
// refused them — the key-value rule (planUIStrings).
|
|
905
|
+
const plan = planUIStrings({
|
|
906
|
+
diff, sourceFlat, targetFlat: existingFlat, nsPrefix, localeState, pairConfig, redo,
|
|
907
|
+
fallbackPrefix: config.fallbackPrefix,
|
|
908
|
+
});
|
|
909
|
+
const heldSet = new Set(plan.held);
|
|
910
|
+
const lk = (k) => nsPrefix + k;
|
|
911
|
+
|
|
700
912
|
let keysProcessed = 0;
|
|
701
913
|
let keysCopied = 0;
|
|
914
|
+
let keysWritten = 0; // translations written (the model, its fallback, the cache)
|
|
702
915
|
const failedKeys = [];
|
|
916
|
+
const heldKeys = [];
|
|
917
|
+
const fates = { retry: [], held: [] };
|
|
703
918
|
|
|
704
919
|
if (diff.toProcess.length > 0 || diff.noTranslate.length > 0) {
|
|
705
|
-
|
|
920
|
+
// The per-file line counts what goes to the pipeline; held keys are
|
|
921
|
+
// said on their own line below.
|
|
922
|
+
const shown = heldSet.size === 0 ? diff : Object.fromEntries(Object.entries(diff).map(([k, v]) => (
|
|
923
|
+
[k, Array.isArray(v) && k !== 'noTranslate' && k !== 'extra' ? v.filter(x => !heldSet.has(x)) : v])));
|
|
924
|
+
const label = diffLabel(shown);
|
|
925
|
+
output.info(`${filename} — ${plan.held.length > 0 ? `${label === 'fully synced' ? '' : `${label} + `}${plan.held.length} held back` : label}`);
|
|
926
|
+
if (plan.held.length > 0) {
|
|
927
|
+
output.warn(describeHeldKeys({ filename, keys: plan.held, pairConfig, command: redoCommand(plan.held, { pair: pairKey }) }));
|
|
928
|
+
}
|
|
929
|
+
if (plan.heldFromPrimary.length > 0) {
|
|
930
|
+
output.info(describeFallbackOnlyKeys({ filename, keys: plan.heldFromPrimary, pairConfig }));
|
|
931
|
+
}
|
|
932
|
+
// Named for a redo but served from the cache (the model is not asked
|
|
933
|
+
// without --fresh): said, with the command that asks the model again.
|
|
934
|
+
const namedQueued = redo.fresh ? [] : diff.toProcess.filter(k => redo.named.has(k) && typeof sourceFlat[k] === 'string');
|
|
706
935
|
|
|
707
936
|
if (!dryRun) {
|
|
708
937
|
// Merged output starts from what's on disk, then takes the verbatim
|
|
@@ -712,6 +941,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
712
941
|
const mergedFlat = { ...existingFlat };
|
|
713
942
|
for (const key of diff.noTranslate) {
|
|
714
943
|
mergedFlat[key] = sourceFlat[key];
|
|
944
|
+
delete localeState.refused[lk(key)];
|
|
715
945
|
}
|
|
716
946
|
keysCopied = diff.noTranslate.length;
|
|
717
947
|
const writeMerged = () => {
|
|
@@ -719,32 +949,77 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
719
949
|
fs.mkdirSync(path.dirname(targetFilePath), { recursive: true });
|
|
720
950
|
fs.writeFileSync(targetFilePath, JSON.stringify(docuOutput, null, 2) + '\n', 'utf-8');
|
|
721
951
|
};
|
|
952
|
+
// What the gate refused this run is remembered per key (held back
|
|
953
|
+
// by the next plain sync); what got no usable answer is asked again.
|
|
954
|
+
const settleUnfilled = (keys, refusedBy = {}) => {
|
|
955
|
+
for (const k of keys) {
|
|
956
|
+
recordRefusal(localeState, lk(k), sourceFlat[k], refusedBy[k]);
|
|
957
|
+
fates[(refusedBy[k] || []).length > 0 ? 'held' : 'retry'].push(k);
|
|
958
|
+
}
|
|
959
|
+
};
|
|
722
960
|
|
|
723
961
|
if (keysCopied > 0) {
|
|
724
962
|
const sample = diff.noTranslate.slice(0, 3)
|
|
725
963
|
.map(k => `${k} (${noTranslate.reason(k, sourceFlat[k])})`)
|
|
726
964
|
.join(', ');
|
|
727
965
|
const more = keysCopied > 3 ? `, +${keysCopied - 3} more` : '';
|
|
728
|
-
output.info(`${
|
|
966
|
+
output.info(`${filename} — copied ${keysCopied} no-translate key(s) verbatim: ${sample}${more}`);
|
|
729
967
|
}
|
|
730
968
|
|
|
731
969
|
let translated = null;
|
|
970
|
+
let result = null;
|
|
732
971
|
const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
|
|
733
972
|
|
|
734
973
|
if (stringKeys.length > 0) {
|
|
735
|
-
// Shared pipeline: TM partition → API call →
|
|
736
|
-
|
|
737
|
-
|
|
974
|
+
// Shared pipeline: TM partition → API call → quality gate → TM
|
|
975
|
+
// store, then the pair's fallback (if any) for what that left
|
|
976
|
+
// untranslated (lib/translate-pair.js translateWithFallback).
|
|
977
|
+
// Keys held back: the cache is still read, the method not asked.
|
|
978
|
+
result = await translateWithFallback(stringKeys, sourceFlat, pairConfig, pairKey, {
|
|
979
|
+
apiKey, tm, targetCode: code, descriptions, budget: fallbackBudget, cwd,
|
|
980
|
+
// One per locale, as on the standard path: a memorized sentence
|
|
981
|
+
// repeated across UI strings is refused (validate.js).
|
|
982
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
983
|
+
noSendPrimary: new Set([...plan.held, ...plan.heldFromPrimary]),
|
|
984
|
+
noSendFallback: heldSet,
|
|
738
985
|
});
|
|
739
986
|
translated = result.translated;
|
|
987
|
+
if (keyFallbackTallies.has(pairKey)) addToTally(keyFallbackTallies.get(pairKey), result.fallback);
|
|
988
|
+
const heldHere = new Set(result.heldKeys || []);
|
|
989
|
+
heldKeys.push(...stringKeys.filter(k => heldHere.has(k) && !(translated && k in translated)));
|
|
740
990
|
|
|
741
991
|
if (translated) {
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
992
|
+
const notDone = stringKeys.filter(k => !(k in translated));
|
|
993
|
+
const refusedHere = notDone.filter(k => (result.refusedBy?.[k] || []).length > 0);
|
|
994
|
+
const heldNow = notDone.filter(k => heldHere.has(k));
|
|
995
|
+
const noAnswer = notDone.length - refusedHere.length - heldNow.length;
|
|
996
|
+
output.progressDone(filename, notDone.length === 0 ? '[OK]' : `[WARN] ${notDone.length} of ${stringKeys.length} key(s) not translated (${[
|
|
997
|
+
refusedHere.length > 0 && `${refusedHere.length} refused by the quality gate`,
|
|
998
|
+
heldNow.length > 0 && `${heldNow.length} held back`,
|
|
999
|
+
noAnswer > 0 && `${noAnswer} not returned by the method`,
|
|
1000
|
+
].filter(Boolean).join(', ')})`);
|
|
1001
|
+
const answered = new Set(result.answeredKeys || []);
|
|
1002
|
+
const fromCache = namedQueued.filter(k => k in translated && !answered.has(k));
|
|
1003
|
+
await reportNamedFromCache({
|
|
1004
|
+
filename, keys: fromCache, served: translated, onDisk: existingFlat, pairConfig, cwd,
|
|
1005
|
+
command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
|
|
1006
|
+
});
|
|
1007
|
+
} else if (heldKeys.length === stringKeys.length) {
|
|
1008
|
+
// Everything queued was held back: nothing was asked, nothing failed anew.
|
|
1009
|
+
output.progressDone(filename, `[WARN] ${heldKeys.length} key(s) held back, not translated`);
|
|
1010
|
+
for (const k of heldKeys) fates.held.push(k);
|
|
1011
|
+
if (keysCopied > 0) writeMerged();
|
|
1012
|
+
return { keys: 0, copied: keysCopied, failedKeys: [], heldKeys, fates };
|
|
1013
|
+
} else if (result.apiReturnedNull || result.failures.length > 0) {
|
|
1014
|
+
output.progressDone(filename, result.apiReturnedNull ? '[ERR] translation failed' : '[ERR] all failed quality gate');
|
|
1015
|
+
output.error(result.apiReturnedNull
|
|
1016
|
+
? `${filename}: Translation failed. Check API key and method configuration.`
|
|
1017
|
+
: `${filename}: all translations were rejected by the quality gate${pairConfig.fallback ? ` (and by its fallback, ${pairConfig.fallback.method})` : ''} — see the gate failures above.`);
|
|
746
1018
|
if (keysCopied > 0) writeMerged();
|
|
747
|
-
|
|
1019
|
+
const failedNow = stringKeys.filter(k => !heldKeys.includes(k));
|
|
1020
|
+
settleUnfilled(failedNow, result.refusedBy || {});
|
|
1021
|
+
for (const k of heldKeys) fates.held.push(k);
|
|
1022
|
+
return { keys: 0, copied: keysCopied, failedKeys: failedNow, heldKeys, fates };
|
|
748
1023
|
}
|
|
749
1024
|
}
|
|
750
1025
|
|
|
@@ -753,6 +1028,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
753
1028
|
// for it, fallbacks first, and a value with unmappable letters
|
|
754
1029
|
// stays whole in the working script (warned, not failed).
|
|
755
1030
|
const scriptConverterKey = pairConfig.scriptResolution?.converterKey || null;
|
|
1031
|
+
const heldNow = new Set(heldKeys);
|
|
756
1032
|
for (const key of diff.toProcess) {
|
|
757
1033
|
if (translated && key in translated) {
|
|
758
1034
|
let value = translated[key];
|
|
@@ -763,50 +1039,87 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
763
1039
|
value = converted;
|
|
764
1040
|
} else {
|
|
765
1041
|
output.warn(
|
|
766
|
-
`${
|
|
1042
|
+
`${filename}: key "${key}" kept in working script — `
|
|
767
1043
|
+ `unmapped letter(s): ${unmapped.join(', ')} (see "scriptFallback")`
|
|
768
1044
|
);
|
|
769
1045
|
}
|
|
770
1046
|
}
|
|
771
1047
|
mergedFlat[key] = value;
|
|
1048
|
+
keysWritten++;
|
|
1049
|
+
// Filled (by the model, its fallback or the cache): no longer refused.
|
|
1050
|
+
delete localeState.refused[lk(key)];
|
|
1051
|
+
} else if (heldNow.has(key)) {
|
|
1052
|
+
// Held back: said above; not sent, not counted as failed.
|
|
1053
|
+
fates.held.push(key);
|
|
772
1054
|
} else if (typeof sourceFlat[key] === 'string') {
|
|
773
|
-
|
|
1055
|
+
const fate = (result?.refusedBy?.[key] || []).length > 0 ? 'held' : 'retry';
|
|
1056
|
+
output.warn(`${filename}: key "${key}" ${keyFateNote(fate, pairConfig)}`);
|
|
774
1057
|
failedKeys.push(key);
|
|
1058
|
+
settleUnfilled([key], result?.refusedBy || {});
|
|
775
1059
|
} else {
|
|
776
1060
|
mergedFlat[key] = sourceFlat[key];
|
|
777
1061
|
}
|
|
778
1062
|
}
|
|
779
1063
|
|
|
780
|
-
keysProcessed = diff.toProcess.length;
|
|
1064
|
+
keysProcessed = diff.toProcess.length - heldKeys.length;
|
|
781
1065
|
|
|
782
1066
|
// Inject back into Docusaurus format and write
|
|
783
1067
|
writeMerged();
|
|
784
1068
|
} else {
|
|
785
|
-
|
|
1069
|
+
// Held keys would not be sent: not counted.
|
|
1070
|
+
keysProcessed = diff.toProcess.length - plan.held.length;
|
|
786
1071
|
keysCopied = diff.noTranslate.length;
|
|
1072
|
+
if (namedQueued.length > 0) {
|
|
1073
|
+
// What the real run would serve from the cache (the pair's entry,
|
|
1074
|
+
// then its fallback's — the same partition the estimate makes).
|
|
1075
|
+
const served = {};
|
|
1076
|
+
for (const k of namedQueued.filter(x => !heldSet.has(x))) {
|
|
1077
|
+
for (const mk of tmKeys) {
|
|
1078
|
+
const v = peekTM(tm, sourceFlat[k], code, mk);
|
|
1079
|
+
if (v !== null) { served[k] = v; break; }
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
const fromCache = Object.keys(served);
|
|
1083
|
+
await reportNamedFromCache({
|
|
1084
|
+
filename, keys: fromCache, served, onDisk: existingFlat, pairConfig, cwd, dryRun: true,
|
|
1085
|
+
command: redoCommand(fromCache, { pair: pairKey, fresh: true }),
|
|
1086
|
+
});
|
|
1087
|
+
}
|
|
1088
|
+
heldKeys.push(...plan.held);
|
|
787
1089
|
}
|
|
788
1090
|
}
|
|
789
1091
|
|
|
790
1092
|
if (diff.extra.length > 0) {
|
|
791
|
-
output.warn(`${
|
|
1093
|
+
output.warn(`${filename} — ${diff.extra.length} extra key(s)`);
|
|
792
1094
|
}
|
|
793
1095
|
|
|
794
|
-
return { keys: keysProcessed, copied: keysCopied, failedKeys };
|
|
1096
|
+
return { keys: keysProcessed, written: keysWritten, copied: keysCopied, failedKeys, heldKeys, fates };
|
|
795
1097
|
}, { concurrency: jsonConcurrency });
|
|
796
1098
|
|
|
797
1099
|
// Aggregate results for this JSON file
|
|
798
1100
|
const fileFailedKeys = new Set();
|
|
799
|
-
|
|
1101
|
+
// Held keys keep their manifest hash too (an edited source must still
|
|
1102
|
+
// re-fire), but they are not failures.
|
|
1103
|
+
const fileUnfilledKeys = new Set();
|
|
1104
|
+
pairResults.forEach((r, i) => {
|
|
800
1105
|
totalJSONKeys += r.keys;
|
|
1106
|
+
totalJSONWritten += r.written || 0;
|
|
801
1107
|
totalJSONCopied += r.copied || 0;
|
|
802
|
-
|
|
803
|
-
|
|
1108
|
+
totalJSONHeld += (r.heldKeys || []).length;
|
|
1109
|
+
for (const k of r.failedKeys || []) { fileFailedKeys.add(k); fileUnfilledKeys.add(k); }
|
|
1110
|
+
for (const k of r.heldKeys || []) fileUnfilledKeys.add(k);
|
|
1111
|
+
const pair = pairEntries[i][0];
|
|
1112
|
+
for (const fate of ['retry', 'held']) {
|
|
1113
|
+
for (const key of r.fates?.[fate] || []) jsonFates[fate].push({ pair, key });
|
|
1114
|
+
}
|
|
1115
|
+
});
|
|
1116
|
+
totalJSONFailed += fileFailedKeys.size;
|
|
804
1117
|
|
|
805
1118
|
// Update the manifest for this file: record the current hash ONLY for
|
|
806
1119
|
// keys that succeeded in every locale that attempted them. A failed
|
|
807
|
-
// key keeps its OLD hash (or none) so it is detected as
|
|
808
|
-
// RE-FIRES on the next sync — advancing the hash for a failed
|
|
809
|
-
// would silently mark the stale translation as current forever.
|
|
1120
|
+
// (or held) key keeps its OLD hash (or none) so it is detected as
|
|
1121
|
+
// changed and RE-FIRES on the next sync — advancing the hash for a failed
|
|
1122
|
+
// key would silently mark the stale translation as current forever.
|
|
810
1123
|
if (!dryRun) {
|
|
811
1124
|
for (const nsKey of Object.keys(updatedLockManifest)) {
|
|
812
1125
|
// Own-property check: `in` walks the prototype chain, so a source
|
|
@@ -818,7 +1131,7 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
818
1131
|
}
|
|
819
1132
|
for (const [key, value] of Object.entries(sourceFlat)) {
|
|
820
1133
|
const nsKey = nsPrefix + key;
|
|
821
|
-
if (
|
|
1134
|
+
if (fileUnfilledKeys.has(key)) {
|
|
822
1135
|
// Restore/keep the pre-sync state for failed keys.
|
|
823
1136
|
if (Object.prototype.hasOwnProperty.call(lockManifest, nsKey)) {
|
|
824
1137
|
updatedLockManifest[nsKey] = lockManifest[nsKey];
|
|
@@ -832,25 +1145,68 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
832
1145
|
}
|
|
833
1146
|
}
|
|
834
1147
|
|
|
835
|
-
// Persist the Phase 1 source-hash manifest
|
|
836
|
-
// must not mark changed keys as resolved).
|
|
1148
|
+
// Persist the Phase 1 source-hash manifest and the refusal records (skip
|
|
1149
|
+
// in dry-run — a preview must not mark changed keys as resolved). Records
|
|
1150
|
+
// of UI strings the source no longer has are dropped for the locales this
|
|
1151
|
+
// run processed.
|
|
837
1152
|
if (!dryRun && sourceJSONFiles.length > 0) {
|
|
838
|
-
|
|
1153
|
+
for (const [, pc] of pairEntries) {
|
|
1154
|
+
const refused = lockState.of(pc.target).refused;
|
|
1155
|
+
for (const k of Object.keys(refused)) {
|
|
1156
|
+
if (k.startsWith('docusaurus:') && !liveLockKeys.has(k)) delete refused[k];
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
writeManifest(cwd, updatedLockManifest, lockState.toJSON());
|
|
1160
|
+
}
|
|
1161
|
+
for (const [pairKey, tally] of keyFallbackTallies) {
|
|
1162
|
+
printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
|
|
1163
|
+
warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally);
|
|
839
1164
|
}
|
|
840
1165
|
|
|
841
1166
|
const copiedNote = totalJSONCopied > 0
|
|
842
1167
|
? ` (+${totalJSONCopied} copied verbatim, no-translate)`
|
|
843
1168
|
: '';
|
|
844
|
-
|
|
1169
|
+
const heldNote = totalJSONHeld > 0
|
|
1170
|
+
? `; ${totalJSONHeld} held back (refused before; not sent, not billed)`
|
|
1171
|
+
: '';
|
|
1172
|
+
const failedNote = totalJSONFailed > 0 ? `; ${totalJSONFailed} failed (listed above)` : '';
|
|
1173
|
+
// A real run counts what it wrote; a dry run what it would send.
|
|
1174
|
+
const shownKeys = dryRun ? totalJSONKeys : totalJSONWritten;
|
|
1175
|
+
if (shownKeys > 0) {
|
|
845
1176
|
const action = dryRun ? 'Would process' : 'Synced';
|
|
846
|
-
output.ok(`${action} ${
|
|
1177
|
+
output.ok(`${action} ${shownKeys} JSON key(s)${copiedNote}${failedNote}${heldNote}`);
|
|
1178
|
+
} else if (totalJSONHeld > 0 || totalJSONFailed > 0) {
|
|
1179
|
+
output.warn(`No JSON key ${dryRun ? 'would be' : 'was'} translated${copiedNote}${failedNote}${heldNote}`);
|
|
847
1180
|
} else if (totalJSONCopied > 0) {
|
|
848
1181
|
output.ok(`All JSON files fully synced${copiedNote}`);
|
|
849
1182
|
} else {
|
|
850
1183
|
output.ok('All JSON files fully synced');
|
|
851
1184
|
}
|
|
1185
|
+
// What the next sync does with the UI strings this run could not translate
|
|
1186
|
+
// — said per outcome, in the key-value lane's words.
|
|
1187
|
+
if (!dryRun && (jsonFates.retry.length > 0 || jsonFates.held.length > 0)) {
|
|
1188
|
+
if (jsonFates.retry.length > 0) {
|
|
1189
|
+
output.warn(` ${jsonFates.retry.length} UI string(s) got no usable answer (missing from the response, or the method failed) — the next sync asks for them again.`);
|
|
1190
|
+
}
|
|
1191
|
+
// Ids are named bare (a name re-queues that id in every JSON file that has it).
|
|
1192
|
+
const heldNext = [];
|
|
1193
|
+
const seen = new Set();
|
|
1194
|
+
for (const { pair, key } of jsonFates.held) {
|
|
1195
|
+
if (seen.has(`${pair}\u0000${key}`)) continue;
|
|
1196
|
+
seen.add(`${pair}\u0000${key}`);
|
|
1197
|
+
heldNext.push({ pair, key });
|
|
1198
|
+
}
|
|
1199
|
+
for (const line of describeHeldNext(heldNext, (keys, pair) => redoCommand(keys, { pair }))) output.warn(line);
|
|
1200
|
+
}
|
|
852
1201
|
|
|
853
1202
|
// ── Phase 2: Markdown content (docs + blog) ───────────────────
|
|
1203
|
+
let contentFailures = 0; // read after the TM save, outside the content block
|
|
1204
|
+
let contentTranslated = 0;
|
|
1205
|
+
let contentFailedItems = [];
|
|
1206
|
+
// Blocks/fields refused before and held back this run, and refused this run.
|
|
1207
|
+
let contentHeldBack = 0;
|
|
1208
|
+
const contentHeldBackItems = [];
|
|
1209
|
+
let contentRefused = 0;
|
|
854
1210
|
|
|
855
1211
|
if (contentSources.length === 0) {
|
|
856
1212
|
output.info('No docs/ or blog/ directories found — skipping content sync.');
|
|
@@ -872,7 +1228,14 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
872
1228
|
const concurrency = config.contentConcurrency || 48;
|
|
873
1229
|
|
|
874
1230
|
const totalWork = workItems.length;
|
|
875
|
-
|
|
1231
|
+
// Refusals per translation (lib/content-refusals.js), applied to the lock
|
|
1232
|
+
// with each item's outcome.
|
|
1233
|
+
const refusalStates = new Map();
|
|
1234
|
+
// Every failed (file × locale), with what it was left as on disk — the
|
|
1235
|
+
// end-of-run list an operator needs to tell "failed, previous
|
|
1236
|
+
// translation kept" from "failed, will retry" (dogfood 2026-08-28,
|
|
1237
|
+
// finding 5).
|
|
1238
|
+
const failedItems = [];
|
|
876
1239
|
// Warn at most once per source file about front matter fields we can't
|
|
877
1240
|
// translate (arrays / nested blocks like `related:`). The check + add are
|
|
878
1241
|
// synchronous (no await between), so this is race-free under pMap.
|
|
@@ -905,23 +1268,67 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
905
1268
|
await pMap(workItems, async (item) => {
|
|
906
1269
|
const {
|
|
907
1270
|
parsed, fieldsToTranslate, pageTitle, sourceHash,
|
|
908
|
-
relPath, dirName, pairConfig, code,
|
|
1271
|
+
relPath, dirName, pairKey, pairConfig, code,
|
|
909
1272
|
targetPath, manifestKey, action,
|
|
910
1273
|
} = item;
|
|
911
1274
|
|
|
1275
|
+
let itemFailed = false;
|
|
1276
|
+
// This translation's refusals, recorded with its outcome (held,
|
|
1277
|
+
// failed or written) — in the incremental lock writes too.
|
|
1278
|
+
const settleRefusals = () => {
|
|
1279
|
+
const st = refusalStates.get(manifestKey);
|
|
1280
|
+
if (!st || dryRun) return;
|
|
1281
|
+
storeRefusals(updatedDocuManifest, manifestKey, nextRefusals(st.prior, st));
|
|
1282
|
+
manifestDirty = true;
|
|
1283
|
+
};
|
|
912
1284
|
try {
|
|
913
1285
|
if (action === 'changed') {
|
|
914
1286
|
totalContentRetranslated++;
|
|
915
1287
|
}
|
|
916
1288
|
|
|
917
1289
|
if (dryRun) {
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1290
|
+
// What the quality gate refused before is held back by the real
|
|
1291
|
+
// run (lib/content-refusals.js) — said here.
|
|
1292
|
+
const preview = previewHeld({
|
|
1293
|
+
tm, code, pairConfig, fields: parsed.hasFrontMatter ? fieldsToTranslate : {}, body: parsed.body,
|
|
1294
|
+
segMode: pairConfig.contentSegmentation || config.contentSegmentation || 'block', holds: item.holds,
|
|
1295
|
+
});
|
|
1296
|
+
const held = preview.names.length > 0
|
|
1297
|
+
? ` — ${preview.pageHeld ? 'would hold the page back:' : 'would hold back'} ${preview.names.join(', ')} (refused before; not sent)`
|
|
1298
|
+
: '';
|
|
1299
|
+
output.raw(` [DRY] ${dirName}/${relPath} → ${code}${held}`);
|
|
1300
|
+
output.event('file', {
|
|
1301
|
+
lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: preview.pageHeld ? 'would-hold' : 'would-translate', action,
|
|
1302
|
+
...(preview.names.length > 0 && { held: preview.names.length }),
|
|
1303
|
+
});
|
|
1304
|
+
if (!preview.pageHeld) totalContent++;
|
|
921
1305
|
return;
|
|
922
1306
|
}
|
|
923
1307
|
|
|
924
1308
|
const { rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parsed;
|
|
1309
|
+
const segModeHere = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
1310
|
+
const label = `${dirName}/${relPath}`;
|
|
1311
|
+
const refusal = {
|
|
1312
|
+
file: label, locale: code, pageHeld: false,
|
|
1313
|
+
prior: item.refusalsBefore || {},
|
|
1314
|
+
refused: [], filled: new Set(), held: [],
|
|
1315
|
+
fields: hasFrontMatter ? fieldsToTranslate : {},
|
|
1316
|
+
blockSources: segModeHere === 'block' && body.trim() ? translatableBlockSources(body) : [],
|
|
1317
|
+
pageSource: segModeHere === 'page' && body.trim() ? body : null,
|
|
1318
|
+
};
|
|
1319
|
+
refusalStates.set(manifestKey, refusal);
|
|
1320
|
+
const { holds } = item;
|
|
1321
|
+
// A page translated whole that the gate refused before (and that
|
|
1322
|
+
// the cache does not hold): held before anything of it is sent —
|
|
1323
|
+
// its front matter included (a pure look; the body read below is
|
|
1324
|
+
// the one that serves).
|
|
1325
|
+
if (refusal.pageSource && holds.page(body) === 'held'
|
|
1326
|
+
&& ![tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])]
|
|
1327
|
+
.some(k => peekTM(tm, body, code, k) !== null)) {
|
|
1328
|
+
const err = new Error('held');
|
|
1329
|
+
err.heldNames = [PAGE_NAME];
|
|
1330
|
+
throw err;
|
|
1331
|
+
}
|
|
925
1332
|
|
|
926
1333
|
// Never silently drop translatable-looking nested/array front matter
|
|
927
1334
|
// (e.g. `related:` lists). The flat parser can't reach them — surface
|
|
@@ -942,6 +1349,25 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
942
1349
|
// not re-serve. Mirrors content-sync.js.
|
|
943
1350
|
const tmKey = tmMethodKey(pairConfig);
|
|
944
1351
|
|
|
1352
|
+
// This page's cached content through the repeat check, in one batch
|
|
1353
|
+
// and before any of it is served (lib/fallback.js): a text cached
|
|
1354
|
+
// for several different source strings is evicted, so the lookups
|
|
1355
|
+
// below miss it and it is translated again, checked the same way.
|
|
1356
|
+
const docLabel = `content:${dirName}/${relPath}`;
|
|
1357
|
+
const repeated = refuseRepeatedCachedPage({
|
|
1358
|
+
label: docLabel,
|
|
1359
|
+
fields: hasFrontMatter ? fieldsToTranslate : {},
|
|
1360
|
+
body,
|
|
1361
|
+
tm, code, pairConfig,
|
|
1362
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
1363
|
+
});
|
|
1364
|
+
if (repeated.length > 0) {
|
|
1365
|
+
output.warn(
|
|
1366
|
+
`${dirName}/${relPath} → ${code}: cached ${repeated.join(', ')} held the same text as other, different source strings ` +
|
|
1367
|
+
'(a memorized sentence, not a translation) — removed from the cache and translated again.'
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
|
|
945
1371
|
// Translate front matter fields — TM first, API only for misses.
|
|
946
1372
|
// Each field is cached on its own source text (exactly like a
|
|
947
1373
|
// key-value sync key): a title edit re-pays only the title.
|
|
@@ -954,61 +1380,93 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
954
1380
|
// gateless pipeline (hollowed titles were cached here) must not
|
|
955
1381
|
// outlive the gate. Failing hits are evicted and re-billed.
|
|
956
1382
|
for (const [field, cachedValue] of Object.entries(fmHits)) {
|
|
957
|
-
if (
|
|
1383
|
+
if (contentGateFault(fieldsToTranslate[field], cachedValue, pairConfig)) {
|
|
958
1384
|
evictTM(tm, fieldsToTranslate[field], code, tmKey);
|
|
959
1385
|
delete fmHits[field];
|
|
960
1386
|
fmMisses.push(field);
|
|
961
1387
|
}
|
|
962
1388
|
}
|
|
963
1389
|
Object.assign(translatedFields, fmHits);
|
|
1390
|
+
// Then the fallback's cache (lib/fallback.js ladder).
|
|
1391
|
+
const fbCache = serveFieldsFromFallbackCache(tm, fieldsToTranslate, fmMisses, code, pairConfig);
|
|
1392
|
+
Object.assign(translatedFields, fbCache.hits);
|
|
1393
|
+
if (Object.keys(fbCache.hits).length > 0) notePage(pairKey, label);
|
|
1394
|
+
const fieldsToSend = fbCache.misses;
|
|
1395
|
+
for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
|
|
1396
|
+
// Refused before: a held field fails its page as a refused one
|
|
1397
|
+
// does — nothing of it is sent, nothing written, lock not advanced.
|
|
1398
|
+
const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
|
|
1399
|
+
const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
|
|
1400
|
+
if (heldFields.length > 0) {
|
|
1401
|
+
const err = new Error('held');
|
|
1402
|
+
err.heldNames = heldFields.map(f => `front matter "${f}"`);
|
|
1403
|
+
throw err;
|
|
1404
|
+
}
|
|
964
1405
|
|
|
965
|
-
if (
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
1406
|
+
if (fieldsToSend.length > 0) {
|
|
1407
|
+
await requirePairReady(pairKey, pairConfig, code, 'content sync');
|
|
1408
|
+
// The pair's method, then its fallback (if any) for every field
|
|
1409
|
+
// it returned nothing for or hollowed. Content-preservation
|
|
1410
|
+
// gate on every value — Phase 2 front matter once reached disk
|
|
1411
|
+
// AND the TM with no validation at all, so a hollowed page
|
|
1412
|
+
// title was written silently and then cached forever. Nothing
|
|
1413
|
+
// is cached until no field is left failing; a failing field
|
|
1414
|
+
// skips the file and leaves its lock entry alone, so the next
|
|
1415
|
+
// sync retries it.
|
|
1416
|
+
const fm = await translateFieldsWithFallback({
|
|
1417
|
+
fields: fieldsToTranslate,
|
|
1418
|
+
misses: fieldsToSend,
|
|
1419
|
+
pairConfig,
|
|
1420
|
+
budget: fallbackBudget,
|
|
1421
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
1422
|
+
label: `${docLabel} front matter`,
|
|
1423
|
+
translate: (keys, cfg) => translateBatch(
|
|
1424
|
+
keys, fieldsToTranslate, cfg,
|
|
1425
|
+
{ apiKey, model: cfg.model, batchSize: cfg.batchSize || 30 },
|
|
1426
|
+
),
|
|
1427
|
+
fallbackOnly: fallbackOnlyFields,
|
|
1428
|
+
});
|
|
1429
|
+
tallyContent(pairKey, fm.report, label);
|
|
1430
|
+
// Refused fields are remembered, whatever happens to the page.
|
|
1431
|
+
for (const [field, methods] of Object.entries(fm.refusedBy || {})) {
|
|
1432
|
+
refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
|
|
1433
|
+
}
|
|
1434
|
+
if (fm.hollowed.length > 0) {
|
|
1435
|
+
const h = fm.hollowed[0];
|
|
1436
|
+
const err = new Error(
|
|
1437
|
+
`Docusaurus content sync for ${code}: front matter "${h.field}" — ${h.reason}.\n` +
|
|
1438
|
+
` source: ${JSON.stringify(fieldsToTranslate[h.field])}\n` +
|
|
1439
|
+
` got: ${JSON.stringify(h.value)}\n` +
|
|
1440
|
+
(h.fallbackReason ? ` the fallback (${pairConfig.fallback.method}) failed it too: ${h.fallbackReason}\n` : '') +
|
|
1441
|
+
' Nothing was written or cached. If this is a low-coverage target\n' +
|
|
1442
|
+
' language, the model has no vocabulary for this string.\n' +
|
|
1443
|
+
` ${describeNewHold({ file: label, pairKey, pairConfig, count: fm.hollowed.length })}`
|
|
971
1444
|
);
|
|
1445
|
+
err.heldNext = true;
|
|
1446
|
+
throw err;
|
|
972
1447
|
}
|
|
973
|
-
|
|
974
|
-
fmMisses, fieldsToTranslate, pairConfig,
|
|
975
|
-
{ apiKey, model: pairConfig.model, batchSize: pairConfig.batchSize || 30 },
|
|
976
|
-
);
|
|
977
|
-
if (fmResult) {
|
|
978
|
-
// Content-preservation gate — Phase 2 front matter reached
|
|
979
|
-
// disk AND the TM with no validation at all, so a hollowed
|
|
980
|
-
// page title was written silently and then cached forever.
|
|
981
|
-
// Throwing skips the file and leaves its lock entry alone,
|
|
982
|
-
// so the next sync retries it.
|
|
983
|
-
for (const [field, value] of Object.entries(fmResult)) {
|
|
984
|
-
const sourceValue = fieldsToTranslate[field];
|
|
985
|
-
if (typeof value !== 'string' || typeof sourceValue !== 'string') continue;
|
|
986
|
-
const hollowed = checkContentPreservation(sourceValue, value);
|
|
987
|
-
if (hollowed) {
|
|
988
|
-
throw new Error(
|
|
989
|
-
`Docusaurus content sync for ${code}: front matter "${field}" — ${hollowed.reason}.\n` +
|
|
990
|
-
` source: ${JSON.stringify(sourceValue)}\n` +
|
|
991
|
-
` got: ${JSON.stringify(value)}\n` +
|
|
992
|
-
' Nothing was written or cached. If this is a low-coverage target\n' +
|
|
993
|
-
' language, the model has no vocabulary for this string.'
|
|
994
|
-
);
|
|
995
|
-
}
|
|
996
|
-
}
|
|
997
|
-
Object.assign(translatedFields, fmResult);
|
|
998
|
-
// Cache only what the API actually returned (the same
|
|
999
|
-
// non-null check that gates writing it to the target file).
|
|
1000
|
-
for (const [field, value] of Object.entries(fmResult)) {
|
|
1001
|
-
if (typeof value === 'string' && typeof fieldsToTranslate[field] === 'string') {
|
|
1002
|
-
storeTM(tm, fieldsToTranslate[field], code, tmKey, value);
|
|
1003
|
-
}
|
|
1004
|
-
}
|
|
1005
|
-
} else {
|
|
1448
|
+
if (fm.noResults) {
|
|
1006
1449
|
// Front matter translation failed — loud error
|
|
1007
1450
|
throw new Error(
|
|
1008
|
-
`Docusaurus content sync for ${code}: front matter translation returned no results
|
|
1451
|
+
`Docusaurus content sync for ${code}: front matter translation returned no results`
|
|
1452
|
+
+ `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
|
|
1009
1453
|
' Check your API key and method configuration.'
|
|
1010
1454
|
);
|
|
1011
1455
|
}
|
|
1456
|
+
// Cache only gate-passing values, each under the TM key of the
|
|
1457
|
+
// method that produced it.
|
|
1458
|
+
for (const st of fm.stores) storeTM(tm, st.text, code, st.tmKey, st.value);
|
|
1459
|
+
// A field only the fallback may be asked for that it did not
|
|
1460
|
+
// translate (skipped by --max-cost, or no answer): still refused
|
|
1461
|
+
// by the pair's method — the page is held, as with no fallback.
|
|
1462
|
+
const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
|
|
1463
|
+
if (unfilled.length > 0) {
|
|
1464
|
+
const err = new Error('held');
|
|
1465
|
+
err.heldNames = unfilled.map(f => `front matter "${f}"`);
|
|
1466
|
+
throw err;
|
|
1467
|
+
}
|
|
1468
|
+
Object.assign(translatedFields, fm.translated);
|
|
1469
|
+
for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
|
|
1012
1470
|
}
|
|
1013
1471
|
}
|
|
1014
1472
|
|
|
@@ -1020,40 +1478,92 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1020
1478
|
let translatedBody = body;
|
|
1021
1479
|
let bodyUsedFallback = false;
|
|
1022
1480
|
if (body.trim()) {
|
|
1023
|
-
|
|
1024
|
-
|
|
1481
|
+
// The pair's cache, then its fallback's (lib/fallback.js ladder).
|
|
1482
|
+
const wholeBodyCached = lookupContentTM(tm, body, code, pairConfig);
|
|
1025
1483
|
if (wholeBodyCached !== null) {
|
|
1026
|
-
translatedBody = wholeBodyCached;
|
|
1484
|
+
translatedBody = wholeBodyCached.text;
|
|
1485
|
+
for (const src of refusal.blockSources) refusal.filled.add(blockUnit(src));
|
|
1486
|
+
if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
|
|
1027
1487
|
} else {
|
|
1028
1488
|
const segMode = pairConfig.contentSegmentation || config.contentSegmentation || 'block';
|
|
1029
1489
|
const { protectedBody, blocks } = protectBlocks(body);
|
|
1030
1490
|
const promptOptions = {
|
|
1031
1491
|
sourceLanguageName: DEFAULT_REGISTERS[inputLocale]?.name || inputLocale,
|
|
1032
1492
|
promptContext: pairConfig.promptContext || null,
|
|
1493
|
+
protectedTerms: pairConfig.protectedTerms || [],
|
|
1033
1494
|
};
|
|
1034
1495
|
|
|
1035
1496
|
// Block stores are deferred until the reassembled body passes
|
|
1036
1497
|
// the orphaned-placeholder check — the TM must never hold a
|
|
1037
|
-
// value that would fail the gate on re-serve.
|
|
1498
|
+
// value that would fail the gate on re-serve. Each carries the
|
|
1499
|
+
// TM key of the method that produced it.
|
|
1038
1500
|
const pendingBlockStores = [];
|
|
1501
|
+
// The whole body is cached under one method's key — only when
|
|
1502
|
+
// one method produced all of it.
|
|
1503
|
+
let wholeBodyKey = tmKey;
|
|
1504
|
+
let mixedProducers = false;
|
|
1505
|
+
// Page mode: whose whole page the checks below refuse (the
|
|
1506
|
+
// lane remembers it — lib/content-refusals.js "page").
|
|
1507
|
+
let pageRefusedBy = null;
|
|
1039
1508
|
|
|
1040
1509
|
if (segMode === 'page') {
|
|
1041
|
-
// Whole-page prompt — today's single-call behavior.
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
);
|
|
1510
|
+
// Whole-page prompt — today's single-call behavior. Refused
|
|
1511
|
+
// before by this method (and the cache does not hold it): not
|
|
1512
|
+
// sent again (lib/content-refusals.js).
|
|
1513
|
+
const pageHold = holds.page(body);
|
|
1514
|
+
if (pageHold === 'held') {
|
|
1515
|
+
const err = new Error('held');
|
|
1516
|
+
err.heldNames = [PAGE_NAME];
|
|
1517
|
+
throw err;
|
|
1518
|
+
}
|
|
1519
|
+
await requirePairReady(pairKey, pairConfig, code, 'body translation');
|
|
1520
|
+
const runPage = (cfg) => translateRawContent(
|
|
1521
|
+
buildContentPrompt(protectedBody, cfg, promptOptions), { apiKey, pairConfig: cfg });
|
|
1522
|
+
let bodyResult;
|
|
1523
|
+
if (pairConfig.fallback) {
|
|
1524
|
+
// When both fail, the primary's page goes to the checks
|
|
1525
|
+
// below, which fail the file exactly as without a fallback.
|
|
1526
|
+
// Refused before by the pair's method: only the fallback is asked.
|
|
1527
|
+
const page = await translatePageWithFallback({
|
|
1528
|
+
body, blocks, pairConfig, runPage, budget: fallbackBudget, fallbackOnly: pageHold === 'fallback-only',
|
|
1529
|
+
});
|
|
1530
|
+
tallyContent(pairKey, page.report, label);
|
|
1531
|
+
if (page.refusedBy.length > 0) {
|
|
1532
|
+
refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: page.refusedBy });
|
|
1533
|
+
}
|
|
1534
|
+
if (pageHold === 'fallback-only' && page.body === null) {
|
|
1535
|
+
// The fallback did not translate it: the pair's method
|
|
1536
|
+
// refused it before, so it is not asked — refused now
|
|
1537
|
+
// by the fallback too, or held (no answer, or skipped
|
|
1538
|
+
// by --max-cost).
|
|
1539
|
+
if (page.refusedBy.length > 0) {
|
|
1540
|
+
const err = new Error(
|
|
1541
|
+
`Docusaurus body for ${code}: the fallback (${pairConfig.fallback.method}) failed the whole-page checks too.\n` +
|
|
1542
|
+
` Nothing was written or cached. ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`
|
|
1543
|
+
);
|
|
1544
|
+
err.heldNext = true;
|
|
1545
|
+
throw err;
|
|
1546
|
+
}
|
|
1547
|
+
const err = new Error('held');
|
|
1548
|
+
err.heldNames = [PAGE_NAME];
|
|
1549
|
+
throw err;
|
|
1550
|
+
}
|
|
1551
|
+
bodyResult = page.body ?? page.primaryBody;
|
|
1552
|
+
if (page.body !== null) wholeBodyKey = page.tmKey;
|
|
1553
|
+
else if (page.refusedBy.length > 0) pageRefusedBy = page.refusedBy;
|
|
1554
|
+
} else {
|
|
1555
|
+
const raw = await runPage(pairConfig);
|
|
1556
|
+
bodyResult = raw ? restoreBlocks(raw, blocks) : null;
|
|
1557
|
+
if (bodyResult) pageRefusedBy = [tmKey];
|
|
1047
1558
|
}
|
|
1048
|
-
const prompt = buildContentPrompt(protectedBody, pairConfig, promptOptions);
|
|
1049
|
-
const bodyResult = await translateRawContent(prompt, { apiKey, pairConfig });
|
|
1050
1559
|
if (!bodyResult) {
|
|
1051
1560
|
throw new Error(
|
|
1052
|
-
`Docusaurus body for ${code}: translation returned no results
|
|
1561
|
+
`Docusaurus body for ${code}: translation returned no results`
|
|
1562
|
+
+ `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
|
|
1053
1563
|
' Check your API key and method configuration.'
|
|
1054
1564
|
);
|
|
1055
1565
|
}
|
|
1056
|
-
translatedBody =
|
|
1566
|
+
translatedBody = bodyResult;
|
|
1057
1567
|
} else {
|
|
1058
1568
|
// Block mode: segment the PROTECTED body (placeholders are
|
|
1059
1569
|
// single tokens, so they can never be split), serve blocks
|
|
@@ -1073,72 +1583,140 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1073
1583
|
}));
|
|
1074
1584
|
|
|
1075
1585
|
const missed = [];
|
|
1586
|
+
const heldBlocks = []; // refused before: not sent, '[EN] ' kept
|
|
1587
|
+
let position = 0; // place among the translatable blocks
|
|
1076
1588
|
for (const r of rendered) {
|
|
1077
1589
|
if (r.seg.type !== 'translatable') {
|
|
1078
1590
|
// Separators + passthrough blocks: copied verbatim, never billed.
|
|
1079
1591
|
r.out = r.source;
|
|
1080
1592
|
continue;
|
|
1081
1593
|
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1594
|
+
// Its name in the repeat check (content:<file>#<n>).
|
|
1595
|
+
r.pos = position++;
|
|
1596
|
+
const cached = lookupContentTM(tm, r.source, code, pairConfig);
|
|
1084
1597
|
if (cached !== null) {
|
|
1085
|
-
r.out = cached;
|
|
1086
|
-
|
|
1087
|
-
|
|
1598
|
+
r.out = cached.text;
|
|
1599
|
+
if (cached.fromFallback) { mixedProducers = true; notePage(pairKey, label); }
|
|
1600
|
+
refusal.filled.add(blockUnit(r.source));
|
|
1601
|
+
continue;
|
|
1602
|
+
}
|
|
1603
|
+
const hold = holds.block(r.source);
|
|
1604
|
+
if (hold === 'held') {
|
|
1605
|
+
// Refused before by this method (and its fallback): the
|
|
1606
|
+
// honest last resort stays, nothing is sent or billed.
|
|
1607
|
+
r.out = restoreBlocks(config.fallbackPrefix + r.seg.text, blocks);
|
|
1608
|
+
heldBlocks.push(r);
|
|
1609
|
+
continue;
|
|
1088
1610
|
}
|
|
1611
|
+
r.fallbackOnly = hold === 'fallback-only';
|
|
1612
|
+
missed.push(r);
|
|
1613
|
+
}
|
|
1614
|
+
if (heldBlocks.length > 0) {
|
|
1615
|
+
bodyUsedFallback = true;
|
|
1616
|
+
const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
|
|
1617
|
+
refusal.held.push(...names);
|
|
1618
|
+
output.warn(describeHeld({ file: label, code, pairKey, pairConfig, names, fallbackPrefix: config.fallbackPrefix }));
|
|
1089
1619
|
}
|
|
1090
1620
|
|
|
1091
1621
|
if (missed.length > 0) {
|
|
1092
|
-
|
|
1093
|
-
throw new Error(
|
|
1094
|
-
`Docusaurus body translation for ${code}: no API key available.\n` +
|
|
1095
|
-
' Set OPENROUTER_API_KEY in .env.local to translate content.'
|
|
1096
|
-
);
|
|
1097
|
-
}
|
|
1622
|
+
await requirePairReady(pairKey, pairConfig, code, 'body translation');
|
|
1098
1623
|
// Self-repair ladder (translateBlockBatchResilient): full
|
|
1099
|
-
// batch → one missing-segments-only retry →
|
|
1624
|
+
// batch → one missing-segments-only retry → (with a
|
|
1625
|
+
// fallback method: one batch through it for every block the
|
|
1626
|
+
// primary dropped or damaged — lib/fallback.js) → honest
|
|
1100
1627
|
// '[EN] '-prefixed source for anything still missing. A
|
|
1101
1628
|
// duplicate/unknown marker or an empty first response
|
|
1102
|
-
// still fails the file whole.
|
|
1103
|
-
const
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1629
|
+
// still fails the file whole when no fallback rescues it.
|
|
1630
|
+
const outcome = await translateBlocksWithFallback({
|
|
1631
|
+
missed,
|
|
1632
|
+
blocks,
|
|
1633
|
+
pairConfig,
|
|
1634
|
+
fallbackPrefix: config.fallbackPrefix,
|
|
1635
|
+
budget: fallbackBudget,
|
|
1636
|
+
sharedOutputs: sharedOutputsFor(code),
|
|
1637
|
+
label: docLabel,
|
|
1638
|
+
runBatch: (texts, cfg) => translateBlockBatchResilient({
|
|
1639
|
+
texts,
|
|
1640
|
+
buildPrompt: (t) => buildBlockBatchPrompt(t, cfg, { ...promptOptions, pageTitle }),
|
|
1641
|
+
callModel: (prompt) => translateRawContent(prompt, { apiKey, pairConfig: cfg }),
|
|
1109
1642
|
fallbackPrefix: config.fallbackPrefix,
|
|
1110
|
-
})
|
|
1111
|
-
|
|
1643
|
+
}),
|
|
1644
|
+
fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
|
|
1645
|
+
});
|
|
1646
|
+
tallyContent(pairKey, outcome.report, label);
|
|
1647
|
+
// Refused blocks are remembered; translated ones drop a record.
|
|
1648
|
+
const fellSet = new Set(outcome.fellBack);
|
|
1649
|
+
const newlyHeld = [];
|
|
1650
|
+
missed.forEach((r, i) => {
|
|
1651
|
+
if (!fellSet.has(i)) { refusal.filled.add(blockUnit(r.source)); return; }
|
|
1652
|
+
const methods = outcome.refusedBy?.[i] || [];
|
|
1653
|
+
if (methods.length > 0) {
|
|
1654
|
+
refusal.refused.push({ unit: blockUnit(r.source), source: r.source, methods });
|
|
1655
|
+
newlyHeld.push(i);
|
|
1656
|
+
}
|
|
1657
|
+
});
|
|
1658
|
+
if (outcome.fromFallback > 0) mixedProducers = true;
|
|
1659
|
+
if ((outcome.sharedOutput || []).length > 0) {
|
|
1660
|
+
output.warn(
|
|
1661
|
+
`Docusaurus body for ${code}: ${outcome.sharedOutput.length} block(s) of ${dirName}/${relPath} came back as the same text ` +
|
|
1662
|
+
'the model gave for other, different source strings (a memorized sentence, not a translation) — refused.'
|
|
1663
|
+
);
|
|
1664
|
+
}
|
|
1665
|
+
if ((outcome.refused || []).length > 0) {
|
|
1666
|
+
output.warn(
|
|
1667
|
+
`Docusaurus body for ${code}: ${outcome.refused.length} block(s) of ${dirName}/${relPath} refused by the quality gate — `
|
|
1668
|
+
+ outcome.refused.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
|
|
1669
|
+
+ `${outcome.refused.length > 3 ? '; …' : ''}.`
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
const { fellBack } = outcome;
|
|
1112
1673
|
if (fellBack.length > 0) {
|
|
1113
1674
|
bodyUsedFallback = true;
|
|
1675
|
+
const refusedHere = (outcome.sharedOutput || []).length + (outcome.refused || []).length;
|
|
1676
|
+
const why = outcome.report?.attempted > 0
|
|
1677
|
+
? `neither the primary (${pairConfig.method}) nor its fallback (${pairConfig.fallback.method}) translated safely`
|
|
1678
|
+
: refusedHere >= fellBack.length ? 'refused (above)'
|
|
1679
|
+
: refusedHere > 0 ? `refused (${refusedHere}, above) or missing from the model response after a retry`
|
|
1680
|
+
: 'missing from the model response after a retry';
|
|
1114
1681
|
output.warn(
|
|
1115
1682
|
`Docusaurus body for ${code}: ${fellBack.length} of ${missed.length} ` +
|
|
1116
|
-
`block(s)
|
|
1117
|
-
`'${config.fallbackPrefix}'-prefixed source.
|
|
1118
|
-
|
|
1683
|
+
`block(s) ${why} — written as ` +
|
|
1684
|
+
`'${config.fallbackPrefix}'-prefixed source. ` +
|
|
1685
|
+
describeFallenBack({
|
|
1686
|
+
file: label, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: outcome.refusedBy,
|
|
1687
|
+
})
|
|
1119
1688
|
);
|
|
1120
1689
|
}
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
if (!fellBackSet.has(i)) {
|
|
1126
|
-
pendingBlockStores.push({ source: r.source, translation: r.out });
|
|
1127
|
-
}
|
|
1128
|
-
});
|
|
1690
|
+
// Fallen-back segments are never TM-cached — an error
|
|
1691
|
+
// cached is an error forever (they re-bill next sync).
|
|
1692
|
+
missed.forEach((r, i) => { r.out = outcome.outs[i]; });
|
|
1693
|
+
pendingBlockStores.push(...outcome.stores);
|
|
1129
1694
|
}
|
|
1130
1695
|
|
|
1131
1696
|
// Reassemble in order with the source's exact separators.
|
|
1132
1697
|
translatedBody = rendered.map(r => r.out).join('');
|
|
1133
1698
|
}
|
|
1134
1699
|
|
|
1700
|
+
// A page translated whole that fails the checks below is
|
|
1701
|
+
// remembered as refused: the next plain sync does not send it
|
|
1702
|
+
// to the same method again (lib/content-refusals.js "page").
|
|
1703
|
+
const refusePage = (err) => {
|
|
1704
|
+
if (!pageRefusedBy) return err;
|
|
1705
|
+
if (!refusal.refused.some(r => r.unit === PAGE_UNIT)) {
|
|
1706
|
+
refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: pageRefusedBy });
|
|
1707
|
+
}
|
|
1708
|
+
err.message += `\n ${describeNewHold({ file: label, pairKey, pairConfig, count: 1 })}`;
|
|
1709
|
+
err.heldNext = true;
|
|
1710
|
+
return err;
|
|
1711
|
+
};
|
|
1712
|
+
|
|
1135
1713
|
// Orphaned-placeholder check on the REASSEMBLED body — the
|
|
1136
1714
|
// same gate for both modes.
|
|
1137
1715
|
if (hasOrphanedPlaceholders(translatedBody)) {
|
|
1138
|
-
throw new Error(
|
|
1716
|
+
throw refusePage(new Error(
|
|
1139
1717
|
`Docusaurus body for ${code}: placeholder corruption detected.\n` +
|
|
1140
1718
|
' Code blocks were corrupted during translation.'
|
|
1141
|
-
);
|
|
1719
|
+
));
|
|
1142
1720
|
}
|
|
1143
1721
|
|
|
1144
1722
|
// Content-preservation check on the REASSEMBLED body — same
|
|
@@ -1147,22 +1725,23 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1147
1725
|
// source text and is neither cached nor lock-advanced already.
|
|
1148
1726
|
const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
|
|
1149
1727
|
if (bodyHollowed) {
|
|
1150
|
-
throw new Error(
|
|
1728
|
+
throw refusePage(new Error(
|
|
1151
1729
|
`Docusaurus body for ${code}: ${bodyHollowed.reason}.\n` +
|
|
1152
1730
|
' Nothing was written or cached. If this is a low-coverage target\n' +
|
|
1153
1731
|
' language, the model has no vocabulary for this text.'
|
|
1154
|
-
);
|
|
1732
|
+
));
|
|
1155
1733
|
}
|
|
1734
|
+
if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
|
|
1156
1735
|
|
|
1157
1736
|
// Store per-block AND whole-body entries only after the check
|
|
1158
1737
|
// passes (whole-body makes reverts/lock-loss re-runs free). A
|
|
1159
1738
|
// fallback body is NEVER stored whole — it contains
|
|
1160
1739
|
// untranslated '[EN] ' text.
|
|
1161
1740
|
for (const s of pendingBlockStores) {
|
|
1162
|
-
storeTM(tm, s.source, code, tmKey, s.translation);
|
|
1741
|
+
storeTM(tm, s.source, code, s.tmKey, s.translation);
|
|
1163
1742
|
}
|
|
1164
|
-
if (!bodyUsedFallback) {
|
|
1165
|
-
storeTM(tm, body, code,
|
|
1743
|
+
if (!bodyUsedFallback && !mixedProducers) {
|
|
1744
|
+
storeTM(tm, body, code, wholeBodyKey, translatedBody);
|
|
1166
1745
|
}
|
|
1167
1746
|
}
|
|
1168
1747
|
}
|
|
@@ -1186,10 +1765,44 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1186
1765
|
}
|
|
1187
1766
|
|
|
1188
1767
|
} catch (contentErr) {
|
|
1768
|
+
if (contentErr.heldNames) {
|
|
1769
|
+
// A front-matter field refused before: held back, not failed —
|
|
1770
|
+
// nothing was sent, nothing written, the lock not advanced.
|
|
1771
|
+
const refusal = refusalStates.get(manifestKey);
|
|
1772
|
+
refusal.held.push(...contentErr.heldNames);
|
|
1773
|
+
refusal.pageHeld = true;
|
|
1774
|
+
settleRefusals();
|
|
1775
|
+
output.warn(describeHeld({
|
|
1776
|
+
file: `${dirName}/${relPath}`, code, pairKey, pairConfig, names: contentErr.heldNames, pageHeld: true,
|
|
1777
|
+
fallbackPrefix: config.fallbackPrefix,
|
|
1778
|
+
}));
|
|
1779
|
+
completed++;
|
|
1780
|
+
output.raw(` [${completed}/${totalWork}] ${dirName}/${relPath} → ${code} [HELD]`);
|
|
1781
|
+
output.event('file', { lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code, status: 'held-refused', held: contentErr.heldNames.length, action });
|
|
1782
|
+
if (completed % MANIFEST_WRITE_INTERVAL === 0) writeManifestIfDirty();
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1785
|
+
itemFailed = true;
|
|
1189
1786
|
contentFailures++;
|
|
1190
1787
|
output.error(`${dirName}/${relPath} → ${code} — ${contentErr.message}`);
|
|
1788
|
+
// Nothing was written for this item, and its lock entry was not
|
|
1789
|
+
// advanced. If the lock already matches this source, the file on
|
|
1790
|
+
// disk is the PREVIOUS translation of the same text (a forced
|
|
1791
|
+
// re-translate that failed): it stays, and a plain sync will not
|
|
1792
|
+
// redo it. Otherwise the next sync retries it.
|
|
1793
|
+
const keptPrevious = updatedDocuManifest[manifestKey] === sourceHash && fs.existsSync(targetPath);
|
|
1794
|
+
failedItems.push({
|
|
1795
|
+
file: `${dirName}/${relPath}`,
|
|
1796
|
+
locale: code,
|
|
1797
|
+
error: contentErr.message,
|
|
1798
|
+
// held-back: refused by the quality gate — the next plain sync
|
|
1799
|
+
// does not send it again (lib/content-refusals.js).
|
|
1800
|
+
state: keptPrevious ? 'previous-translation-kept' : contentErr.heldNext ? 'held-back' : 'will-retry',
|
|
1801
|
+
});
|
|
1191
1802
|
}
|
|
1192
1803
|
|
|
1804
|
+
settleRefusals();
|
|
1805
|
+
|
|
1193
1806
|
// Progress reporting
|
|
1194
1807
|
completed++;
|
|
1195
1808
|
const pct = Math.round(100 * completed / totalWork);
|
|
@@ -1198,13 +1811,16 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1198
1811
|
const remainingMs = msPerItem * (totalWork - completed);
|
|
1199
1812
|
const remainingSec = Math.ceil(remainingMs / 1000);
|
|
1200
1813
|
const etaStr = remainingSec > 5 ? ` (~${remainingSec}s left)` : '';
|
|
1201
|
-
|
|
1814
|
+
// The tag describes THIS item. (It used to read the run-wide failure
|
|
1815
|
+
// count, so whichever item finished last was tagged FAIL.)
|
|
1816
|
+
const displayTag = itemFailed
|
|
1202
1817
|
? 'FAIL'
|
|
1203
1818
|
: action === 're-translate' ? 'RE-TRANSLATE' : action === 'changed' ? 'CHANGED' : 'OK';
|
|
1204
|
-
// Show [FAIL] for items that just errored (contentErr was caught above)
|
|
1205
|
-
const itemFailed = updatedDocuManifest[manifestKey] !== sourceHash;
|
|
1206
|
-
const displayTag = itemFailed && !dryRun ? 'FAIL' : tag;
|
|
1207
1819
|
output.raw(` [${completed}/${totalWork}] (${pct}%) ${dirName}/${relPath} → ${code} [${displayTag}]${etaStr}`);
|
|
1820
|
+
output.event('file', {
|
|
1821
|
+
lane: 'docusaurus', file: `${dirName}/${relPath}`, locale: code,
|
|
1822
|
+
status: itemFailed ? 'failed' : dryRun ? 'would-translate' : 'translated', action,
|
|
1823
|
+
});
|
|
1208
1824
|
|
|
1209
1825
|
// Incremental manifest write
|
|
1210
1826
|
if (completed % MANIFEST_WRITE_INTERVAL === 0) {
|
|
@@ -1231,26 +1847,122 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
|
|
|
1231
1847
|
output.ok(`${action} ${totalContent} content file(s)${retranslateNote}, ${totalContentSkipped} unchanged`);
|
|
1232
1848
|
}
|
|
1233
1849
|
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1850
|
+
contentTranslated = totalContent;
|
|
1851
|
+
contentFailedItems = failedItems;
|
|
1852
|
+
for (const st of refusalStates.values()) {
|
|
1853
|
+
if (st.held.length > 0) {
|
|
1854
|
+
contentHeldBack += st.held.length;
|
|
1855
|
+
contentHeldBackItems.push({ file: st.file, locale: st.locale, units: st.held, ...(st.pageHeld && { pageNotWritten: true }) });
|
|
1856
|
+
}
|
|
1857
|
+
contentRefused += new Set(st.refused.map(r => r.unit)).size;
|
|
1858
|
+
}
|
|
1859
|
+
if (contentHeldBack > 0) {
|
|
1860
|
+
output.warn(
|
|
1861
|
+
`${contentHeldBack} content block(s)/field(s) held back in ${contentHeldBackItems.length} translation(s) — the quality gate refused `
|
|
1862
|
+
+ 'the method\'s translation of their current text before; not sent, not billed (listed above). Ask again for all of them: '
|
|
1863
|
+
+ '`champollion sync --redo content`; for one page: `champollion sync --redo files:<page>`.'
|
|
1239
1864
|
);
|
|
1240
1865
|
}
|
|
1866
|
+
for (const [pairKey, tally] of contentFallbackTallies) {
|
|
1867
|
+
printFallbackReport(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)', producedLabel: 'pages' });
|
|
1868
|
+
warnFallbackMajority(pairKey, pairEntries.find(([k]) => k === pairKey)[1].method, tally, { unit: 'content segment(s)' });
|
|
1869
|
+
}
|
|
1870
|
+
if (failedItems.length > 0) {
|
|
1871
|
+
output.raw('');
|
|
1872
|
+
output.raw(` Failed content translations (${failedItems.length}):`);
|
|
1873
|
+
for (const f of failedItems.sort((a, b) => (a.file + a.locale).localeCompare(b.file + b.locale))) {
|
|
1874
|
+
const state = f.state === 'previous-translation-kept'
|
|
1875
|
+
? 'previous translation kept (it matches the current source); re-run with --force-content to try again'
|
|
1876
|
+
: f.state === 'held-back'
|
|
1877
|
+
? `refused by the quality gate — held back: the next sync does not send it again (\`--redo files:${f.file}\` asks again)`
|
|
1878
|
+
: 'not recorded as done — the next sync retries it';
|
|
1879
|
+
output.raw(` ${f.file} → ${f.locale}: ${state}`);
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1241
1882
|
}
|
|
1242
1883
|
|
|
1243
1884
|
// Save TM if it was mutated during this Docusaurus sync (stores OR
|
|
1244
1885
|
// evictions — a size check would miss eviction-only runs and same-key
|
|
1245
|
-
// replacements). Skip when --no-tm is active.
|
|
1886
|
+
// replacements). Skip when --no-tm is active. Saved BEFORE any failure is
|
|
1887
|
+
// raised: the files that succeeded paid for their translations, and
|
|
1888
|
+
// throwing first discarded every one of them from the cache.
|
|
1246
1889
|
if (!dryRun && !noTM && isTMDirty(tm)) {
|
|
1247
|
-
const tmFinalSize = tmSize(tm);
|
|
1248
|
-
const delta = tmFinalSize - tmInitialSize;
|
|
1249
1890
|
saveTM(cwd, tm);
|
|
1250
|
-
output.info(`[TM] Saved ${
|
|
1891
|
+
output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1894
|
+
// Failures are reported, not thrown: a partly failed run still did real
|
|
1895
|
+
// work, and the exit code says so (2 = partial, 1 = nothing succeeded).
|
|
1896
|
+
if (contentFailures > 0) {
|
|
1897
|
+
output.error(
|
|
1898
|
+
`${contentFailures} content translation(s) failed (listed above). ` +
|
|
1899
|
+
'Completed files and their translations are saved; re-run sync to retry.'
|
|
1900
|
+
);
|
|
1251
1901
|
}
|
|
1252
1902
|
|
|
1903
|
+
// The --max-cost verdict, said once — at the end, before the summary
|
|
1904
|
+
// (lib/cost-report.js reportDryRunMaxCost).
|
|
1905
|
+
if (dryMaxCost) output[dryMaxCost.level](dryMaxCost.message);
|
|
1906
|
+
|
|
1907
|
+
// One machine-readable record of the run — the Docusaurus path never had
|
|
1908
|
+
// one, so --json consumers saw nothing to tell success from failure.
|
|
1909
|
+
output.summary({
|
|
1910
|
+
command: 'sync',
|
|
1911
|
+
format: 'docusaurus',
|
|
1912
|
+
dryRun,
|
|
1913
|
+
totalProcessed: totalJSONKeys,
|
|
1914
|
+
totalFailed: totalJSONFailed,
|
|
1915
|
+
totalCopied: totalJSONCopied,
|
|
1916
|
+
// UI strings held back: the gate refused this method's translation of
|
|
1917
|
+
// their current text before — not sent, not billed (lib/locale-state.js).
|
|
1918
|
+
// A dry run sends nothing either way: said above, not counted (as on
|
|
1919
|
+
// the key-value path).
|
|
1920
|
+
totalHeld: dryRun ? 0 : totalJSONHeld,
|
|
1921
|
+
// What each pair's fallback method did for JSON keys (pairs with a
|
|
1922
|
+
// fallback, real runs); Markdown segments are under content.fallback.
|
|
1923
|
+
...(keyFallbackTallies.size > 0 && {
|
|
1924
|
+
fallback: [...keyFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
|
|
1925
|
+
}),
|
|
1926
|
+
content: {
|
|
1927
|
+
translated: contentTranslated,
|
|
1928
|
+
skipped: contentScan.totalContentSkipped,
|
|
1929
|
+
failed: contentFailedItems.length,
|
|
1930
|
+
failedItems: contentFailedItems,
|
|
1931
|
+
// Refused before by the method, not sent this run (lib/content-refusals.js),
|
|
1932
|
+
// and refused this run (held back from the next).
|
|
1933
|
+
heldBack: contentHeldBack,
|
|
1934
|
+
heldBackItems: contentHeldBackItems,
|
|
1935
|
+
refused: contentRefused,
|
|
1936
|
+
...(contentFallbackTallies.size > 0 && {
|
|
1937
|
+
fallback: [...contentFallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
|
|
1938
|
+
}),
|
|
1939
|
+
},
|
|
1940
|
+
costEstimate,
|
|
1941
|
+
// Dry runs with --max-cost: whether the real run would stop at the cap.
|
|
1942
|
+
...(dryMaxCost && { maxCost: {
|
|
1943
|
+
cap: dryMaxCost.cap, estimatedCost: dryMaxCost.estimatedCost, wouldStop: dryMaxCost.wouldStop,
|
|
1944
|
+
...(dryMaxCost.wouldStop && { exitCode: 2, reason: dryMaxCost.reason }),
|
|
1945
|
+
...(dryMaxCost.stopsEarlier && { exitCode: 1, stopsEarlier: dryMaxCost.stopsEarlier }),
|
|
1946
|
+
} }),
|
|
1947
|
+
// Keys named for a redo that match no UI string (the run exits 1), each
|
|
1948
|
+
// with the closest ids that exist.
|
|
1949
|
+
...(unmatchedNamed.length > 0 && { unmatchedKeys: unmatchedKeysSummary(unmatchedNamed) }),
|
|
1950
|
+
});
|
|
1253
1951
|
output.raw('');
|
|
1952
|
+
// Named keys that matched nothing: said again last, where a reader (or CI)
|
|
1953
|
+
// looks for the verdict — and the run fails (lib/commands/sync.js).
|
|
1954
|
+
reportUnmatchedKeys(unmatchedNamed, cliArgs);
|
|
1955
|
+
return {
|
|
1956
|
+
totalProcessed: totalJSONKeys,
|
|
1957
|
+
totalFailed: totalJSONFailed,
|
|
1958
|
+
totalCopied: totalJSONCopied,
|
|
1959
|
+
totalHeld: dryRun ? 0 : totalJSONHeld,
|
|
1960
|
+
contentTranslated,
|
|
1961
|
+
contentFailed: contentFailedItems.length,
|
|
1962
|
+
contentHeldBack,
|
|
1963
|
+
contentRefused,
|
|
1964
|
+
unmatchedKeys: unmatchedNamed.map(m => m.name),
|
|
1965
|
+
};
|
|
1254
1966
|
}
|
|
1255
1967
|
|
|
1256
1968
|
export { runDocusaurusSync, discoverDocusaurusJSONFiles };
|