champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
package/lib/sync.js
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
* This is the core "do the thing" module. It:
|
|
5
5
|
* 1. Prints version banner (e.g., "champollion v3.4.0")
|
|
6
6
|
* 2. Reads the source locale file (JSON, TOML, or YAML)
|
|
7
|
-
* 3. Logs detected format and
|
|
7
|
+
* 3. Logs detected format and content directory (e.g., "Detected format: json (auto)";
|
|
8
|
+
* "Detected framework: Hugo (hugo.toml)" only on real Hugo evidence)
|
|
8
9
|
* 4. Loads the hash manifest to detect changed English content
|
|
9
10
|
* 5. Iterates over all target pairs (v3 pair graph)
|
|
10
11
|
* 6. Diffs each one against the source (missing + fallback + changed + forced)
|
|
@@ -25,7 +26,7 @@
|
|
|
25
26
|
* - lib/translate-pair.js — shared TM→API→gate pipeline (used by both sync paths)
|
|
26
27
|
* - lib/docusaurus-sync.js — Docusaurus JSON + Markdown sync
|
|
27
28
|
* - lib/cost-report.js — pre-sync cost estimation display
|
|
28
|
-
* - lib/content-sync.js — Hugo
|
|
29
|
+
* - lib/content-sync.js — contentDir Markdown sync (Hugo or any Markdown folder)
|
|
29
30
|
* - lib/watch.js — file watcher for auto-sync
|
|
30
31
|
* - lib/output.js — banner(), progressBar(), and all CLI output
|
|
31
32
|
*/
|
|
@@ -33,14 +34,27 @@
|
|
|
33
34
|
import fs from 'node:fs';
|
|
34
35
|
import path from 'node:path';
|
|
35
36
|
import { createRequire } from 'node:module';
|
|
36
|
-
import { flattenKeys, setNestedValue } from './flatten.js';
|
|
37
|
-
import { diffLocale, diffLabel } from './diff.js';
|
|
38
|
-
import {
|
|
39
|
-
import { resolveConfig, autoDetectLanguages, DEFAULT_JSON_CONCURRENCY } from './config.js';
|
|
37
|
+
import { flattenKeys, setNestedValue, deleteNestedValue, assignInOrder } from './flatten.js';
|
|
38
|
+
import { diffLocale, diffLabel, queueReasons } from './diff.js';
|
|
39
|
+
import { getMethod } from './translate.js';
|
|
40
|
+
import { resolveConfig, autoDetectLanguages, DEFAULT_JSON_CONCURRENCY, DEFAULT_BATCH_SIZE } from './config.js';
|
|
40
41
|
import { compileNoTranslate } from './no-translate.js';
|
|
41
|
-
import { buildHashManifest, detectChangedKeys,
|
|
42
|
-
import {
|
|
43
|
-
|
|
42
|
+
import { buildHashManifest, detectChangedKeys, readLock, writeManifest, LOCK_FILENAME } from './hash.js';
|
|
43
|
+
import {
|
|
44
|
+
LockState, planQueue, createEditClassifier, encodeWritten, recordReplacedEdits,
|
|
45
|
+
REPLACED_EDITS_FILENAME, localeHealth, decodeWritten, valueHash, encodeForm, decodeForm,
|
|
46
|
+
recordRefusal, describeHeldKeys, describeFallbackOnlyKeys, keyFateNote, describeHeldNext,
|
|
47
|
+
} from './locale-state.js';
|
|
48
|
+
import { splitKeyList } from './redo.js';
|
|
49
|
+
import { applyNamedKeyRule, reportUnmatchedKeys, unmatchedKeysSummary, reportNamedFromCache as sayNamedFromCache } from './named-keys.js';
|
|
50
|
+
import { SharedOutputIndex, sharedOutputItems, droppedTerminalMarks, describeDroppedMarks, isProtectedTermValue } from './validate.js';
|
|
51
|
+
import {
|
|
52
|
+
discoverLocaleLayout, loadSourceUnits, expectedForTarget, readLocaleData, readLocaleFlat,
|
|
53
|
+
writeLocaleData, lockKey, keysForNamespace, NS_SEPARATOR,
|
|
54
|
+
} from './locale-layout.js';
|
|
55
|
+
import { mapSourceKeysToTarget, originKey, describePluralFormChanges, pluralExtraKeys } from './plurals.js';
|
|
56
|
+
import { planGapRedo, recordGap } from './plural-gap-redo.js';
|
|
57
|
+
import { resolvePairs, filterPairGraph, parsePairKey, estimateCost } from './pairs.js';
|
|
44
58
|
import { loadPlugins, resolvePluginForPair } from './plugins.js';
|
|
45
59
|
import { isPathContained } from './security.js';
|
|
46
60
|
import { loadApiKey } from './api-key.js';
|
|
@@ -51,15 +65,37 @@ import {
|
|
|
51
65
|
converterKeyForLocale, formatScriptChoiceError,
|
|
52
66
|
} from './scripts.js';
|
|
53
67
|
import { getLanguageCard } from './registers.js';
|
|
54
|
-
import {
|
|
68
|
+
import {
|
|
69
|
+
loadTM, saveTM, tmSize, isTMDirty, setModelCarryover, setTMReads, tmMethodKey, describeTMChanges,
|
|
70
|
+
tmTranslationsOf, bypassTMFor, findOtherMethodEntries, describeMethodKey, lookupTM, peekTM, tmMethodKeysHolding,
|
|
71
|
+
adoptLegacyCoachingKeys, canonicalWriterKey, fallbackWriterOf,
|
|
72
|
+
} from './tm.js';
|
|
73
|
+
import { tmSourceText, tmTextFor, tmProofTextsFor, splitSharedPluralEntries, createTMEvictor } from './tm-evict.js';
|
|
55
74
|
import { verifyTerminology, logTermViolations } from './terminology.js';
|
|
56
75
|
import { output } from './output.js';
|
|
57
|
-
import {
|
|
76
|
+
import {
|
|
77
|
+
printCostEstimate, parseMaxCost, abortForMaxCost, maxCostVerdict, reportDryRunMaxCost, warnModelSwitchStrandedTM, dryRunCiHint,
|
|
78
|
+
translationsBreakdown,
|
|
79
|
+
preflightStopReason,
|
|
80
|
+
} from './cost-report.js';
|
|
58
81
|
import { runDocusaurusSync } from './docusaurus-sync.js';
|
|
59
|
-
import {
|
|
60
|
-
import {
|
|
82
|
+
import { translateWithFallback, previewRequests } from './translate-pair.js';
|
|
83
|
+
import {
|
|
84
|
+
tmKeysForPair, tmHoldsValue, createFallbackBudget, newFallbackReport, addToTally,
|
|
85
|
+
fallbackSummary, printFallbackReport, warnFallbackMajority,
|
|
86
|
+
} from './fallback.js';
|
|
87
|
+
import { verifyLocales, redoCommand, contentRedoCommand, shellWord, pluralGapsInFile } from './verify.js';
|
|
88
|
+
import { projectSharedOutputIndex } from './shared-output-seed.js';
|
|
89
|
+
import { describeCategories, describePluralGap } from './icu-structure.js';
|
|
90
|
+
import { poPluralSlots, poFuzzyKeys } from './po.js';
|
|
61
91
|
import { pMap } from './concurrent.js';
|
|
92
|
+
import { costLabel } from './cost-label.js';
|
|
93
|
+
import { ciSecretLine, secretNamesIn, missingKeyAdvice, methodSetupAdvice, onCIRunner } from './missing-key.js';
|
|
62
94
|
import { resetTranslationError } from './methods/translation-error.js';
|
|
95
|
+
import { PLAIN_LLM_METHODS } from './methods/llm.js';
|
|
96
|
+
import { compileFileScope } from './file-scope.js';
|
|
97
|
+
import { flutterLocaleLines } from './flutter-locales.js';
|
|
98
|
+
import { discoverContentFiles, detectContentSite } from './content.js';
|
|
63
99
|
|
|
64
100
|
|
|
65
101
|
/**
|
|
@@ -103,13 +139,16 @@ async function resolveRuntime(config, cwd, cliArgs = {}) {
|
|
|
103
139
|
|
|
104
140
|
// Build the pair graph — this is the v3 drivetrain.
|
|
105
141
|
// Each pair carries its method, model, register, and plugin context.
|
|
106
|
-
const pairs = resolvePairs(runtimeConfig);
|
|
142
|
+
const pairs = resolvePairs(runtimeConfig, { cwd });
|
|
107
143
|
const plugins = loadPlugins(cwd);
|
|
108
144
|
|
|
109
145
|
// Resolve plugin configs into each pair that references one.
|
|
110
146
|
let resolvedPairs = new Map();
|
|
111
147
|
for (const [pairKey, rawPairConfig] of pairs) {
|
|
112
|
-
|
|
148
|
+
const resolved = resolvePluginForPair(plugins, rawPairConfig);
|
|
149
|
+
// A fallback may name a plugin too — merged the same way, as its own pair.
|
|
150
|
+
if (resolved.fallback) resolved.fallback = resolvePluginForPair(plugins, resolved.fallback);
|
|
151
|
+
resolvedPairs.set(pairKey, resolved);
|
|
113
152
|
}
|
|
114
153
|
|
|
115
154
|
// ── --pair filter ──────────────────────────────────────────────────
|
|
@@ -129,6 +168,41 @@ async function resolveRuntime(config, cwd, cliArgs = {}) {
|
|
|
129
168
|
// Sort for deterministic output ordering
|
|
130
169
|
const pairEntries = [...resolvedPairs.entries()].sort(([a], [b]) => a.localeCompare(b));
|
|
131
170
|
|
|
171
|
+
// ── GLOSSARY CHECKS, every method ─────────────────────────────────
|
|
172
|
+
// The project glossary (.champollion/coaching/<locale>.json → "dictionary")
|
|
173
|
+
// was only read by the coached method and DeepL, and the terminology check
|
|
174
|
+
// below needs it on the pair — which nothing set, so it never ran. Load it
|
|
175
|
+
// for every pair: the check now warns when ANY method's output skips a
|
|
176
|
+
// glossary term. Stored as `glossary`, not in coachingData, so coached
|
|
177
|
+
// pairs keep their cache keys (tmMethodKey hashes coachingData).
|
|
178
|
+
// Every LLM method is TOLD the glossary terms each batch contains too
|
|
179
|
+
// (lib/methods/llm.js buildUserMessage), so the check never warns about a
|
|
180
|
+
// term the model was not given. null = none for this locale.
|
|
181
|
+
//
|
|
182
|
+
// The file's grammar rules and style notes are coaching: only llm-coached
|
|
183
|
+
// reads them. The plain LLM methods build one and the same prompt, so a
|
|
184
|
+
// project with rules running a plain one is told, once, where they apply.
|
|
185
|
+
{
|
|
186
|
+
const { loadCoachingData, DEFAULT_COACHING_DIR } = await import('./methods/coaching-data.js');
|
|
187
|
+
const coachingDir = path.join(cwd, DEFAULT_COACHING_DIR);
|
|
188
|
+
const cache = new Map();
|
|
189
|
+
for (const [pairKey, pc] of pairEntries) {
|
|
190
|
+
const data = loadCoachingData(coachingDir, pc.target, cache);
|
|
191
|
+
const glossary = data?.dictionary && Object.keys(data.dictionary).length > 0 ? data.dictionary : null;
|
|
192
|
+
pc.glossary = glossary;
|
|
193
|
+
if (pc.fallback) pc.fallback.glossary = glossary;
|
|
194
|
+
const hasCoaching = data && (data.grammar_rules.length > 0 || data.style_notes);
|
|
195
|
+
if (hasCoaching && PLAIN_LLM_METHODS.has(pc.method)) {
|
|
196
|
+
const provider = pc.method === 'llm' ? '' : `, "provider": "${pc.method}"`;
|
|
197
|
+
output.info(
|
|
198
|
+
`${pairKey}: the grammar rules and style notes in ${DEFAULT_COACHING_DIR}/${pc.target}.json are read by the `
|
|
199
|
+
+ `llm-coached method only — ${pc.method} is ${glossary ? 'given its glossary, not them' : 'not given them'}. `
|
|
200
|
+
+ `To use them: "method": "llm-coached"${provider}.`
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
132
206
|
// ── SCRIPT DECISION ────────────────────────────────────────────────
|
|
133
207
|
// For every pair whose locale has a registered script converter, say what
|
|
134
208
|
// will happen and why — once, up front, in dry runs too. Silence here is
|
|
@@ -181,44 +255,166 @@ async function resolveRuntime(config, cwd, cliArgs = {}) {
|
|
|
181
255
|
//
|
|
182
256
|
// No gas, no ignition. If a method can't run, we fail here with
|
|
183
257
|
// clear guidance instead of producing garbage 360 files later.
|
|
184
|
-
//
|
|
185
|
-
//
|
|
186
|
-
|
|
187
|
-
|
|
258
|
+
// Audit (listing what is untranslated) needs no method at all: skipped.
|
|
259
|
+
// A DRY RUN checks too, and warns instead of stopping: it used to skip
|
|
260
|
+
// the check, exit 0 and say nothing, so a dry run in CI could not catch a
|
|
261
|
+
// missing secret (Round 3, Django + Next.js personas). The exit code stays
|
|
262
|
+
// 0 — a dry run is a preview and never fails, the same rule that keeps
|
|
263
|
+
// --max-cost from aborting it — and the JSON summary carries `preflight`.
|
|
264
|
+
let preflightFailures = [];
|
|
265
|
+
// A model server that does not answer (local, LibreTranslate, Apertium),
|
|
266
|
+
// off a CI runner, when the caller decides after its plan: a run that
|
|
267
|
+
// sends nothing to that method (a redo served from the cache) does not
|
|
268
|
+
// need it (Round 11, Django persona). See resolveDeferredProbes.
|
|
269
|
+
let deferredProbeFailures = [];
|
|
270
|
+
if (!cliArgs.audit) {
|
|
188
271
|
const failures = [];
|
|
189
272
|
for (const [pairKey, pairConfig] of pairEntries) {
|
|
190
273
|
const method = getMethod(pairConfig.method || 'llm', pairConfig);
|
|
191
274
|
const readiness = await method.checkReadiness({ apiKey, cwd });
|
|
192
275
|
if (!readiness.ready) {
|
|
193
|
-
failures.push({
|
|
276
|
+
failures.push({
|
|
277
|
+
pairKey, pairConfig, reason: readiness.reason, method,
|
|
278
|
+
unreachable: !!readiness.unreachable, endpoint: readiness.endpoint || null, pair: pairKey, target: pairConfig.target,
|
|
279
|
+
});
|
|
194
280
|
}
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
];
|
|
208
|
-
for (const { pairKey, pairConfig, reason } of failures) {
|
|
209
|
-
lines.push(` ✗ ${pairKey} (method: ${pairConfig.method || 'llm'}): ${reason}`);
|
|
281
|
+
// A configured fallback must be able to run too: discovering a
|
|
282
|
+
// missing key only when the primary first fails would leave those
|
|
283
|
+
// keys untranslated with the reason buried mid-run.
|
|
284
|
+
if (pairConfig.fallback) {
|
|
285
|
+
const fbMethod = getMethod(pairConfig.fallback.method, pairConfig.fallback);
|
|
286
|
+
const fbReadiness = await fbMethod.checkReadiness({ apiKey, cwd });
|
|
287
|
+
if (!fbReadiness.ready) {
|
|
288
|
+
failures.push({
|
|
289
|
+
pairKey: `${pairKey} fallback`, pairConfig: pairConfig.fallback, reason: fbReadiness.reason, method: fbMethod,
|
|
290
|
+
unreachable: !!fbReadiness.unreachable, endpoint: fbReadiness.endpoint || null, pair: pairKey, target: pairConfig.target,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
210
293
|
}
|
|
211
|
-
|
|
294
|
+
}
|
|
295
|
+
// A CI runner keeps the Round 8 rule: a server that does not answer
|
|
296
|
+
// stops the run at once, even when nothing is queued — a workflow that
|
|
297
|
+
// still names "local" fails on its first push, not on the first string
|
|
298
|
+
// that changes.
|
|
299
|
+
const defer = !!cliArgs.deferUnreachable && !onCIRunner();
|
|
300
|
+
deferredProbeFailures = defer ? failures.filter(f => f.unreachable) : [];
|
|
301
|
+
const now = defer ? failures.filter(f => !f.unreachable) : failures;
|
|
302
|
+
preflightFailures = stopForPreflight(now, cliArgs);
|
|
303
|
+
}
|
|
212
304
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
lines.push(...helpLines);
|
|
305
|
+
return { apiKey, resolvedPairs, pairEntries, preflightFailures, deferredProbeFailures };
|
|
306
|
+
}
|
|
216
307
|
|
|
217
|
-
|
|
218
|
-
|
|
308
|
+
/**
|
|
309
|
+
* The preflight's verdict on failures: a dry run warns and returns them (the
|
|
310
|
+
* JSON summary's `preflight.failures`); a real run throws the one error that
|
|
311
|
+
* names every method that is not ready, with the setup help.
|
|
312
|
+
*
|
|
313
|
+
* @param {Array<{ pairKey: string, pairConfig: object, reason: string, method: object }>} failures
|
|
314
|
+
* @param {{ dryRun?: boolean }} cliArgs
|
|
315
|
+
* @returns {Array<{ pair: string, method: string, reason: string }>} for a dry run; [] when nothing failed
|
|
316
|
+
*/
|
|
317
|
+
function stopForPreflight(failures, cliArgs) {
|
|
318
|
+
if (failures.length === 0) return [];
|
|
319
|
+
// The secret(s) a missing-key failure names ("… (OPENROUTER_API_KEY).")
|
|
320
|
+
// and what to do about them, by where the run is (lib/missing-key.js —
|
|
321
|
+
// the one helper every missing-key message goes through).
|
|
322
|
+
const ciLine = ciSecretLine(secretNamesIn(failures.map(f => f.reason)));
|
|
323
|
+
if (cliArgs.dryRun) {
|
|
324
|
+
const preflightFailures = failures.map(({ pairKey, pairConfig, reason }) => ({
|
|
325
|
+
pair: pairKey, method: pairConfig.method || 'llm', reason,
|
|
326
|
+
}));
|
|
327
|
+
output.warn('Dry run only — the real sync would STOP here, before translating anything:');
|
|
328
|
+
for (const f of preflightFailures) output.warn(` ✗ ${f.pair} (method: ${f.method}): ${f.reason}`);
|
|
329
|
+
output.warn(ciLine
|
|
330
|
+
? ` Set what is missing, then run the sync. ${ciLine}`
|
|
331
|
+
: ' Set what is missing (in CI: add it as a secret and pass it to the sync step), then run the sync.');
|
|
332
|
+
return preflightFailures;
|
|
219
333
|
}
|
|
334
|
+
// Build a single, actionable error with all failures + setup help.
|
|
335
|
+
// The FIRST line says what happened: printed after "[ERR] sync failed:",
|
|
336
|
+
// an empty one left that line blank in CI logs (Round 8, i18next
|
|
337
|
+
// persona). It counts METHODS, not pairs: one method on two language
|
|
338
|
+
// pairs is one method that is not ready, said with its pairs (Round 9:
|
|
339
|
+
// "2 methods are not ready" for one).
|
|
340
|
+
const byMethod = new Map();
|
|
341
|
+
for (const f of failures) {
|
|
342
|
+
const m = f.pairConfig.method || 'llm';
|
|
343
|
+
if (!byMethod.has(m)) byMethod.set(m, { pairs: [], reasons: [] });
|
|
344
|
+
const entry = byMethod.get(m);
|
|
345
|
+
entry.pairs.push(f.pairKey);
|
|
346
|
+
const reason = String(f.reason).replace(/[.;\s]+$/, '');
|
|
347
|
+
if (!entry.reasons.includes(reason)) entry.reasons.push(reason);
|
|
348
|
+
}
|
|
349
|
+
const head = byMethod.size === 1
|
|
350
|
+
? (() => { const [[m, e]] = [...byMethod]; return `the ${m} method is not ready for ${e.pairs.join(', ')}: ${e.reasons.join('; ')}`; })()
|
|
351
|
+
: `${byMethod.size} methods are not ready: ${[...byMethod].map(([m, e]) => `${m} (${e.pairs.join(', ')}): ${e.reasons.join('; ')}`).join('; ')}`;
|
|
352
|
+
const lines = [
|
|
353
|
+
`cannot start translating — ${head}.`,
|
|
354
|
+
'',
|
|
355
|
+
' ┌─ PREFLIGHT FAILED ──────────────────────────────────────────────┐',
|
|
356
|
+
' │ Cannot start translation — method prerequisites not met. │',
|
|
357
|
+
' └─────────────────────────────────────────────────────────────────┘',
|
|
358
|
+
'',
|
|
359
|
+
];
|
|
360
|
+
for (const { pairKey, pairConfig, reason } of failures) {
|
|
361
|
+
lines.push(` ✗ ${pairKey} (method: ${pairConfig.method || 'llm'}): ${reason}`);
|
|
362
|
+
}
|
|
363
|
+
lines.push('');
|
|
220
364
|
|
|
221
|
-
|
|
365
|
+
// Setup help from the first failing method (most actionable) — on a CI
|
|
366
|
+
// runner, the repository secret instead of the shell advice (export,
|
|
367
|
+
// .env.local), which is not the fix there.
|
|
368
|
+
lines.push(...missingKeyAdvice({ reasons: failures.map(f => f.reason), setupHelp: failures[0].method.getSetupHelp() }));
|
|
369
|
+
|
|
370
|
+
throw new Error(lines.join('\n'));
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Decide, after the plan, on the model servers that did not answer at
|
|
375
|
+
* startup (resolveRuntime's deferredProbeFailures): a pair whose run sends
|
|
376
|
+
* nothing to its method — every queued key from the cache, or nothing
|
|
377
|
+
* queued, and no Markdown to send — does not need the server: warn that it
|
|
378
|
+
* is down and that nothing needed it, and go on. A pair that sends something
|
|
379
|
+
* stops the run as the preflight always did (a dry run says the real run
|
|
380
|
+
* would stop). A fallback is needed when its pair's method sends anything
|
|
381
|
+
* (it takes what that refuses). No estimate (it failed) = needed.
|
|
382
|
+
*
|
|
383
|
+
* @param {Array<object>} deferred - resolveRuntime's deferredProbeFailures
|
|
384
|
+
* @param {{ costEstimate: object|null, contentByTarget: object, cliArgs: object }} plan
|
|
385
|
+
* @returns {Array<{ pair: string, method: string, reason: string }>} the dry run's added preflight failures
|
|
386
|
+
*/
|
|
387
|
+
function resolveDeferredProbes(deferred, { costEstimate, contentByTarget, cliArgs }) {
|
|
388
|
+
if (!deferred || deferred.length === 0) return [];
|
|
389
|
+
// What the plan sends to the pair's method: key-value keys the cache does
|
|
390
|
+
// not hold, and Markdown the cache does not hold.
|
|
391
|
+
const sends = (f) => {
|
|
392
|
+
if (!costEstimate) return true;
|
|
393
|
+
const row = costEstimate.pairs.find(e => e.pair === f.pair);
|
|
394
|
+
return (row?.keys || 0) > 0 || (contentByTarget[f.target]?.billedChars || 0) > 0;
|
|
395
|
+
};
|
|
396
|
+
const needed = deferred.filter(f => sends(f));
|
|
397
|
+
// One line per server (method + where), naming its pairs.
|
|
398
|
+
const byServer = new Map();
|
|
399
|
+
for (const f of deferred.filter(x => !needed.includes(x))) {
|
|
400
|
+
const method = f.pairConfig.method || 'llm';
|
|
401
|
+
const where = f.endpoint ? `no server answers at ${f.endpoint}` : String(f.reason).replace(/[.;\s]+$/, '');
|
|
402
|
+
const id = `${method}\x00${where}`;
|
|
403
|
+
if (!byServer.has(id)) byServer.set(id, { method, where, pairs: [], cached: 0, held: 0 });
|
|
404
|
+
const g = byServer.get(id);
|
|
405
|
+
g.pairs.push(f.pairKey);
|
|
406
|
+
const row = costEstimate?.pairs.find(e => e.pair === f.pair);
|
|
407
|
+
g.cached += row?.tmHits || 0;
|
|
408
|
+
g.held += row?.held || 0;
|
|
409
|
+
}
|
|
410
|
+
for (const g of byServer.values()) {
|
|
411
|
+
const what = g.cached + g.held > 0
|
|
412
|
+
? `every key queued for ${g.pairs.length > 1 ? 'them' : 'it'} comes from the cache (${g.cached})${g.held ? ` or is held back (${g.held})` : ''}`
|
|
413
|
+
: 'nothing is queued';
|
|
414
|
+
output.warn(`${g.pairs.join(', ')} (method: ${g.method}): ${g.where} — this run does not need it: ${what}, `
|
|
415
|
+
+ 'so it goes on. Start the server before a run that translates.');
|
|
416
|
+
}
|
|
417
|
+
return stopForPreflight(needed, cliArgs);
|
|
222
418
|
}
|
|
223
419
|
|
|
224
420
|
/**
|
|
@@ -236,13 +432,1503 @@ async function resolveRuntime(config, cwd, cliArgs = {}) {
|
|
|
236
432
|
* folding them into totalProcessed would overstate the translation work done.
|
|
237
433
|
* @returns {{ ok: boolean, message: string }} ok=false → caller logs as a warning
|
|
238
434
|
*/
|
|
239
|
-
function formatSyncSummary(dryRun, totalProcessed, totalFailed, totalCopied = 0) {
|
|
435
|
+
function formatSyncSummary(dryRun, totalProcessed, totalFailed, totalCopied = 0, work = null, totalHeld = 0, { contentPending = false, pluralGaps = 0, pluralRepair = null, pluralMarked = false } = {}) {
|
|
240
436
|
const verb = dryRun ? 'Would have processed' : 'Synced';
|
|
241
437
|
const copied = totalCopied > 0 ? ` (+${totalCopied} copied verbatim, no-translate)` : '';
|
|
242
|
-
|
|
243
|
-
|
|
438
|
+
// "2 key(s) sent to the model, 10 served from the cache (free)" — what was
|
|
439
|
+
// asked for (and billed, by a paid method) versus reused.
|
|
440
|
+
let split = '';
|
|
441
|
+
if (work && work.known && totalProcessed > 0) {
|
|
442
|
+
const retried = work.retried > 0 ? ` (+${work.retried} re-sent with the quality gate's feedback)` : '';
|
|
443
|
+
split = dryRun
|
|
444
|
+
? ` — ${work.sent} would be sent to the model, ${work.cached} served from the cache (free)`
|
|
445
|
+
: ` — ${work.sent} key(s) sent to the model${retried}, ${work.cached} served from the cache (free)`;
|
|
446
|
+
}
|
|
447
|
+
// A plural message written without a form the language uses for ordinary
|
|
448
|
+
// counts (Russian `few`, filled with the "other" form) is not a finished
|
|
449
|
+
// translation: never an [OK] line over it, and the run exits 2 (Round 8,
|
|
450
|
+
// Django persona: exit 0 over two marked gaps).
|
|
451
|
+
if (totalFailed > 0 || totalHeld > 0 || pluralGaps > 0) {
|
|
452
|
+
const bad = [
|
|
453
|
+
totalFailed > 0 && `${totalFailed} failed`,
|
|
454
|
+
totalHeld > 0 && `${totalHeld} held back (refused before; not sent, not billed)`,
|
|
455
|
+
pluralGaps > 0 && `${pluralGaps} plural message(s) lack a form the language uses for ordinary counts (the "other" form stands in — listed above)`,
|
|
456
|
+
].filter(Boolean).join(', ');
|
|
457
|
+
const where = totalFailed > 0 || totalHeld > 0 ? ' (see summary below)' : '';
|
|
458
|
+
// A re-sync with nothing to translate still says what it cost — nothing
|
|
459
|
+
// — and why it exits 2 (Round 10, Django persona: the "nothing billed"
|
|
460
|
+
// line disappeared whenever a marked plural gap kept the run at exit 2).
|
|
461
|
+
let after = '';
|
|
462
|
+
if (!dryRun && totalProcessed === 0) {
|
|
463
|
+
after = ' Nothing was sent to a model and nothing was billed.';
|
|
464
|
+
if (pluralGaps > 0 && totalFailed === 0 && totalHeld === 0) {
|
|
465
|
+
after += ` The exit code is 2 because of ${pluralGaps === 1 ? 'that plural message' : 'those plural messages'}: `
|
|
466
|
+
+ `write the missing forms by hand${pluralMarked ? ' (and delete each "# champollion:" line)' : ''}, or ask again`
|
|
467
|
+
+ `${pluralRepair ? `: \`${pluralRepair}\`` : ' with the command listed above for each file'}.`;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return { ok: false, message: `${verb} ${totalProcessed} key(s)${copied}${split}; ${bad}${where}.${after}` };
|
|
471
|
+
}
|
|
472
|
+
// Nothing queued: say what that costs, which is the point of a re-sync —
|
|
473
|
+
// a dry run too (Round 8: it ended on a bare "Would have processed 0 keys
|
|
474
|
+
// total"). With content files still to translate, only the keys are free.
|
|
475
|
+
if (totalProcessed === 0) {
|
|
476
|
+
if (contentPending) {
|
|
477
|
+
return {
|
|
478
|
+
ok: true,
|
|
479
|
+
message: `${dryRun ? 'Would have processed' : 'Synced'} 0 keys total${copied} — every key ${dryRun ? 'is' : 'was'} already up to date `
|
|
480
|
+
+ `(no key ${dryRun ? 'would be' : 'was'} sent); the content files ${dryRun ? 'are previewed' : 'follow'} below.`,
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
return dryRun
|
|
484
|
+
? { ok: true, message: `Would have processed 0 keys total${copied} — $0: nothing would be sent or billed.` }
|
|
485
|
+
: { ok: true, message: `Synced 0 keys total${copied} — everything was already up to date; no model calls, nothing billed.` };
|
|
486
|
+
}
|
|
487
|
+
return { ok: true, message: `${verb} ${totalProcessed} keys total${copied}${split}.` };
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* The price of sending some keys to several pairs, in the words of
|
|
492
|
+
* lib/cost-label.js: "$0 API cost (runs on this machine)" when every pair
|
|
493
|
+
* runs on this machine, else the sum of the known prices ("est. ~$0.0022"),
|
|
494
|
+
* or "cost unknown" when any pair has no published price.
|
|
495
|
+
*
|
|
496
|
+
* @param {Array<[number, object]>} sends - [key count, pair config]
|
|
497
|
+
* @param {{ cwd?: string }} [context]
|
|
498
|
+
* @returns {Promise<string>}
|
|
499
|
+
*/
|
|
500
|
+
async function priceOfSends(sends, context = {}) {
|
|
501
|
+
const estimates = [];
|
|
502
|
+
for (const [n, pc] of sends) {
|
|
503
|
+
if (n <= 0) continue;
|
|
504
|
+
try { estimates.push(await estimateCost(n, pc, context)); } catch { estimates.push(null); }
|
|
505
|
+
}
|
|
506
|
+
if (estimates.length === 0) return costLabel({ estimatedCost: 0 });
|
|
507
|
+
if (estimates.every(e => e?.local && e.estimatedCost === 0)) return costLabel(estimates[0]);
|
|
508
|
+
if (estimates.some(e => !e || typeof e.estimatedCost !== 'number')) return costLabel({ estimatedCost: null });
|
|
509
|
+
return costLabel({ estimatedCost: estimates.reduce((sum, e) => sum + e.estimatedCost, 0) });
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/** "a", "b" and "c" — a short human list. */
|
|
513
|
+
function listWords(words) {
|
|
514
|
+
const quoted = words.map(w => `"${w}"`);
|
|
515
|
+
return quoted.length <= 1 ? quoted.join('') : `${quoted.slice(0, -1).join(', ')} and ${quoted[quoted.length - 1]}`;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** "llm (model x)" — the method a refusal is remembered for, in words. */
|
|
519
|
+
/** Does this pair's method read per-key instructions (lib/methods/base.js)? */
|
|
520
|
+
function methodTakesInstructions(pairConfig) {
|
|
521
|
+
try { return getMethod(pairConfig.method || 'llm', pairConfig).acceptsKeyInstructions === true; } catch { return false; }
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* Say which plural messages lack forms the target language has — never a
|
|
526
|
+
* silent fill (Round 3, Django persona: Russian few/many were written from
|
|
527
|
+
* `other` and verify said "All checks passed").
|
|
528
|
+
*
|
|
529
|
+
* everyday categories (Russian few for 2, 3, 4): a warning per file,
|
|
530
|
+
* naming the keys, what the file now holds instead, and the command
|
|
531
|
+
* that asks again (--fresh: the cache holds the incomplete answer).
|
|
532
|
+
* rare categories (French many: 1 000 000): one info line.
|
|
533
|
+
*
|
|
534
|
+
* @param {object} p
|
|
535
|
+
* @param {Object<string, { everyday: string[], rare: string[], type: string }>} p.gaps
|
|
536
|
+
*/
|
|
537
|
+
function reportPluralGaps({ gaps, filename, format, pairKey, pairConfig, code, layout, ns }) {
|
|
538
|
+
const entries = Object.entries(gaps || {});
|
|
539
|
+
if (entries.length === 0) return;
|
|
540
|
+
const name = pairConfig.name || code;
|
|
541
|
+
const everyday = entries.filter(([, g]) => g.everyday.length > 0);
|
|
542
|
+
if (everyday.length > 0) {
|
|
543
|
+
const cats = [...new Set(everyday.flatMap(([, g]) => g.everyday))];
|
|
544
|
+
const type = everyday[0][1].type;
|
|
545
|
+
const shown = everyday.slice(0, 5).map(([k, g]) => `"${k}" (${g.everyday.join(', ')})`).join(', ');
|
|
546
|
+
const more = everyday.length > 5 ? `, +${everyday.length - 5} more` : '';
|
|
547
|
+
const instead = format === 'po'
|
|
548
|
+
? `those msgstr[] repeat the "other" form (marked with a "# champollion:" comment in ${filename})`
|
|
549
|
+
: 'the app will show the "other" form for those counts';
|
|
550
|
+
const asked = methodTakesInstructions(pairConfig)
|
|
551
|
+
? 'The model was asked for every form and left these out.'
|
|
552
|
+
: `${pairConfig.method} cannot be told which plural forms to write.`;
|
|
553
|
+
const fix = redoCommand(everyday.map(([k]) => k), { pair: pairKey, ns: layout.namespaced ? ns : '', fresh: true });
|
|
554
|
+
output.warn(`${filename}: ${everyday.length} plural message(s) have ${describePluralGap(code, cats, type, name, 'everyday')}`
|
|
555
|
+
+ `: ${shown}${more} — ${instead}. ${asked} `
|
|
556
|
+
+ `Write them by hand, or ask again (a stronger --model helps): \`${fix}\``);
|
|
557
|
+
}
|
|
558
|
+
const rare = entries.filter(([, g]) => g.everyday.length === 0 && g.rare.length > 0);
|
|
559
|
+
if (rare.length > 0) {
|
|
560
|
+
const cats = [...new Set(rare.flatMap(([, g]) => g.rare))];
|
|
561
|
+
const type = rare[0][1].type;
|
|
562
|
+
output.info(`${filename}: ${rare.length} plural message(s) have ${describePluralGap(code, cats, type, name, 'rare')}`
|
|
563
|
+
+ `${format === 'po' ? ' (marked in the catalog)' : ''}.`);
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Plural messages in the run's target files that still lack a form the
|
|
569
|
+
* language uses for ordinary counts — read from disk once the run has
|
|
570
|
+
* written. A gap the model left is held to the same rule as a held refusal:
|
|
571
|
+
* every sync exits 2 while it is there, not only the sync that wrote it
|
|
572
|
+
* (Round 9, Django persona: the first sync exited 2, a re-sync with the same
|
|
573
|
+
* marked gap on disk exited 0). The same entries `audit` counts and
|
|
574
|
+
* `verify --strict` fails on (lib/verify.js pluralGapsInFile).
|
|
575
|
+
*
|
|
576
|
+
* @returns {Map<string, Array<{ unit: object, file: object, gaps: Array<{ key: string, missing: string[], marked: boolean }> }>>}
|
|
577
|
+
* pair key → its files with gaps
|
|
578
|
+
*/
|
|
579
|
+
function pluralGapsOnDisk({ layout, units, inputLocale, pairEntries }) {
|
|
580
|
+
const out = new Map();
|
|
581
|
+
for (const [pairKey, pc] of pairEntries) {
|
|
582
|
+
const code = pc.target;
|
|
583
|
+
const files = [];
|
|
584
|
+
for (const unit of units) {
|
|
585
|
+
let file;
|
|
586
|
+
try { file = layout.fileFor(code, unit.ns); } catch { continue; }
|
|
587
|
+
if (!file || !fs.existsSync(file.path)) continue;
|
|
588
|
+
let targetFlat;
|
|
589
|
+
try { targetFlat = readLocaleFlat(file) || {}; } catch { continue; }
|
|
590
|
+
let expected;
|
|
591
|
+
try { expected = expectedForTarget(unit, inputLocale, code).flat; } catch { continue; }
|
|
592
|
+
const gaps = pluralGapsInFile({ file, expected, targetFlat, locale: code });
|
|
593
|
+
if (gaps.length > 0) files.push({ unit, file, gaps });
|
|
594
|
+
}
|
|
595
|
+
if (files.length > 0) out.set(pairKey, files);
|
|
596
|
+
}
|
|
597
|
+
return out;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Say the plural gaps a run found on disk that it did not write itself (an
|
|
602
|
+
* earlier sync did): per file, the keys, why the run is incomplete, and the
|
|
603
|
+
* repair — never a silent exit 2.
|
|
604
|
+
*/
|
|
605
|
+
function reportPluralGapsOnDisk({ pairKey, code, file, unit, gaps, layout, dryRun }) {
|
|
606
|
+
const shown = gaps.slice(0, 5).map(g => `"${String(g.key).replace(/\u0004/g, '\u2404')}" (${g.missing.join(', ')})`).join(', ')
|
|
607
|
+
+ (gaps.length > 5 ? `, +${gaps.length - 5} more` : '');
|
|
608
|
+
const marked = gaps.some(g => g.marked);
|
|
609
|
+
const fix = redoCommand(gaps.map(g => g.key), { pair: pairKey, ns: layout.namespaced ? unit.ns : '', fresh: true });
|
|
610
|
+
output.warn(`${file.rel}: ${gaps.length} plural message(s) still lack a form ${code} uses for ordinary counts — `
|
|
611
|
+
+ `the "other" form stands in${marked ? ` (marked "# champollion:" in ${file.rel})` : ''}: ${shown}. `
|
|
612
|
+
+ `${dryRun ? 'A real sync exits 2' : 'Every sync exits 2'} while they remain, as for keys held back. `
|
|
613
|
+
+ `Write them by hand${marked ? ' and delete the "# champollion:" line above each entry' : ''}, `
|
|
614
|
+
+ `or ask again (a stronger --model helps): \`${fix}\`. A sync that runs another method or model asks for them again `
|
|
615
|
+
+ `by itself; \`champollion sync --pair ${shellWord(pairKey)} --redo gaps\` asks for every such message (add --model for a stronger one).`);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
// -----------------------------------------------------------------
|
|
619
|
+
// Keys named for a redo (--redo keys: / --force-keys) that match nothing
|
|
620
|
+
// (the rule itself: lib/named-keys.js, shared with the Docusaurus path)
|
|
621
|
+
// -----------------------------------------------------------------
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* This lane's key space for the named-key rule: every source key, and every
|
|
625
|
+
* plural form a target gets (French `count_many`), namespaced as the lock
|
|
626
|
+
* names them in a layout with several files.
|
|
627
|
+
*
|
|
628
|
+
* @returns {Set<string>}
|
|
629
|
+
*/
|
|
630
|
+
function namedKeySpace({ layout, units, inputLocale, targets }) {
|
|
631
|
+
const known = new Set();
|
|
632
|
+
for (const unit of units) {
|
|
633
|
+
const keys = new Set(Object.keys(unit.flat));
|
|
634
|
+
for (const code of targets) {
|
|
635
|
+
try { for (const k of Object.keys(expectedForTarget(unit, inputLocale, code).flat)) keys.add(k); } catch { /* unknown locale */ }
|
|
636
|
+
}
|
|
637
|
+
for (const k of keys) known.add(layout.namespaced ? `${unit.ns}${NS_SEPARATOR}${k}` : k);
|
|
638
|
+
}
|
|
639
|
+
return known;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
/** A named key without its file's namespace ("common::nav.home" → "nav.home"). */
|
|
643
|
+
function bareNamedKey(layout, k) {
|
|
644
|
+
return layout.namespaced && k.includes(NS_SEPARATOR) ? k.slice(k.indexOf(NS_SEPARATOR) + NS_SEPARATOR.length) : k;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/** A pair config (and its fallback) carrying a gettext catalog's plural slots for the gate. */
|
|
648
|
+
function withPluralSlots(pairConfig, pluralSlots) {
|
|
649
|
+
return {
|
|
650
|
+
...pairConfig,
|
|
651
|
+
pluralSlots,
|
|
652
|
+
...(pairConfig.fallback && { fallback: { ...pairConfig.fallback, pluralSlots } }),
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* i18next plural categories the SOURCE does not have (French `_many` from
|
|
658
|
+
* English, which has only one/other): each is translated from the source's
|
|
659
|
+
* `_other` text. Say so, and whether the method could be told which form to
|
|
660
|
+
* write — an LLM is asked for it; a machine translation engine just
|
|
661
|
+
* translates the `_other` text again (Round 3, i18next persona).
|
|
662
|
+
*/
|
|
663
|
+
function reportGeneratedPluralKeys({ keys, expansion, filename, pairConfig, code, sourceLocale }) {
|
|
664
|
+
// Generated = translated from a DIFFERENT source key (French count_many
|
|
665
|
+
// from count_other). gettext/ARB notes ride an identity expansion with
|
|
666
|
+
// no origin map, so they never count.
|
|
667
|
+
const generated = keys.filter(k => expansion?.origin?.[k] && expansion.origin[k] !== k);
|
|
668
|
+
if (generated.length === 0) return;
|
|
669
|
+
const cats = [...new Set(generated.map(k => k.slice(k.lastIndexOf('_') + 1)))];
|
|
670
|
+
const shown = generated.slice(0, 4).join(', ') + (generated.length > 4 ? `, +${generated.length - 4} more` : '');
|
|
671
|
+
const name = pairConfig.name || code;
|
|
672
|
+
if (methodTakesInstructions(pairConfig)) {
|
|
673
|
+
output.info(`${filename}: ${shown} — ${listWords(cats)} ${cats.length === 1 ? 'is a' : 'are'} ${name} plural form(s) `
|
|
674
|
+
+ `the ${sourceLocale} source has no key for; each is translated from the "_other" text, and the model is asked to write that form `
|
|
675
|
+
+ `(${describeCategories(code, cats, 'cardinal')}).`);
|
|
676
|
+
} else {
|
|
677
|
+
output.warn(`${filename}: ${shown} — ${listWords(cats)} ${cats.length === 1 ? 'is a' : 'are'} ${name} plural form(s) `
|
|
678
|
+
+ `the ${sourceLocale} source has no key for, translated from the "_other" text by ${pairConfig.method}, which cannot be told `
|
|
679
|
+
+ 'which plural form to write: they hold the "other" form. Review them, or use an LLM method for this pair.');
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
/**
|
|
684
|
+
* The per-key notes the method is given beside the source text: gettext
|
|
685
|
+
* msgctxt and `#.` comments, ARB descriptions, the plural form a generated
|
|
686
|
+
* i18next key needs (lib/locale-layout.js expectedForTarget). ONE helper for
|
|
687
|
+
* the real call and for `--show-prompt`.
|
|
688
|
+
*
|
|
689
|
+
* @param {string[]} keys
|
|
690
|
+
* @param {object|null} expansion
|
|
691
|
+
* @returns {object}
|
|
692
|
+
*/
|
|
693
|
+
function promptNotesFor(keys, expansion) {
|
|
694
|
+
const notes = {};
|
|
695
|
+
if (!expansion?.descriptions) return notes;
|
|
696
|
+
for (const k of keys) {
|
|
697
|
+
if (expansion.descriptions[k]) notes[k] = expansion.descriptions[k];
|
|
698
|
+
}
|
|
699
|
+
return notes;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/** A captured request, readable: the messages as text, the rest as JSON. */
|
|
703
|
+
function renderRequest(request) {
|
|
704
|
+
const lines = [` ${request.method} ${request.url}`, ` headers: ${JSON.stringify(request.headers)}`];
|
|
705
|
+
const body = request.body;
|
|
706
|
+
if (body && typeof body === 'object' && Array.isArray(body.messages)) {
|
|
707
|
+
const { messages, system, ...rest } = body;
|
|
708
|
+
lines.push(` ${JSON.stringify(rest)}`);
|
|
709
|
+
const blocks = [];
|
|
710
|
+
if (typeof system === 'string') blocks.push(['system', system]);
|
|
711
|
+
for (const m of messages) {
|
|
712
|
+
const text = typeof m.content === 'string' ? m.content : JSON.stringify(m.content, null, 2);
|
|
713
|
+
blocks.push([m.role, text]);
|
|
714
|
+
}
|
|
715
|
+
for (const [role, text] of blocks) {
|
|
716
|
+
lines.push(` ── ${role} ──`);
|
|
717
|
+
for (const l of String(text).split('\n')) lines.push(` ${l}`);
|
|
718
|
+
}
|
|
719
|
+
} else {
|
|
720
|
+
for (const l of JSON.stringify(body, null, 2).split('\n')) lines.push(` ${l}`);
|
|
721
|
+
}
|
|
722
|
+
return lines;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* `sync --dry --show-prompt [key]` for one file of one locale: build the
|
|
727
|
+
* request through the method's own code (lib/translate-pair.js
|
|
728
|
+
* previewRequests) and print it. Nothing is sent; secrets are redacted.
|
|
729
|
+
* Without a key it shows the first request this file would send (a cache
|
|
730
|
+
* miss); with a key, the request for that key alone, whether or not it is
|
|
731
|
+
* queued — so `verb␄Open` shows whether its msgctxt reaches the model.
|
|
732
|
+
*/
|
|
733
|
+
async function showRequestPreview({ cliArgs, layout, unit, pairKey, pairConfig, code, filename, sourceFlat, expansion, apiKey, tm, queued, held = new Set(), noteShown = null, cwd = null }) {
|
|
734
|
+
const named = typeof cliArgs['show-prompt'] === 'string' ? cliArgs['show-prompt'] : null;
|
|
735
|
+
let keys;
|
|
736
|
+
let why;
|
|
737
|
+
if (named) {
|
|
738
|
+
// The whole value as one key first: a gettext msgid is a sentence, and
|
|
739
|
+
// its comma is not a list separator ("Welcome back, %(name)s!").
|
|
740
|
+
const asOne = keysForNamespace(layout, [named], unit.ns).filter(k => typeof sourceFlat[k] === 'string');
|
|
741
|
+
keys = asOne.length > 0 ? asOne
|
|
742
|
+
: keysForNamespace(layout, splitKeyList(named), unit.ns).filter(k => typeof sourceFlat[k] === 'string');
|
|
743
|
+
if (keys.length === 0) return;
|
|
744
|
+
why = keys.length === 1 ? 'the key you named' : 'the keys you named';
|
|
745
|
+
// A named key a real run would not send: shown anyway (that is what was
|
|
746
|
+
// asked for), and said plainly — the preview printed nothing at all for a
|
|
747
|
+
// key already translated (Round 6, Django persona).
|
|
748
|
+
const queuedSet = new Set(queued);
|
|
749
|
+
const tmKey = tmMethodKey(pairConfig);
|
|
750
|
+
const notSent = [];
|
|
751
|
+
for (const k of keys) {
|
|
752
|
+
let reason = null;
|
|
753
|
+
if (held.has(k)) reason = 'held back (the quality gate refused this method\'s translation of its current text before)';
|
|
754
|
+
else if (!queuedSet.has(k)) reason = 'up to date (translated, and its source has not changed)';
|
|
755
|
+
else if (peekTM(tm, tmTextFor(k, sourceFlat[k], expansion), code, tmKey) !== null) reason = 'served from the cache';
|
|
756
|
+
if (reason) notSent.push([k, reason]);
|
|
757
|
+
}
|
|
758
|
+
if (notSent.length > 0) {
|
|
759
|
+
const ns = layout.namespaced ? unit.ns : '';
|
|
760
|
+
output.info(`${pairKey} ${filename}: a real run would send nothing for ${notSent.map(([k, r]) => `${JSON.stringify(k.replace(/\u0004/g, '\u2404'))} — ${r}`).join('; ')}. `
|
|
761
|
+
+ 'The request below is what re-translating it would send; '
|
|
762
|
+
+ `\`${redoCommand(notSent.map(([k]) => k), { pair: pairKey, ns, fresh: true })}\` sends it. Nothing is sent now.`);
|
|
763
|
+
}
|
|
764
|
+
} else {
|
|
765
|
+
// What this file would send: queued, not held back, not in the cache.
|
|
766
|
+
const tmKey = tmMethodKey(pairConfig);
|
|
767
|
+
const texts = {};
|
|
768
|
+
for (const k of queued) texts[k] = tmTextFor(k, sourceFlat[k], expansion);
|
|
769
|
+
const misses = queued.filter(k => lookupTM(tm, texts[k], code, tmKey) === null);
|
|
770
|
+
const batch = pairConfig.batchSize || DEFAULT_BATCH_SIZE;
|
|
771
|
+
keys = misses.slice(0, batch);
|
|
772
|
+
if (keys.length === 0) return;
|
|
773
|
+
why = misses.length > batch
|
|
774
|
+
? `the first ${batch} of the ${misses.length} key(s) this file would send`
|
|
775
|
+
: `the ${misses.length} key(s) this file would send`;
|
|
776
|
+
}
|
|
777
|
+
const { supported, requests } = await previewRequests(keys, sourceFlat, pairConfig, {
|
|
778
|
+
apiKey, targetCode: code, descriptions: promptNotesFor(keys, expansion), cwd,
|
|
779
|
+
});
|
|
780
|
+
if (!supported) {
|
|
781
|
+
output.info(`${pairKey} ${filename}: no request preview for ${pairConfig.method} — it is sent the source text of each key `
|
|
782
|
+
+ 'and nothing else (no per-key notes or context reach it).');
|
|
783
|
+
return;
|
|
244
784
|
}
|
|
245
|
-
|
|
785
|
+
if (noteShown) noteShown();
|
|
786
|
+
for (const request of requests) {
|
|
787
|
+
output.event('request', { pair: pairKey, file: filename, keys, request });
|
|
788
|
+
}
|
|
789
|
+
const lines = ['', ` Request preview — ${pairKey}, ${filename}, ${why} (${keys.length}): what ${pairConfig.method} would be sent. Not sent; secrets redacted.`];
|
|
790
|
+
if (requests.length === 0) lines.push(' (the method built no request for these keys)');
|
|
791
|
+
for (const request of requests) lines.push(...renderRequest(request));
|
|
792
|
+
lines.push('');
|
|
793
|
+
output.raw(lines.join('\n'));
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* Sync ONE target file: diff it against the source file it mirrors,
|
|
798
|
+
* translate what is pending, write it back. A flat locale is one call; a
|
|
799
|
+
* folder-per-locale project makes one call per namespace file.
|
|
800
|
+
*
|
|
801
|
+
* Returns the per-file tallies runSync aggregates per locale. `failedKeys`
|
|
802
|
+
* are in the shared lock key space (namespaced for multi-file layouts) so
|
|
803
|
+
* the manifest restore stays exact. `backendDown` tells the caller the
|
|
804
|
+
* method returned nothing at all, so the locale's remaining files are not
|
|
805
|
+
* attempted against a dead backend.
|
|
806
|
+
*
|
|
807
|
+
* @param {object} ctx - Run context: { config, layout, inputLocale, dryRun,
|
|
808
|
+
* cliArgs, apiKey, tm, noTranslate, pluralFallbackReported }
|
|
809
|
+
* @param {import('./locale-layout.js').SourceUnit} unit - Source file
|
|
810
|
+
* @param {string} pairKey
|
|
811
|
+
* @param {object} pairConfig
|
|
812
|
+
* @param {{ backendDown?: boolean, fallbackDown?: boolean }} [state] - An
|
|
813
|
+
* earlier file of this locale got nothing from the pair's method
|
|
814
|
+
* (backendDown) or from its fallback (fallbackDown). With a working
|
|
815
|
+
* fallback, a file after a dead primary goes straight to the fallback.
|
|
816
|
+
* @returns {Promise<object>}
|
|
817
|
+
*/
|
|
818
|
+
async function syncLocaleFile(ctx, unit, pairKey, pairConfig, { backendDown = false, fallbackDown = false } = {}) {
|
|
819
|
+
const { config, layout, inputLocale, dryRun, cliArgs, apiKey, tm, noTranslate } = ctx;
|
|
820
|
+
const code = pairConfig.target;
|
|
821
|
+
const file = layout.fileFor(code, unit.ns);
|
|
822
|
+
const filename = file.rel;
|
|
823
|
+
const filePath = file.path;
|
|
824
|
+
const format = file.format;
|
|
825
|
+
|
|
826
|
+
// What THIS target must contain for this file: the source map, or — for
|
|
827
|
+
// a file with i18next plural keys — the target's own CLDR plural forms,
|
|
828
|
+
// each mapped back to the source key it is translated from.
|
|
829
|
+
const { flat: sourceFlat, expansion } = expectedForTarget(unit, inputLocale, code);
|
|
830
|
+
if (expansion?.unknownLocale && !ctx.pluralFallbackReported.has(code)) {
|
|
831
|
+
ctx.pluralFallbackReported.add(code);
|
|
832
|
+
output.info(`${code}: CLDR has no plural rules for this locale — keeping the source's plural forms.`);
|
|
833
|
+
}
|
|
834
|
+
// Key in the shared space (lock manifest, dry-run lists). Failed keys map
|
|
835
|
+
// back to the SOURCE key they came from, so a failed generated `_many`
|
|
836
|
+
// keeps the old hash of the `_other` it is translated from.
|
|
837
|
+
const toLock = (key) => lockKey(layout, unit.ns, originKey(key, expansion));
|
|
838
|
+
// Display form of a target key (dry-run lists): namespaced, NOT mapped to
|
|
839
|
+
// its origin — the list names the keys this file will actually receive.
|
|
840
|
+
const nsKey = (key) => lockKey(layout, unit.ns, key);
|
|
841
|
+
|
|
842
|
+
// Security: verify the resolved write path is still within localesDir.
|
|
843
|
+
// Prevents path traversal via crafted language codes like "../../../etc/passwd".
|
|
844
|
+
// A refusal means NO file was written — count it as a failure so the run
|
|
845
|
+
// reports it and exits non-zero, instead of printing [OK] and exiting 0.
|
|
846
|
+
if (!isPathContained(filePath, layout.baseDir)) {
|
|
847
|
+
output.error(`${filename} — refusing to write outside locales directory`);
|
|
848
|
+
// Nothing ran for this locale, so every changed key is unresolved here.
|
|
849
|
+
// Only changed keys matter for manifest retry-safety: missing keys
|
|
850
|
+
// re-fire via missing-key detection regardless of the manifest.
|
|
851
|
+
return { processed: 0, tmHits: 0, failed: 1, failedKeys: unit.changedKeys.map(k => lockKey(layout, unit.ns, k)), pairKey };
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
// If locale file doesn't exist yet, create it as empty
|
|
855
|
+
let data = {};
|
|
856
|
+
const existed = fs.existsSync(filePath);
|
|
857
|
+
if (existed) {
|
|
858
|
+
data = readLocaleData(file);
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
// For JSON, flatten the nested structure. TOML/YAML is already flat.
|
|
862
|
+
const targetFlat = format === 'json' ? flattenKeys(data) : { ...data };
|
|
863
|
+
// Source-echo requeue suppression: a target value equal to its source is
|
|
864
|
+
// only requeued when the TM does NOT confirm the echo came from the
|
|
865
|
+
// pipeline. lookupTM === sourceValue means a previous run translated this
|
|
866
|
+
// exact text to itself and the gate approved it — skip, don't re-bill.
|
|
867
|
+
// With --no-tm the TM is empty, so nothing is suppressed.
|
|
868
|
+
// The TM is consulted under the text the pipeline cached the key under —
|
|
869
|
+
// a gettext entry with a msgctxt folds its context in (lib/tm-evict.js),
|
|
870
|
+
// so "Open" the verb is confirmed by the verb's entry, not the adjective's.
|
|
871
|
+
// With a fallback, a value may be cached under the fallback's key: the
|
|
872
|
+
// confirmations below consult both (lib/fallback.js tmKeysForPair).
|
|
873
|
+
const tmKeys = tmKeysForPair(pairConfig);
|
|
874
|
+
// A borrowed i18next plural form (French `_many` from `_other`) has its
|
|
875
|
+
// own entry (lib/tm-evict.js tmTextFor).
|
|
876
|
+
const cachedAs = (key, sourceValue) => tmTextFor(key, sourceValue, expansion);
|
|
877
|
+
|
|
878
|
+
// Plural forms this locale does not use (Japanese has no `_one`) that an
|
|
879
|
+
// earlier, plural-unaware sync wrote. Removed ONLY when the TM proves the
|
|
880
|
+
// pipeline produced the value on disk — a hand-written value is kept and
|
|
881
|
+
// reported as an extra key, never deleted.
|
|
882
|
+
const removedPlurals = [];
|
|
883
|
+
if (expansion && expansion.unused.length > 0 && format === 'json') {
|
|
884
|
+
for (const key of expansion.unused) {
|
|
885
|
+
if (!Object.prototype.hasOwnProperty.call(targetFlat, key)) continue;
|
|
886
|
+
const src = unit.flat[key];
|
|
887
|
+
if (typeof src === 'string' && tmHoldsValue(tm, cachedAs(key, src), code, tmKeys, targetFlat[key])) {
|
|
888
|
+
removedPlurals.push(key);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
for (const key of removedPlurals) {
|
|
892
|
+
delete targetFlat[key];
|
|
893
|
+
if (!dryRun) deleteNestedValue(data, key);
|
|
894
|
+
}
|
|
895
|
+
if (removedPlurals.length > 0) {
|
|
896
|
+
output.info(`${filename} — ${dryRun ? 'would remove' : 'removed'} ${removedPlurals.length} plural form(s) ${code} does not use (written by an earlier sync): ${removedPlurals.join(', ')}`);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// --prune plural-extras: i18next keys for a plural form this language does
|
|
901
|
+
// not have (Spanish `count_two`), whoever wrote them — the keys `verify`
|
|
902
|
+
// names, and only those (lib/plurals.js pluralExtraKeys). Never without
|
|
903
|
+
// the flag: a value may be a person's (Round 13, i18next persona).
|
|
904
|
+
const prunedPlurals = [];
|
|
905
|
+
if (ctx.prune?.has('plural-extras')) {
|
|
906
|
+
const extras = pluralExtraKeys(unit, targetFlat, code);
|
|
907
|
+
for (const { key } of extras) {
|
|
908
|
+
prunedPlurals.push(key);
|
|
909
|
+
delete targetFlat[key];
|
|
910
|
+
if (!dryRun) {
|
|
911
|
+
if (format === 'json') deleteNestedValue(data, key);
|
|
912
|
+
else delete data[key];
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
if (extras.length > 0) {
|
|
916
|
+
ctx.pruned.push(...extras.map(e => ({ locale: code, file: filename, key: nsKey(e.key), category: e.category })));
|
|
917
|
+
output.info(`${filename} — ${dryRun ? 'would remove' : 'removed'} ${extras.length} plural key(s) for a form ${code} does not have `
|
|
918
|
+
+ `(CLDR ${code}: ${extras[0].cats.join(', ')}): ${extras.map(e => nsKey(e.key)).join(', ')} — --prune plural-extras removes these and nothing else.`);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
const removedAny = removedPlurals.length > 0 || prunedPlurals.length > 0;
|
|
922
|
+
|
|
923
|
+
// Forced / changed keys arrive in the unit's source key space; plural
|
|
924
|
+
// expansion maps them onto the target keys translated from them.
|
|
925
|
+
//
|
|
926
|
+
// PENDING keys (a redo that could not finish, lib/locale-state.js) are
|
|
927
|
+
// queued again as forced keys; they are already in the target key space.
|
|
928
|
+
const localeState = ctx.lockState.of(code);
|
|
929
|
+
const pendingKeys = new Set(
|
|
930
|
+
keysForNamespace(layout, Object.keys(localeState.pending), unit.ns)
|
|
931
|
+
.filter(k => Object.prototype.hasOwnProperty.call(sourceFlat, k) && typeof sourceFlat[k] === 'string'),
|
|
932
|
+
);
|
|
933
|
+
const namedKeys = new Set(mapSourceKeysToTarget(keysForNamespace(layout, ctx.namedKeys, unit.ns), expansion));
|
|
934
|
+
// Plural messages on disk a model left without a form the language uses
|
|
935
|
+
// (the "other" form standing in, marked in a catalog): asked again when
|
|
936
|
+
// this run's setup has not answered them yet, or under --redo gaps
|
|
937
|
+
// (lib/plural-gap-redo.js — the estimate makes the same plan).
|
|
938
|
+
const gapPlan = existed
|
|
939
|
+
? planGapRedo({ file, expected: sourceFlat, targetFlat, locale: code, localeState, lockKeyOf: nsKey, pairConfig, all: !!ctx.redoGaps })
|
|
940
|
+
: { keys: [], gaps: [], by: {}, missing: {} };
|
|
941
|
+
ctx.gapsSeen += gapPlan.gaps.length;
|
|
942
|
+
// A gap record for a message that has none on disk any more (its forms
|
|
943
|
+
// were written by hand): forgotten.
|
|
944
|
+
if (!dryRun && localeState.gaps) {
|
|
945
|
+
const open = new Set(gapPlan.gaps.map(g => nsKey(g.key)));
|
|
946
|
+
for (const k of keysForNamespace(layout, Object.keys(localeState.gaps), unit.ns)) {
|
|
947
|
+
if (!open.has(nsKey(k))) delete localeState.gaps[nsKey(k)];
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
const forceKeys = [...new Set([
|
|
951
|
+
...mapSourceKeysToTarget(keysForNamespace(layout, config.forceKeys, unit.ns), expansion),
|
|
952
|
+
...pendingKeys,
|
|
953
|
+
...gapPlan.keys,
|
|
954
|
+
])];
|
|
955
|
+
const changedKeys = mapSourceKeysToTarget(unit.changedKeys, expansion);
|
|
956
|
+
const diff = diffLocale(
|
|
957
|
+
sourceFlat, targetFlat, config.fallbackPrefix, forceKeys, changedKeys,
|
|
958
|
+
(key, sourceValue) => tmHoldsValue(tm, cachedAs(key, sourceValue), code, tmKeys, sourceValue),
|
|
959
|
+
noTranslate.active ? noTranslate.matches : null
|
|
960
|
+
);
|
|
961
|
+
|
|
962
|
+
// What the queue becomes once the per-locale record is consulted: hand
|
|
963
|
+
// edits a bulk redo keeps, keys refused before (held back from the method
|
|
964
|
+
// that refused them), pending keys retried from the model (shared with the
|
|
965
|
+
// cost estimate — lib/locale-state.js planQueue).
|
|
966
|
+
const plan = planQueue({
|
|
967
|
+
diff, sourceFlat, targetFlat, lockKeyOf: nsKey, localeState,
|
|
968
|
+
named: namedKeys, bulk: !!cliArgs.force, pending: pendingKeys, fresh: !!cliArgs['no-tm'], pairConfig,
|
|
969
|
+
classify: createEditClassifier({ tm, locale: code, written: localeState.written, expansion }),
|
|
970
|
+
fallbackPrefix: config.fallbackPrefix,
|
|
971
|
+
redoGaps: ctx.redoGaps ? new Set(gapPlan.keys) : null,
|
|
972
|
+
});
|
|
973
|
+
const queue = plan.toProcess;
|
|
974
|
+
const keptSet = new Set(plan.kept);
|
|
975
|
+
const heldSet = new Set(plan.held);
|
|
976
|
+
// The plural messages this run asks again: sent to the model, never served
|
|
977
|
+
// from the cache — it holds the incomplete answer (a later model's
|
|
978
|
+
// carry-over would serve it straight back).
|
|
979
|
+
const gapAsked = gapPlan.keys.filter(k => queue.includes(k) && !heldSet.has(k));
|
|
980
|
+
if (gapAsked.length > 0) {
|
|
981
|
+
if (!dryRun) bypassTMFor(tm, code, gapAsked.map(k => cachedAs(k, sourceFlat[k])));
|
|
982
|
+
const asked = ctx.gapsAsked.get(pairKey) || [];
|
|
983
|
+
asked.push(...gapAsked.map(nsKey));
|
|
984
|
+
ctx.gapsAsked.set(pairKey, asked);
|
|
985
|
+
const writers = [...new Set(gapAsked.map(k => gapPlan.by[k]).filter(Boolean))].map(describeMethodKey);
|
|
986
|
+
const forms = [...new Set(gapAsked.flatMap(k => gapPlan.missing[k] || []))];
|
|
987
|
+
const shownKeys = gapAsked.slice(0, 3).map(k => JSON.stringify(nsKey(k).replace(/\u0004/g, '\u2404'))).join(', ')
|
|
988
|
+
+ (gapAsked.length > 3 ? `, +${gapAsked.length - 3} more` : '');
|
|
989
|
+
const who = ctx.redoGaps
|
|
990
|
+
? '--redo gaps'
|
|
991
|
+
: `${writers.join('; ') || 'an earlier setup'} left ${gapAsked.length === 1 ? 'it' : 'them'} so, and ${describeMethodKey(tmMethodKey(pairConfig))} has not been asked`;
|
|
992
|
+
output.info(`${filename} — ${dryRun ? 'would ask' : 'asking'} the model again for ${gapAsked.length} plural message(s) without a form `
|
|
993
|
+
+ `${code} uses for ordinary counts (${forms.join(', ')}): ${shownKeys} (${who}). A marked gap is not a translation: it is sent `
|
|
994
|
+
+ `to the model, not served from the cache, which holds the incomplete answer${dryRun ? ' (the estimate above prices it)' : ''}. `
|
|
995
|
+
+ 'If the answer lacks the forms too, the gap stays marked.');
|
|
996
|
+
}
|
|
997
|
+
// Borrowed plural forms whose own cache entry does not exist yet while the
|
|
998
|
+
// entry they used to share does (a cache from before this release): each
|
|
999
|
+
// goes to the model once. Said, because the estimate prices it.
|
|
1000
|
+
if (expansion?.borrowed && queue.length > 0) {
|
|
1001
|
+
const pk = tmMethodKey(pairConfig);
|
|
1002
|
+
const firstOwn = queue.filter(k => expansion.borrowed[k] && !heldSet.has(k) && typeof sourceFlat[k] === 'string'
|
|
1003
|
+
&& peekTM(tm, cachedAs(k, sourceFlat[k]), code, pk) === null
|
|
1004
|
+
&& peekTM(tm, tmSourceText(k, sourceFlat[k]), code, pk) !== null);
|
|
1005
|
+
if (firstOwn.length > 0) {
|
|
1006
|
+
output.info(`${filename} — ${firstOwn.length} plural form(s) translated from another form's source text (${firstOwn.slice(0, 3).map(nsKey).join(', ')}`
|
|
1007
|
+
+ `${firstOwn.length > 3 ? ', …' : ''}) ${dryRun ? 'would be' : 'are'} sent to the model once: until this release each shared a cache entry with `
|
|
1008
|
+
+ 'the form it borrows from, which holds that form\'s translation, so it is not reused. From now on each form has its own entry.');
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
const redoFor = (keys, opts = {}) => redoCommand(keys, { pair: pairKey, ns: layout.namespaced ? unit.ns : '', ...opts });
|
|
1012
|
+
/**
|
|
1013
|
+
* Keys NAMED for a redo that the cache answered: the model was not asked,
|
|
1014
|
+
* and a redo without --fresh said nothing about it — the persona's
|
|
1015
|
+
* `--redo keys:<k>` wrote back the text it already had (Round 7, Django).
|
|
1016
|
+
* Said, with the --fresh command and what it costs.
|
|
1017
|
+
*/
|
|
1018
|
+
const reportNamedFromCache = async (keys, served = {}) => {
|
|
1019
|
+
if (keys.length === 0 || cliArgs['no-tm']) return;
|
|
1020
|
+
await sayNamedFromCache({
|
|
1021
|
+
filename, keys, shown: keys.map(nsKey), served, onDisk: targetFlat, pairConfig, cwd: ctx.cwd, dryRun,
|
|
1022
|
+
command: redoFor(keys, { fresh: true }),
|
|
1023
|
+
});
|
|
1024
|
+
};
|
|
1025
|
+
const sample = (keys, n = 3) => `${keys.slice(0, n).join(', ')}${keys.length > n ? `, +${keys.length - n} more` : ''}`;
|
|
1026
|
+
|
|
1027
|
+
// Lock state for this file, applied once its outcome is known (never in a
|
|
1028
|
+
// dry run). `written` holds the values as they are now on disk.
|
|
1029
|
+
const settle = ({ written = {}, failed = [], refusedBy = {}, producedBy = {}, askedForms = {}, gapped = {} } = {}) => {
|
|
1030
|
+
if (dryRun) return;
|
|
1031
|
+
for (const [k, value] of Object.entries(written)) {
|
|
1032
|
+
const lk = nsKey(k);
|
|
1033
|
+
if (typeof sourceFlat[k] !== 'string' || value === undefined) continue;
|
|
1034
|
+
localeState.written[lk] = encodeWritten(sourceFlat[k], value);
|
|
1035
|
+
// A plural message answered without a form the language uses: the
|
|
1036
|
+
// setup that answered is remembered, so it is not asked again for this
|
|
1037
|
+
// text — another setup is (lib/plural-gap-redo.js). A complete answer
|
|
1038
|
+
// clears the record.
|
|
1039
|
+
if (gapped[k] && producedBy[k]) recordGap(localeState, lk, sourceFlat[k], producedBy[k]);
|
|
1040
|
+
else if (!gapped[k] && localeState.gaps) delete localeState.gaps[lk];
|
|
1041
|
+
// A borrowed plural form the model was asked for as that form this run:
|
|
1042
|
+
// recorded with its value's fingerprint (verify decides from it). A
|
|
1043
|
+
// record for an earlier answer stays only while it is still the value.
|
|
1044
|
+
if (askedForms[k]) {
|
|
1045
|
+
localeState.forms[lk] = encodeForm(askedForms[k], value);
|
|
1046
|
+
} else {
|
|
1047
|
+
const prior = decodeForm(localeState.forms[lk]);
|
|
1048
|
+
if (!prior || prior.value !== valueHash(value)) delete localeState.forms[lk];
|
|
1049
|
+
}
|
|
1050
|
+
// Which method key produced it (a verbatim copy: none).
|
|
1051
|
+
if (producedBy[k]) localeState.by[lk] = producedBy[k];
|
|
1052
|
+
else delete localeState.by[lk];
|
|
1053
|
+
delete localeState.pending[lk];
|
|
1054
|
+
delete localeState.refused[lk];
|
|
1055
|
+
}
|
|
1056
|
+
for (const k of plan.kept) delete localeState.pending[nsKey(k)];
|
|
1057
|
+
for (const k of failed) {
|
|
1058
|
+
const lk = nsKey(k);
|
|
1059
|
+
// Refused under an explicit redo: the redo promised one more try, so
|
|
1060
|
+
// the next plain sync gets it once (then it is held back).
|
|
1061
|
+
recordRefusal(localeState, lk, sourceFlat[k], refusedBy[k], { redo: plan.forcedByRedo.has(k) });
|
|
1062
|
+
// A redo that could not finish: remembered until it does.
|
|
1063
|
+
if (plan.forcedByRedo.has(k)) localeState.pending[lk] = ctx.redoLabel;
|
|
1064
|
+
}
|
|
1065
|
+
};
|
|
1066
|
+
// What happens to a key this run could not translate, on the next sync:
|
|
1067
|
+
// a key an explicit redo queued is PENDING (asked once more); a key the
|
|
1068
|
+
// gate refused otherwise is HELD BACK (not re-sent to the same method);
|
|
1069
|
+
// a key that got no usable answer is simply asked again.
|
|
1070
|
+
const fateOf = (k, refusedBy) => {
|
|
1071
|
+
if (plan.forcedByRedo.has(k)) return 'pending-retry';
|
|
1072
|
+
return (refusedBy[k] || []).length > 0 ? 'held' : 'retry';
|
|
1073
|
+
};
|
|
1074
|
+
|
|
1075
|
+
/**
|
|
1076
|
+
* Keys whose value on disk an EARLIER model of this pair wrote (the lock's
|
|
1077
|
+
* `by` record, still matching the file): model → count. A plain sync after
|
|
1078
|
+
* a model switch translated nothing and said nothing about it — only
|
|
1079
|
+
* `status` did (Round 6, Next.js persona).
|
|
1080
|
+
*/
|
|
1081
|
+
const otherModelText = () => {
|
|
1082
|
+
const current = tmMethodKey(pairConfig);
|
|
1083
|
+
const [m, , r, c] = current.split('|');
|
|
1084
|
+
let onDisk = targetFlat;
|
|
1085
|
+
if (!dryRun) { try { onDisk = readLocaleFlat(file) || targetFlat; } catch { onDisk = targetFlat; } }
|
|
1086
|
+
const counts = {};
|
|
1087
|
+
for (const k of Object.keys(sourceFlat)) {
|
|
1088
|
+
const lk = nsKey(k);
|
|
1089
|
+
const mk = localeState.by?.[lk];
|
|
1090
|
+
if (!mk || mk === current) continue;
|
|
1091
|
+
const parts = mk.split('|');
|
|
1092
|
+
if (parts.length !== 4 || parts[0] !== m || parts[2] !== r || parts[3] !== c) continue;
|
|
1093
|
+
const record = decodeWritten(localeState.written[lk]);
|
|
1094
|
+
const v = onDisk[k];
|
|
1095
|
+
if (!record || typeof v !== 'string' || record.value !== valueHash(v)) continue;
|
|
1096
|
+
const model = parts[1] || '(none)';
|
|
1097
|
+
counts[model] = (counts[model] || 0) + 1;
|
|
1098
|
+
}
|
|
1099
|
+
return counts;
|
|
1100
|
+
};
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* Keys whose value on disk another METHOD (or register, or coaching file)
|
|
1104
|
+
* of this pair wrote — the lock's `by` record, still matching the file —
|
|
1105
|
+
* method key → count. A change of method re-translates nothing on a plain
|
|
1106
|
+
* sync, and a dry run said "fully synced, 0 keys" with no word that the
|
|
1107
|
+
* files keep the other method's text (Round 7, Next.js persona). Values the
|
|
1108
|
+
* pair's own fallback wrote are its by design, never counted.
|
|
1109
|
+
*/
|
|
1110
|
+
// A dry run of a redo (--redo all / keys:) re-translates what it forces:
|
|
1111
|
+
// those keys are not "left with the other method's text" — the preview
|
|
1112
|
+
// counts them as what the run would send (Round 8, Next.js persona: a dry
|
|
1113
|
+
// `--redo all` after a method change said "Nothing would change" and
|
|
1114
|
+
// recommended the very command being previewed).
|
|
1115
|
+
const otherMethodForced = {};
|
|
1116
|
+
const otherMethodText = () => {
|
|
1117
|
+
const current = tmMethodKey(pairConfig);
|
|
1118
|
+
const [m, , r, c] = current.split('|');
|
|
1119
|
+
const fbKey = pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null;
|
|
1120
|
+
let onDisk = targetFlat;
|
|
1121
|
+
if (!dryRun) { try { onDisk = readLocaleFlat(file) || targetFlat; } catch { onDisk = targetFlat; } }
|
|
1122
|
+
const counts = {};
|
|
1123
|
+
for (const k of Object.keys(sourceFlat)) {
|
|
1124
|
+
const lk = nsKey(k);
|
|
1125
|
+
const mk = localeState.by?.[lk];
|
|
1126
|
+
if (!mk || mk === current || mk === fbKey) continue;
|
|
1127
|
+
const parts = mk.split('|');
|
|
1128
|
+
if (parts.length !== 4 || (parts[0] === m && parts[2] === r && parts[3] === c)) continue;
|
|
1129
|
+
const record = decodeWritten(localeState.written[lk]);
|
|
1130
|
+
const v = onDisk[k];
|
|
1131
|
+
if (!record || typeof v !== 'string' || record.value !== valueHash(v)) continue;
|
|
1132
|
+
if (dryRun && plan.forcedByRedo.has(k) && !heldSet.has(k)) {
|
|
1133
|
+
otherMethodForced[mk] = (otherMethodForced[mk] || 0) + 1;
|
|
1134
|
+
continue;
|
|
1135
|
+
}
|
|
1136
|
+
counts[mk] = (counts[mk] || 0) + 1;
|
|
1137
|
+
}
|
|
1138
|
+
return counts;
|
|
1139
|
+
};
|
|
1140
|
+
|
|
1141
|
+
if (queue.length === 0 && diff.noTranslate.length === 0 && diff.extra.length === 0
|
|
1142
|
+
&& !removedAny) {
|
|
1143
|
+
if (plan.kept.length > 0) reportKept();
|
|
1144
|
+
// `--dry --show-prompt <key>` for a key this file holds up to date: the
|
|
1145
|
+
// request is still shown, with why a real run would not send it.
|
|
1146
|
+
if (dryRun && typeof cliArgs['show-prompt'] === 'string') {
|
|
1147
|
+
await showRequestPreview({
|
|
1148
|
+
cliArgs, layout, unit, pairKey, pairConfig, code, filename, sourceFlat, expansion, apiKey, tm,
|
|
1149
|
+
queued: [], held: heldSet, noteShown: ctx.notePreviewShown, cwd: ctx.cwd,
|
|
1150
|
+
});
|
|
1151
|
+
}
|
|
1152
|
+
// A namespace file with nothing to translate (an empty source file)
|
|
1153
|
+
// still belongs in every locale's folder — i18next requests it, and a
|
|
1154
|
+
// 404 per page load is not "fully synced". Flat layouts keep their
|
|
1155
|
+
// historical behaviour: no file is written for an empty source.
|
|
1156
|
+
if (!existed && layout.namespaced && !dryRun) {
|
|
1157
|
+
writeLocaleData(file, data, unit.yamlStyle);
|
|
1158
|
+
output.ok(`${filename} — created (source file has no keys)`);
|
|
1159
|
+
return { processed: 0, tmHits: 0 };
|
|
1160
|
+
}
|
|
1161
|
+
settle();
|
|
1162
|
+
adoptRecords();
|
|
1163
|
+
if (plan.kept.length === 0) {
|
|
1164
|
+
// A plural form left out (Russian few/many, the "other" form standing
|
|
1165
|
+
// in — marked "# champollion:" in a .po) is not "fully synced": never
|
|
1166
|
+
// an [OK] line over it (Round 10, Django persona). The keys and the
|
|
1167
|
+
// repair are said once the run has read every file.
|
|
1168
|
+
let gaps = [];
|
|
1169
|
+
try { gaps = existed ? pluralGapsInFile({ file, expected: sourceFlat, targetFlat, locale: code }) : []; } catch { gaps = []; }
|
|
1170
|
+
const forms = gaps.reduce((n, g) => n + g.missing.length, 0);
|
|
1171
|
+
if (forms > 0) {
|
|
1172
|
+
output.warn(`${filename} — nothing to translate, but INCOMPLETE: ${forms} plural form(s) `
|
|
1173
|
+
+ `${gaps.some(g => g.marked) ? 'marked "# champollion:"' : 'missing'} (the "other" form stands in; the repair is below)`);
|
|
1174
|
+
} else {
|
|
1175
|
+
output.ok(`${filename} — fully synced`);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
return {
|
|
1179
|
+
processed: 0, tmHits: 0, kept: plan.kept.length, keptKeys: plan.kept.map(nsKey),
|
|
1180
|
+
otherModels: otherModelText(), otherMethods: otherMethodText(), otherMethodsForced: otherMethodForced,
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
/** "kept N hand-edited value(s)" — said whenever a bulk redo leaves a person's text. */
|
|
1185
|
+
function reportKept() {
|
|
1186
|
+
const unrecorded = plan.kept.filter(k => plan.keptUnrecorded.includes(k));
|
|
1187
|
+
const edited = plan.kept.filter(k => !plan.keptUnrecorded.includes(k));
|
|
1188
|
+
const parts = [];
|
|
1189
|
+
if (edited.length > 0) parts.push(`${edited.length} hand-edited value(s) (${sample(edited)})`);
|
|
1190
|
+
if (unrecorded.length > 0) {
|
|
1191
|
+
parts.push(`${unrecorded.length} value(s) Champollion has no record of writing — made by hand, by another tool, `
|
|
1192
|
+
+ `or by an older version (${sample(unrecorded)})`);
|
|
1193
|
+
}
|
|
1194
|
+
output.info(`${filename} — ${dryRun ? 'would keep' : 'kept'} ${parts.join(' and ')}: a bulk redo never replaces a person's text. `
|
|
1195
|
+
+ `\`${redoFor([plan.kept[0]])}\` replaces one (name each key to replace).`);
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
/**
|
|
1199
|
+
* First record of values this version never recorded, when the cache proves
|
|
1200
|
+
* they are Champollion's translation of the CURRENT source text (the
|
|
1201
|
+
* bootstrap rule — lib/locale-state.js). Unproven values stay unrecorded,
|
|
1202
|
+
* so a bulk redo keeps them.
|
|
1203
|
+
*/
|
|
1204
|
+
function adoptRecords(skip = new Set()) {
|
|
1205
|
+
if (dryRun) return;
|
|
1206
|
+
for (const [k, src] of Object.entries(sourceFlat)) {
|
|
1207
|
+
if (typeof src !== 'string' || skip.has(k)) continue;
|
|
1208
|
+
const lk = nsKey(k);
|
|
1209
|
+
if (localeState.written[lk] !== undefined) continue;
|
|
1210
|
+
const v = targetFlat[k];
|
|
1211
|
+
if (typeof v !== 'string' || v.trim() === '' || v.startsWith(config.fallbackPrefix)) continue;
|
|
1212
|
+
if (tmProofTextsFor(k, src, expansion).some(t => tmTranslationsOf(tm, t, code).has(v))) {
|
|
1213
|
+
localeState.written[lk] = encodeWritten(src, v);
|
|
1214
|
+
// The writer, when exactly one cached method holds this text; with
|
|
1215
|
+
// several (identical answers from two models) it stays unknown.
|
|
1216
|
+
const writers = new Set(tmProofTextsFor(k, src, expansion).flatMap(t => tmMethodKeysHolding(tm, t, code, v)));
|
|
1217
|
+
if (writers.size === 1) localeState.by[lk] = [...writers][0];
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
let localeProcessed = 0;
|
|
1223
|
+
let localeTMHits = 0;
|
|
1224
|
+
let localeSent = 0;
|
|
1225
|
+
let localeRetried = 0;
|
|
1226
|
+
// Plural messages accepted without forms the language uses (key → forms).
|
|
1227
|
+
const localeGaps = {};
|
|
1228
|
+
let localeFailed = 0;
|
|
1229
|
+
let localeCopied = 0;
|
|
1230
|
+
let localeKeptWorkingScript = 0;
|
|
1231
|
+
// Key NAMES that failed in this locale — threaded back to the aggregator
|
|
1232
|
+
// so writeManifest can restore their OLD hashes. Persisting the NEW hash
|
|
1233
|
+
// for a failed key would mark it resolved and it would never be retried.
|
|
1234
|
+
const localeFailedKeys = [];
|
|
1235
|
+
// Per-key outcome of what was not translated: next-sync fate → keys.
|
|
1236
|
+
const fates = { retry: [], 'pending-retry': [], held: [] };
|
|
1237
|
+
let localeHeldKeys = [];
|
|
1238
|
+
// Held back THIS run (not sent): lock keys.
|
|
1239
|
+
const heldThisRun = [];
|
|
1240
|
+
const replacedNotes = [];
|
|
1241
|
+
// What the pair's fallback did for this file (null without a fallback),
|
|
1242
|
+
// and whether either method returned nothing at all — so the locale's
|
|
1243
|
+
// next files skip a dead primary even when the fallback saved this one.
|
|
1244
|
+
let fallbackReport = null;
|
|
1245
|
+
let primaryDown = false;
|
|
1246
|
+
let secondDown = false;
|
|
1247
|
+
|
|
1248
|
+
// Counts for the per-file line: what is queued, minus the hand edits kept
|
|
1249
|
+
// and the keys held back (said on their own lines below).
|
|
1250
|
+
// (Plural messages asked again are said on their own line, above.)
|
|
1251
|
+
const hidden = new Set([...keptSet, ...heldSet, ...gapAsked]);
|
|
1252
|
+
const shown = hidden.size === 0 ? diff : Object.fromEntries(Object.entries(diff).map(([k, v]) => (
|
|
1253
|
+
[k, Array.isArray(v) && k !== 'noTranslate' && k !== 'extra' ? v.filter(x => !hidden.has(x)) : v])));
|
|
1254
|
+
// gettext entries flagged fuzzy (makemessages after a msgid change) read as
|
|
1255
|
+
// missing; they are named as fuzzy (lib/po.js poFuzzyKeys).
|
|
1256
|
+
const fuzzyKeys = format === 'po' && existed && shown.missing.length > 0
|
|
1257
|
+
? (() => { const f = poFuzzyKeys(fs.readFileSync(filePath, 'utf-8')); return shown.missing.filter(k => f.has(k)); })()
|
|
1258
|
+
: [];
|
|
1259
|
+
if (queue.length > 0 || diff.noTranslate.length > 0) {
|
|
1260
|
+
const pendingPart = plan.pendingRetry.length > 0 ? ` (${plan.pendingRetry.length} pending from an unfinished redo)` : '';
|
|
1261
|
+
const label = diffLabel({ ...shown, fuzzy: fuzzyKeys, fuzzyVerb: dryRun ? 'would be re-translated' : 're-translating' });
|
|
1262
|
+
const parts = [
|
|
1263
|
+
label === 'fully synced' ? null : label,
|
|
1264
|
+
gapAsked.length > 0 && `${gapAsked.length} plural message(s) asked again`,
|
|
1265
|
+
plan.held.length > 0 && `${plan.held.length} held back`,
|
|
1266
|
+
].filter(Boolean);
|
|
1267
|
+
const heldPart = parts.length > 0 ? parts.join(' + ') : label;
|
|
1268
|
+
output.info(`${filename} — ${heldPart}${pendingPart}`);
|
|
1269
|
+
}
|
|
1270
|
+
if (plan.kept.length > 0) reportKept();
|
|
1271
|
+
if (plan.pendingRetry.length > 0) {
|
|
1272
|
+
const reasons = [...new Set(plan.pendingRetry.map(k => localeState.pending[nsKey(k)]).filter(Boolean))];
|
|
1273
|
+
output.info(`${filename} — ${dryRun ? 'would retry' : 'retrying'} ${plan.pendingRetry.length} key(s) an earlier `
|
|
1274
|
+
+ `\`${reasons.join('`, `') || '--redo'}\` could not finish (${sample(plan.pendingRetry)}): sent to the model again, not served from the cache.`);
|
|
1275
|
+
}
|
|
1276
|
+
if (!dryRun && plan.pendingAll.length > 0) bypassTMFor(tm, code, plan.pendingAll.map(k => cachedAs(k, sourceFlat[k])));
|
|
1277
|
+
if (plan.held.length > 0) {
|
|
1278
|
+
output.warn(describeHeldKeys({ filename, keys: plan.held, pairConfig, command: redoFor(plan.held) }));
|
|
1279
|
+
}
|
|
1280
|
+
if (plan.heldFromPrimary.length > 0) {
|
|
1281
|
+
output.info(describeFallbackOnlyKeys({ filename, keys: plan.heldFromPrimary, pairConfig }));
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
// ── No-translate keys: copy the source value, verbatim ─────────────
|
|
1285
|
+
// Runs BEFORE the translation block so a dead backend can't strand a
|
|
1286
|
+
// corrupted URL for another cycle (see flushNoTranslateOnBail below).
|
|
1287
|
+
// No API call, no quality gate, no cost — and byte-identical by
|
|
1288
|
+
// construction, which is the only correct outcome for these values.
|
|
1289
|
+
if (diff.noTranslate.length > 0) {
|
|
1290
|
+
for (const key of diff.noTranslate) {
|
|
1291
|
+
if (dryRun) continue;
|
|
1292
|
+
if (format === 'json') setNestedValue(data, key, sourceFlat[key]);
|
|
1293
|
+
else assignInOrder(data, key, sourceFlat[key]);
|
|
1294
|
+
}
|
|
1295
|
+
localeCopied = diff.noTranslate.length;
|
|
1296
|
+
const sampleNT = diff.noTranslate.slice(0, 3)
|
|
1297
|
+
.map(k => `${k} (${noTranslate.reason(k, sourceFlat[k])})`)
|
|
1298
|
+
.join(', ');
|
|
1299
|
+
const more = diff.noTranslate.length > 3 ? `, +${diff.noTranslate.length - 3} more` : '';
|
|
1300
|
+
output.info(`${filename} — ${dryRun ? 'would copy' : 'copied'} ${localeCopied} no-translate key(s) verbatim: ${sampleNT}${more}`);
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
/** The values on disk after a write, for the written-record (null if unreadable). */
|
|
1304
|
+
const rereadAfterWrite = () => {
|
|
1305
|
+
try { return readLocaleFlat(file) || {}; } catch { return null; }
|
|
1306
|
+
};
|
|
1307
|
+
/** Record what a write left on disk for `keys`. */
|
|
1308
|
+
const writtenValues = (keys) => {
|
|
1309
|
+
const onDisk = rereadAfterWrite();
|
|
1310
|
+
if (!onDisk) return {};
|
|
1311
|
+
const out = {};
|
|
1312
|
+
for (const k of keys) if (onDisk[k] !== undefined) out[k] = onDisk[k];
|
|
1313
|
+
return out;
|
|
1314
|
+
};
|
|
1315
|
+
|
|
1316
|
+
// A whole-locale translation failure returns early and writes nothing, so
|
|
1317
|
+
// the verbatim copies above would be lost with it. They do not depend on
|
|
1318
|
+
// the backend, so flush them: a repair that is already computed and free
|
|
1319
|
+
// must not wait on an unrelated outage.
|
|
1320
|
+
const flushNoTranslateOnBail = () => {
|
|
1321
|
+
if (dryRun || diff.noTranslate.length === 0) return 0;
|
|
1322
|
+
try {
|
|
1323
|
+
writeLocaleData(file, data, unit.yamlStyle);
|
|
1324
|
+
return localeCopied;
|
|
1325
|
+
} catch (err) {
|
|
1326
|
+
output.error(`${filename} — failed to write no-translate copies: ${err.message}`);
|
|
1327
|
+
return 0;
|
|
1328
|
+
}
|
|
1329
|
+
};
|
|
1330
|
+
/** A whole-file failure: settle the lock (pending redo keys, refusals), then flush the copies. */
|
|
1331
|
+
const bail = (failedKeys, refusedBy = {}) => {
|
|
1332
|
+
const copied = flushNoTranslateOnBail();
|
|
1333
|
+
settle({ written: copied > 0 ? writtenValues(diff.noTranslate) : {}, failed: failedKeys, refusedBy });
|
|
1334
|
+
for (const k of failedKeys) fates[fateOf(k, refusedBy)].push(nsKey(k));
|
|
1335
|
+
return copied;
|
|
1336
|
+
};
|
|
1337
|
+
|
|
1338
|
+
// The locale's shared-output index starts with what this file already
|
|
1339
|
+
// holds for keys this run leaves alone: a model repeating one memorized
|
|
1340
|
+
// sentence across syncs (a key added per sync) used to be caught by no run,
|
|
1341
|
+
// because each run's index started empty (Round 5, hospital persona). Only
|
|
1342
|
+
// the new answers are refused; verify names the written ones.
|
|
1343
|
+
{
|
|
1344
|
+
const queuedSet = new Set(queue);
|
|
1345
|
+
const seed = [];
|
|
1346
|
+
for (const [k, v] of Object.entries(targetFlat)) {
|
|
1347
|
+
if (typeof v !== 'string' || typeof sourceFlat[k] !== 'string' || queuedSet.has(k)) continue;
|
|
1348
|
+
seed.push(...sharedOutputItems(nsKey(k), sourceFlat[k], v));
|
|
1349
|
+
}
|
|
1350
|
+
if (seed.length > 0) ctx.sharedOutputsFor(code).add(seed);
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
// `--dry --show-prompt [key]`: the exact request the method would get —
|
|
1354
|
+
// for the named key (queued or not), else for what this file would send.
|
|
1355
|
+
if (dryRun && cliArgs['show-prompt']) {
|
|
1356
|
+
await showRequestPreview({
|
|
1357
|
+
cliArgs, layout, unit, pairKey, pairConfig, code, filename, sourceFlat, expansion, apiKey, tm,
|
|
1358
|
+
queued: queue.filter(k => typeof sourceFlat[k] === 'string' && !heldSet.has(k)), held: heldSet,
|
|
1359
|
+
noteShown: ctx.notePreviewShown, cwd: ctx.cwd,
|
|
1360
|
+
});
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
if (queue.length > 0) {
|
|
1364
|
+
if (dryRun) {
|
|
1365
|
+
// Dry-run does no API calls and writes nothing, but it must still
|
|
1366
|
+
// report what it WOULD process — otherwise the summary always reads
|
|
1367
|
+
// "Would have processed 0 keys total." even with pending work.
|
|
1368
|
+
// (Keys held back would not be sent: not counted.)
|
|
1369
|
+
localeProcessed += queue.length - plan.held.length;
|
|
1370
|
+
// Named keys the real run would serve from the cache (the same
|
|
1371
|
+
// partition the estimate makes — the pair's entry, then its fallback's).
|
|
1372
|
+
if (namedKeys.size > 0 && !cliArgs['no-tm']) {
|
|
1373
|
+
const keysToCheck = queue.filter(k => namedKeys.has(k) && typeof sourceFlat[k] === 'string' && !heldSet.has(k));
|
|
1374
|
+
const served = {};
|
|
1375
|
+
for (const k of keysToCheck) {
|
|
1376
|
+
for (const mk of tmKeys) {
|
|
1377
|
+
const v = peekTM(tm, cachedAs(k, sourceFlat[k]), code, mk);
|
|
1378
|
+
if (v !== null) { served[k] = v; break; }
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
await reportNamedFromCache(Object.keys(served), served);
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
// --list-keys: NAME the queued keys, per reason. Counts alone made
|
|
1385
|
+
// investigating a surprise queue impossible without re-implementing
|
|
1386
|
+
// the diff by hand — integrity names damaged keys; a dry run must
|
|
1387
|
+
// name queued ones. (The --json summary always carries these lists
|
|
1388
|
+
// on dry runs; this is the human rendering.)
|
|
1389
|
+
if (cliArgs['list-keys']) {
|
|
1390
|
+
// One key, one reason (the label's own partition).
|
|
1391
|
+
const once = queueReasons(shown);
|
|
1392
|
+
const sections = [
|
|
1393
|
+
['missing', once.missing.filter(k => !fuzzyKeys.includes(k))],
|
|
1394
|
+
['fuzzy (source changed)', fuzzyKeys],
|
|
1395
|
+
['[EN] fallback', once.needsTranslation],
|
|
1396
|
+
['untranslated (unstamped echo)', once.untranslated],
|
|
1397
|
+
['changed', once.changed],
|
|
1398
|
+
['forced', once.forced.filter(k => !plan.pendingRetry.includes(k) && !gapAsked.includes(k))],
|
|
1399
|
+
['asked again (a plural form left out)', gapAsked],
|
|
1400
|
+
['pending (an unfinished redo)', plan.pendingRetry],
|
|
1401
|
+
['held back (refused before)', plan.held],
|
|
1402
|
+
['kept (a person\'s edit)', plan.kept],
|
|
1403
|
+
['copy verbatim (no-translate)', diff.noTranslate],
|
|
1404
|
+
];
|
|
1405
|
+
for (const [label, keys] of sections) {
|
|
1406
|
+
if (keys.length === 0) continue;
|
|
1407
|
+
output.raw(` ${label}:`);
|
|
1408
|
+
for (const k of keys) output.raw(` - ${k}`);
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
if (!dryRun) {
|
|
1414
|
+
let translated = null;
|
|
1415
|
+
|
|
1416
|
+
const stringKeys = queue.filter(k => typeof sourceFlat[k] === 'string');
|
|
1417
|
+
// An earlier file of this locale got NO results from the method (dead
|
|
1418
|
+
// key, outage): every further call would fail the same way. Count this
|
|
1419
|
+
// file's pending keys as failed (they retry next sync) and keep the
|
|
1420
|
+
// free verbatim copies — do not hammer the backend once per namespace.
|
|
1421
|
+
// With a working fallback the file goes straight to the fallback.
|
|
1422
|
+
const skipPrimary = backendDown && !!pairConfig.fallback && !fallbackDown;
|
|
1423
|
+
if (backendDown && !skipPrimary && stringKeys.length > 0) {
|
|
1424
|
+
output.error(`${filename} — not attempted: the translation method returned no results for an earlier file of ${code}.`);
|
|
1425
|
+
const copied = bail(queue);
|
|
1426
|
+
return { processed: 0, tmHits: 0, copied, failed: queue.length, failedKeys: queue.map(toLock), pairKey, backendDown: true, fallbackDown, fates };
|
|
1427
|
+
}
|
|
1428
|
+
let result = null;
|
|
1429
|
+
if (stringKeys.length > 0) {
|
|
1430
|
+
// Prompt context for generated plural categories (French `_many`
|
|
1431
|
+
// is translated from the English `_other` text).
|
|
1432
|
+
const pluralDescriptions = promptNotesFor(stringKeys, expansion);
|
|
1433
|
+
// Shared pipeline: TM partition → API call → quality gate → TM
|
|
1434
|
+
// store, then the pair's fallback (if any) for what that left
|
|
1435
|
+
// untranslated (lib/translate-pair.js translateWithFallback).
|
|
1436
|
+
// A gettext catalog holds only the plural forms its header has slots
|
|
1437
|
+
// for: the gate does not ask again for one it cannot write, and the
|
|
1438
|
+
// report does not call it missing (lib/po.js poPluralSlots).
|
|
1439
|
+
const pluralSlots = format === 'po'
|
|
1440
|
+
? poPluralSlots(existed ? fs.readFileSync(filePath, 'utf-8') : null, code) : null;
|
|
1441
|
+
const runConfig = pluralSlots ? withPluralSlots(pairConfig, pluralSlots) : pairConfig;
|
|
1442
|
+
result = await translateWithFallback(stringKeys, sourceFlat, runConfig, pairKey, {
|
|
1443
|
+
apiKey, tm, targetCode: code, cwd: ctx.cwd,
|
|
1444
|
+
...(Object.keys(pluralDescriptions).length > 0 && { descriptions: pluralDescriptions }),
|
|
1445
|
+
// Borrowed plural forms: cached under their own identity.
|
|
1446
|
+
...(expansion?.borrowed && Object.keys(expansion.borrowed).length > 0 && { pluralForms: expansion.borrowed }),
|
|
1447
|
+
onProgress: (completed, total) => {
|
|
1448
|
+
output.progressBar(completed, total, { item: filename });
|
|
1449
|
+
},
|
|
1450
|
+
skipPrimary,
|
|
1451
|
+
budget: ctx.fallbackBudget || null,
|
|
1452
|
+
noSendPrimary: new Set([...plan.held, ...plan.heldFromPrimary]),
|
|
1453
|
+
noSendFallback: heldSet,
|
|
1454
|
+
sharedOutputs: ctx.sharedOutputsFor(code),
|
|
1455
|
+
});
|
|
1456
|
+
translated = result.translated;
|
|
1457
|
+
fallbackReport = result.fallback;
|
|
1458
|
+
// Named as the run names keys (with the file's namespace).
|
|
1459
|
+
if (fallbackReport && Array.isArray(fallbackReport.produced)) fallbackReport.produced = fallbackReport.produced.map(nsKey);
|
|
1460
|
+
localeHeldKeys = result.heldKeys || [];
|
|
1461
|
+
if (pairConfig.fallback) {
|
|
1462
|
+
primaryDown = !!result.apiReturnedNull;
|
|
1463
|
+
secondDown = !!result.fallbackReturnedNull;
|
|
1464
|
+
}
|
|
1465
|
+
localeTMHits += result.tmHitCount;
|
|
1466
|
+
localeSent += result.sentCount || 0;
|
|
1467
|
+
localeRetried += result.retriedCount || 0;
|
|
1468
|
+
for (const [k, g] of Object.entries(result.pluralGaps || {})) {
|
|
1469
|
+
if (g.everyday.length > 0) localeGaps[nsKey(k)] = g.everyday;
|
|
1470
|
+
}
|
|
1471
|
+
if ((result.sentKeys || []).length > 0) ctx.noteCacheScope(pairKey, pairConfig, code, result.sentKeys.map(k => cachedAs(k, sourceFlat[k])));
|
|
1472
|
+
|
|
1473
|
+
// Plural forms: generated i18next categories, and plural messages
|
|
1474
|
+
// that lack forms the target language has (never a silent fill).
|
|
1475
|
+
// Only forms actually sent: a cache hit was not "asked" anything.
|
|
1476
|
+
reportGeneratedPluralKeys({ keys: result.sentKeys || [], expansion, filename, pairConfig, code, sourceLocale: inputLocale });
|
|
1477
|
+
if (namedKeys.size > 0 && translated) {
|
|
1478
|
+
const answered = new Set(result.answeredKeys || []);
|
|
1479
|
+
await reportNamedFromCache(stringKeys.filter(k => namedKeys.has(k) && k in translated && !answered.has(k)), translated);
|
|
1480
|
+
}
|
|
1481
|
+
if (translated) {
|
|
1482
|
+
reportPluralGaps({ gaps: result.pluralGaps, filename, format, pairKey, pairConfig, code, layout, ns: unit.ns });
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
// Terminology enforcement: check the glossary terms were applied —
|
|
1486
|
+
// for every method (the glossary is loaded per pair above).
|
|
1487
|
+
const glossary = pairConfig.coachingData?.dictionary || pairConfig.glossary;
|
|
1488
|
+
if (translated && glossary) {
|
|
1489
|
+
const { violations } = verifyTerminology(translated, sourceFlat, glossary);
|
|
1490
|
+
if (violations.length > 0) {
|
|
1491
|
+
logTermViolations(violations, pairKey);
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
const heldHere = new Set(localeHeldKeys);
|
|
1496
|
+
const notDone = stringKeys.filter(k => !(translated && k in translated));
|
|
1497
|
+
const refusedHere = notDone.filter(k => (result.refusedBy?.[k] || []).length > 0);
|
|
1498
|
+
const heldNow = notDone.filter(k => heldHere.has(k));
|
|
1499
|
+
const noAnswer = notDone.filter(k => !heldHere.has(k) && !refusedHere.includes(k));
|
|
1500
|
+
if (translated) {
|
|
1501
|
+
// Never [OK] for a file with keys left untranslated (Round 4,
|
|
1502
|
+
// Next.js persona: "fr.json [OK]" over 3 refused keys).
|
|
1503
|
+
// Nor over a plural form the translation left out (Round 10,
|
|
1504
|
+
// Django persona: "ru/LC_MESSAGES/django.po [OK]" over two forms
|
|
1505
|
+
// marked "# champollion:") — the warning above names them.
|
|
1506
|
+
const gapForms = Object.values(result.pluralGaps || {}).reduce((n, g) => n + (g.everyday?.length || 0), 0);
|
|
1507
|
+
const gapNote = gapForms > 0
|
|
1508
|
+
? `${gapForms} plural form(s) ${format === 'po' ? 'marked "# champollion:"' : 'missing — the "other" form stands in'}`
|
|
1509
|
+
: null;
|
|
1510
|
+
if (notDone.length === 0) {
|
|
1511
|
+
output.progressDone(filename, gapNote ? `[INCOMPLETE: ${gapNote}]` : '[OK]');
|
|
1512
|
+
} else {
|
|
1513
|
+
const why = [
|
|
1514
|
+
refusedHere.length > 0 && `${refusedHere.length} refused by the quality gate`,
|
|
1515
|
+
heldNow.length > 0 && `${heldNow.length} held back`,
|
|
1516
|
+
noAnswer.length > 0 && `${noAnswer.length} not returned by the method`,
|
|
1517
|
+
].filter(Boolean).join(', ');
|
|
1518
|
+
output.progressDone(filename, `[WARN] ${notDone.length} of ${stringKeys.length} key(s) not translated (${why})${gapNote ? `; ${gapNote}` : ''}`);
|
|
1519
|
+
}
|
|
1520
|
+
} else if (result.apiReturnedNull) {
|
|
1521
|
+
// Method returned null — fail loud with actionable guidance.
|
|
1522
|
+
// (When the primary was skipped, its help was printed for the
|
|
1523
|
+
// earlier file that found it down.)
|
|
1524
|
+
output.progressDone(filename, '[ERR]');
|
|
1525
|
+
if (!skipPrimary) {
|
|
1526
|
+
output.error(`${pairKey}: Translation method "${pairConfig.method}" returned no results.`);
|
|
1527
|
+
const methodInstance = getMethod(pairConfig.method, pairConfig);
|
|
1528
|
+
for (const line of await methodSetupAdvice(methodInstance, { apiKey, cwd: ctx.cwd })) {
|
|
1529
|
+
output.error(line);
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
const fbDown = !!pairConfig.fallback && !!result.fallbackReturnedNull;
|
|
1533
|
+
if (fbDown) {
|
|
1534
|
+
output.error(`${pairKey}: the fallback method "${pairConfig.fallback.method}" returned no results either.`);
|
|
1535
|
+
for (const line of await methodSetupAdvice(getMethod(pairConfig.fallback.method, pairConfig.fallback), { apiKey, cwd: ctx.cwd })) {
|
|
1536
|
+
output.error(line);
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
// Whole file failed: every pending key must re-fire next sync, and
|
|
1540
|
+
// the locale's remaining files are not attempted (backendDown).
|
|
1541
|
+
const copied = bail(queue, result.refusedBy || {});
|
|
1542
|
+
return { processed: 0, tmHits: localeTMHits, sent: localeSent, retried: localeRetried, copied, failed: queue.length, failedKeys: queue.map(toLock), pairKey, backendDown: true, fallbackDown: fbDown, fallback: fallbackReport, fates };
|
|
1543
|
+
} else if (heldNow.length === stringKeys.length) {
|
|
1544
|
+
// Everything queued was held back: nothing was asked, nothing failed anew.
|
|
1545
|
+
output.progressDone(filename, `[WARN] ${heldNow.length} key(s) held back, not translated`);
|
|
1546
|
+
for (const k of heldNow) fates.held.push(nsKey(k));
|
|
1547
|
+
localeFailedKeys.push(...heldNow.map(toLock));
|
|
1548
|
+
const copied = flushNoTranslateOnBail();
|
|
1549
|
+
settle({ written: copied > 0 ? writtenValues(diff.noTranslate) : {} });
|
|
1550
|
+
adoptRecords(new Set([...queue, ...diff.noTranslate]));
|
|
1551
|
+
return {
|
|
1552
|
+
processed: 0, tmHits: localeTMHits, sent: 0, retried: 0, copied, failed: 0,
|
|
1553
|
+
failedKeys: localeFailedKeys, held: heldNow.length, heldKeys: heldNow.map(nsKey), kept: plan.kept.length,
|
|
1554
|
+
keptKeys: plan.kept.map(nsKey), fates, pairKey, fallback: fallbackReport,
|
|
1555
|
+
};
|
|
1556
|
+
} else if (result.failures.length > 0 && !translated) {
|
|
1557
|
+
// All translations failed quality gate — fail loud
|
|
1558
|
+
output.progressDone(filename, '[ERR] all translations failed quality gate');
|
|
1559
|
+
output.error(`${pairKey}: All translations were rejected by the quality gate${pairConfig.fallback ? ` (and by its fallback, ${pairConfig.fallback.method})` : ''}.`);
|
|
1560
|
+
output.error('Check your method configuration or review the gate failures above.');
|
|
1561
|
+
const failedNow = queue.filter(k => !heldHere.has(k));
|
|
1562
|
+
const copied = bail(failedNow, result.refusedBy || {});
|
|
1563
|
+
for (const k of heldNow) fates.held.push(nsKey(k));
|
|
1564
|
+
return { processed: 0, tmHits: localeTMHits, sent: localeSent, retried: localeRetried, copied, failed: failedNow.length, held: heldNow.length, heldKeys: heldNow.map(nsKey), failedKeys: queue.map(toLock), pairKey, fallback: fallbackReport, fates };
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
// Post-translation script conversion — ONLY when this pair's script
|
|
1569
|
+
// resolution asked for it (config `script:`). The old gate was a bare
|
|
1570
|
+
// registry lookup, which converted every tlh/crk/… project into
|
|
1571
|
+
// display scripts (PUA for the conlangs) whether or not their fonts
|
|
1572
|
+
// could render them. See lib/scripts.js resolveTargetScript.
|
|
1573
|
+
const scriptConverterKey = pairConfig.scriptResolution?.converterKey || null;
|
|
1574
|
+
if (scriptConverterKey && translated && Object.keys(translated).length > 0) {
|
|
1575
|
+
const info = getConverterInfo(scriptConverterKey);
|
|
1576
|
+
output.info(`[SCRIPT] Converting ${info.from} → ${info.to} (${Object.keys(translated).length} keys)`);
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
const writtenKeys = [];
|
|
1580
|
+
const heldHere = new Set(localeHeldKeys);
|
|
1581
|
+
for (const key of queue) {
|
|
1582
|
+
const sourceValue = sourceFlat[key];
|
|
1583
|
+
let value;
|
|
1584
|
+
|
|
1585
|
+
if (translated && key in translated) {
|
|
1586
|
+
value = translated[key];
|
|
1587
|
+
|
|
1588
|
+
if (scriptConverterKey && typeof value === 'string') {
|
|
1589
|
+
// User-declared transliteration fallbacks first (validated at
|
|
1590
|
+
// pair build), then the converter. If letters remain that the
|
|
1591
|
+
// converter cannot map, the output would be an unreadable mix
|
|
1592
|
+
// of both scripts — keep the WHOLE value in the working script
|
|
1593
|
+
// instead, and say which letters and how to map them. Not a
|
|
1594
|
+
// failure: unmappable proper nouns would fail identically on
|
|
1595
|
+
// every retry, and a permanently red sync is the trap this
|
|
1596
|
+
// release exists to close.
|
|
1597
|
+
const prepared = applyScriptFallback(value, pairConfig.scriptFallback);
|
|
1598
|
+
const { converted, unmapped } = convertScript(prepared, scriptConverterKey);
|
|
1599
|
+
if (unmapped.length === 0) {
|
|
1600
|
+
value = converted;
|
|
1601
|
+
} else {
|
|
1602
|
+
const hint = unmapped.map(l => `"${l}": "?"`).join(', ');
|
|
1603
|
+
output.warn(
|
|
1604
|
+
`${pairKey}: key "${key}" kept in ${getConverterInfo(scriptConverterKey).from} — `
|
|
1605
|
+
+ `letter(s) the converter cannot map: ${unmapped.join(', ')}. `
|
|
1606
|
+
+ `To transliterate them, add "scriptFallback": { ${hint} } for ${pairConfig.target}.`
|
|
1607
|
+
);
|
|
1608
|
+
localeKeptWorkingScript++;
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
} else if (typeof sourceValue === 'string') {
|
|
1612
|
+
// Not translated (gate refusal, no answer, or held back) — skip
|
|
1613
|
+
// it, never write garbage. Its old manifest hash is restored so a
|
|
1614
|
+
// changed source is still detected; what the next sync does with
|
|
1615
|
+
// it is said here and in the summary (lib/locale-state.js).
|
|
1616
|
+
localeFailedKeys.push(toLock(key));
|
|
1617
|
+
if (heldHere.has(key)) {
|
|
1618
|
+
heldThisRun.push(nsKey(key));
|
|
1619
|
+
fates.held.push(nsKey(key));
|
|
1620
|
+
continue;
|
|
1621
|
+
}
|
|
1622
|
+
localeFailed++;
|
|
1623
|
+
const fate = fateOf(key, result?.refusedBy || {});
|
|
1624
|
+
fates[fate].push(nsKey(key));
|
|
1625
|
+
output.warn(`${pairKey}: key "${key}" ${keyFateNote(fate, pairConfig)}`);
|
|
1626
|
+
continue;
|
|
1627
|
+
} else {
|
|
1628
|
+
value = sourceValue;
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
// A new plural form lands beside its siblings in CLDR order (both
|
|
1632
|
+
// helpers), never after `_other` (Round 10, i18next persona).
|
|
1633
|
+
if (format === 'json') {
|
|
1634
|
+
setNestedValue(data, key, value);
|
|
1635
|
+
} else {
|
|
1636
|
+
assignInOrder(data, key, value);
|
|
1637
|
+
}
|
|
1638
|
+
writtenKeys.push(key);
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
// A question or exclamation that lost its "?" / "!" (Round 6, hospital
|
|
1642
|
+
// persona): a warning — some languages use a particle instead.
|
|
1643
|
+
const droppedMarks = droppedTerminalMarks(writtenKeys
|
|
1644
|
+
.filter(k => translated && k in translated && !isProtectedTermValue(translated[k], config.protectedTerms))
|
|
1645
|
+
.map(k => [k, sourceFlat[k], translated[k]]));
|
|
1646
|
+
if (droppedMarks.length > 0) {
|
|
1647
|
+
output.warn(`${filename}: ${describeDroppedMarks(droppedMarks, redoFor(droppedMarks.map(f => f.key), { fresh: true }))}`);
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
// Held-back keys were never sent: not counted as processed.
|
|
1651
|
+
localeProcessed += queue.length - heldThisRun.length;
|
|
1652
|
+
|
|
1653
|
+
// Hand edits this run replaces (their source changed, or the key was
|
|
1654
|
+
// named): printed, and recorded once the file is written.
|
|
1655
|
+
const writtenSet = new Set(writtenKeys);
|
|
1656
|
+
for (const r of plan.replacing) {
|
|
1657
|
+
if (!writtenSet.has(r.key)) continue;
|
|
1658
|
+
replacedNotes.push(r);
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
// Write updated file (see below), then settle the lock.
|
|
1662
|
+
if (diff.toProcess.length > 0 || diff.noTranslate.length > 0 || removedAny) {
|
|
1663
|
+
try {
|
|
1664
|
+
writeLocaleData(file, data, unit.yamlStyle);
|
|
1665
|
+
} catch (err) {
|
|
1666
|
+
output.error(`${filename} — failed to write: ${err.message}`);
|
|
1667
|
+
// Everything we attempted for this locale is unwritten → all failed.
|
|
1668
|
+
// That includes the verbatim copies: they were staged in memory only.
|
|
1669
|
+
settle({ failed: queue, refusedBy: result?.refusedBy || {} });
|
|
1670
|
+
return {
|
|
1671
|
+
processed: 0,
|
|
1672
|
+
tmHits: localeTMHits,
|
|
1673
|
+
sent: localeSent,
|
|
1674
|
+
retried: localeRetried,
|
|
1675
|
+
copied: 0,
|
|
1676
|
+
failed: queue.length,
|
|
1677
|
+
failedKeys: queue.map(toLock),
|
|
1678
|
+
pairKey,
|
|
1679
|
+
fallback: fallbackReport,
|
|
1680
|
+
};
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
const failedNow = queue.filter(k => typeof sourceFlat[k] === 'string' && !writtenSet.has(k) && !heldHere.has(k));
|
|
1684
|
+
// Borrowed plural forms this run's method was asked for AS that form
|
|
1685
|
+
// (its own answer, not a cache hit; a method that reads instructions —
|
|
1686
|
+
// a machine-translation engine is only given the "_other" text).
|
|
1687
|
+
const askedForms = {};
|
|
1688
|
+
if (expansion?.borrowed && result?.answeredKeys) {
|
|
1689
|
+
const fbKey = pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null;
|
|
1690
|
+
for (const k of result.answeredKeys) {
|
|
1691
|
+
const form = expansion.borrowed[k];
|
|
1692
|
+
if (!form || !writtenSet.has(k)) continue;
|
|
1693
|
+
const byFallback = fbKey !== null && result.producedBy?.[k] === fbKey;
|
|
1694
|
+
if (methodTakesInstructions(byFallback ? pairConfig.fallback : pairConfig)) askedForms[k] = form;
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
settle({
|
|
1698
|
+
written: writtenValues([...writtenKeys, ...diff.noTranslate]),
|
|
1699
|
+
failed: failedNow,
|
|
1700
|
+
refusedBy: result?.refusedBy || {},
|
|
1701
|
+
producedBy: result?.producedBy || {},
|
|
1702
|
+
askedForms,
|
|
1703
|
+
gapped: Object.fromEntries(Object.entries(result?.pluralGaps || {}).filter(([, g]) => g.everyday?.length > 0).map(([k]) => [k, true])),
|
|
1704
|
+
});
|
|
1705
|
+
adoptRecords(new Set([...queue, ...diff.noTranslate]));
|
|
1706
|
+
for (const r of replacedNotes) {
|
|
1707
|
+
const why = r.why === 'named' ? 'it was named for a redo' : 'its source text changed (the edit was for the old text)';
|
|
1708
|
+
output.warn(`${filename}: "${r.key}" — replaced ${r.unrecorded ? 'a value Champollion had no record of writing' : 'a hand-edited translation'} because ${why}. `
|
|
1709
|
+
+ `The edited wording, to re-apply if it still fits: ${JSON.stringify(r.value)} (kept in ${REPLACED_EDITS_FILENAME})`);
|
|
1710
|
+
ctx.replacedEdits.push({
|
|
1711
|
+
locale: code, file: filename, key: nsKey(r.key), editedValue: r.value,
|
|
1712
|
+
why: r.why === 'named' ? 'named for a redo' : 'source changed', newSource: sourceFlat[r.key],
|
|
1713
|
+
...(r.unrecorded && { note: 'no written-record (made by hand, by another tool, or by an older version)' }),
|
|
1714
|
+
});
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
if (diff.extra.length > 0) {
|
|
1720
|
+
// Plural keys for a form the language does not have are named, with the
|
|
1721
|
+
// flag that removes exactly them (Round 13, i18next persona).
|
|
1722
|
+
const pluralExtras = pluralExtraKeys(unit, targetFlat, code).map(e => e.key);
|
|
1723
|
+
output.warn(`${filename} — ${diff.extra.length} extra key(s) not in source`
|
|
1724
|
+
+ (pluralExtras.length > 0
|
|
1725
|
+
? ` (${pluralExtras.length} for a plural form ${code} does not have: ${pluralExtras.slice(0, 3).map(nsKey).join(', ')}${pluralExtras.length > 3 ? ', …' : ''} — `
|
|
1726
|
+
+ `\`champollion sync --pair ${shellWord(pairKey)} --prune plural-extras\` removes ${pluralExtras.length === 1 ? 'it' : 'them'})`
|
|
1727
|
+
: ''));
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
// Only no-translate copies or plural cleanup (nothing queued to translate):
|
|
1731
|
+
// write and settle here. CRITICAL: isolate the write per-locale. If one
|
|
1732
|
+
// locale file is unwritable (read-only, a full disk, a locked file), a
|
|
1733
|
+
// thrown error must not reject the whole pMap and discard every SIBLING
|
|
1734
|
+
// locale's already-paid translations: count this locale's keys as failed
|
|
1735
|
+
// and let the other locales write.
|
|
1736
|
+
if (!dryRun && queue.length === 0 && (diff.noTranslate.length > 0 || removedAny)) {
|
|
1737
|
+
try {
|
|
1738
|
+
writeLocaleData(file, data, unit.yamlStyle);
|
|
1739
|
+
} catch (err) {
|
|
1740
|
+
output.error(`${filename} — failed to write: ${err.message}`);
|
|
1741
|
+
return { processed: 0, tmHits: 0, copied: 0, failed: 0, failedKeys: [], pairKey };
|
|
1742
|
+
}
|
|
1743
|
+
settle({ written: writtenValues(diff.noTranslate) });
|
|
1744
|
+
adoptRecords(new Set(diff.noTranslate));
|
|
1745
|
+
} else if (!dryRun && queue.length === 0) {
|
|
1746
|
+
settle();
|
|
1747
|
+
adoptRecords();
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
return {
|
|
1751
|
+
processed: localeProcessed,
|
|
1752
|
+
tmHits: localeTMHits,
|
|
1753
|
+
sent: localeSent,
|
|
1754
|
+
retried: localeRetried,
|
|
1755
|
+
otherModels: otherModelText(),
|
|
1756
|
+
otherMethods: otherMethodText(),
|
|
1757
|
+
otherMethodsForced: otherMethodForced,
|
|
1758
|
+
pluralGaps: localeGaps,
|
|
1759
|
+
copied: localeCopied,
|
|
1760
|
+
keptWorkingScript: localeKeptWorkingScript,
|
|
1761
|
+
failed: localeFailed,
|
|
1762
|
+
failedKeys: localeFailedKeys,
|
|
1763
|
+
held: heldThisRun.length,
|
|
1764
|
+
heldKeys: heldThisRun,
|
|
1765
|
+
kept: plan.kept.length,
|
|
1766
|
+
keptKeys: plan.kept.map(nsKey),
|
|
1767
|
+
replaced: replacedNotes.length,
|
|
1768
|
+
fates,
|
|
1769
|
+
pairKey,
|
|
1770
|
+
fallback: fallbackReport,
|
|
1771
|
+
...(primaryDown && { backendDown: true }),
|
|
1772
|
+
...(secondDown && { fallbackDown: true }),
|
|
1773
|
+
// Dry runs carry the NAMES of queued keys per reason, so agents can
|
|
1774
|
+
// read the plan from the --json summary instead of re-deriving the
|
|
1775
|
+
// diff. Omitted on real runs — per-key outcomes are reported there.
|
|
1776
|
+
...(dryRun && {
|
|
1777
|
+
queuedKeys: {
|
|
1778
|
+
missing: queueReasons(shown).missing.map(nsKey),
|
|
1779
|
+
fallback: queueReasons(shown).needsTranslation.map(nsKey),
|
|
1780
|
+
untranslated: queueReasons(shown).untranslated.map(nsKey),
|
|
1781
|
+
changed: queueReasons(shown).changed.map(nsKey),
|
|
1782
|
+
forced: queueReasons(shown).forced.filter(k => !plan.pendingRetry.includes(k) && !gapAsked.includes(k)).map(nsKey),
|
|
1783
|
+
gaps: gapAsked.map(nsKey),
|
|
1784
|
+
noTranslate: diff.noTranslate.map(nsKey),
|
|
1785
|
+
pending: plan.pendingRetry.map(nsKey),
|
|
1786
|
+
held: plan.held.map(nsKey),
|
|
1787
|
+
kept: plan.kept.map(nsKey),
|
|
1788
|
+
},
|
|
1789
|
+
}),
|
|
1790
|
+
};
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1793
|
+
/**
|
|
1794
|
+
* The cache key each pair has as the config FILE sets it — without this
|
|
1795
|
+
* run's --method / --model — or null when neither flag was given.
|
|
1796
|
+
*
|
|
1797
|
+
* A CI job that runs `sync --method llm --model X` over a project whose
|
|
1798
|
+
* config names a local model has not changed the project's setup: the files
|
|
1799
|
+
* keep what the configured setup wrote, by design. Every such run printed,
|
|
1800
|
+
* per language, that the files were "written by local, not by llm" and
|
|
1801
|
+
* offered a paid `--redo all` (Round 12, Django persona). Text the configured
|
|
1802
|
+
* setup wrote is said once, quietly, as kept; a real change of the config
|
|
1803
|
+
* keeps its full note.
|
|
1804
|
+
*
|
|
1805
|
+
* Resolved exactly as the runtime resolves pairs (resolvePairs + plugins),
|
|
1806
|
+
* from the values resolveConfig recorded before the flags applied.
|
|
1807
|
+
*
|
|
1808
|
+
* @returns {Map<string, string>|null} pairKey → tmMethodKey
|
|
1809
|
+
*/
|
|
1810
|
+
function configuredPairKeys(config, cwd, cliArgs) {
|
|
1811
|
+
if (!cliArgs.method && !cliArgs.model) return null;
|
|
1812
|
+
const own = {
|
|
1813
|
+
...config,
|
|
1814
|
+
defaultMethod: config._fileDefaultMethod ?? config.defaultMethod,
|
|
1815
|
+
model: config._fileModel,
|
|
1816
|
+
_modelExplicit: !!config._fileModelExplicit,
|
|
1817
|
+
};
|
|
1818
|
+
delete own._methodOverride;
|
|
1819
|
+
delete own._modelOverride;
|
|
1820
|
+
if (!own.resolvedLanguages || Object.keys(own.resolvedLanguages).length === 0) {
|
|
1821
|
+
own.resolvedLanguages = autoDetectLanguages(own);
|
|
1822
|
+
}
|
|
1823
|
+
try {
|
|
1824
|
+
const plugins = loadPlugins(cwd);
|
|
1825
|
+
const keys = new Map();
|
|
1826
|
+
for (const [pairKey, raw] of resolvePairs(own, { cwd })) keys.set(pairKey, tmMethodKey(resolvePluginForPair(plugins, raw)));
|
|
1827
|
+
return keys;
|
|
1828
|
+
} catch {
|
|
1829
|
+
// Unresolvable without the flags: every note stays the full one.
|
|
1830
|
+
return null;
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
/** "local · model m1": the part of a cache key --method / --model change. */
|
|
1835
|
+
function flagSetupOf(methodKey) {
|
|
1836
|
+
const [method, model] = String(methodKey).split('|');
|
|
1837
|
+
return model ? `${method || 'llm'} · model ${model}` : (method || 'llm');
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
/**
|
|
1841
|
+
* The exit code a real run would end with, from what a dry run knows
|
|
1842
|
+
* (lib/commands/sync.js computeExitCode has the real rules):
|
|
1843
|
+
* 1 — the preflight would stop it (a key or a model server it needs), or a
|
|
1844
|
+
* key named for a redo matches nothing;
|
|
1845
|
+
* 2 — --max-cost would stop it before any API call, keys are held back
|
|
1846
|
+
* (refused before), or plural messages on disk lack a form the
|
|
1847
|
+
* language uses and this run does not ask for them again;
|
|
1848
|
+
* 0 — none of those. A refusal by the quality gate or a failed
|
|
1849
|
+
* verification, which only the real run can find, can still make it 2.
|
|
1850
|
+
*
|
|
1851
|
+
* @returns {{ exitCode: 0|1|2, wouldStop: boolean, reasons: string[] }}
|
|
1852
|
+
*/
|
|
1853
|
+
function predictRealRun({ preflightFailures, dryMaxCost, unmatched, pluralGaps, gapsAskedAgain, costEstimate }) {
|
|
1854
|
+
const reasons = [];
|
|
1855
|
+
if (preflightFailures.length > 0) {
|
|
1856
|
+
reasons.push(`it would stop before translating: ${preflightStopReason(preflightFailures)}`);
|
|
1857
|
+
return { exitCode: 1, wouldStop: true, reasons };
|
|
1858
|
+
}
|
|
1859
|
+
if (unmatched > 0) {
|
|
1860
|
+
reasons.push(`${unmatched} key(s) named for a redo match nothing`);
|
|
1861
|
+
return { exitCode: 1, wouldStop: false, reasons };
|
|
1862
|
+
}
|
|
1863
|
+
if (dryMaxCost?.wouldStop) {
|
|
1864
|
+
// With the figures: a CI log that prints this reason says how far over.
|
|
1865
|
+
const est = typeof dryMaxCost.estimatedCost === 'number' ? `~$${dryMaxCost.estimatedCost.toFixed(4)}` : 'unknown';
|
|
1866
|
+
reasons.push(`--max-cost would stop it before any API call (${String(dryMaxCost.reason || 'over the cap').replace(/\.$/, '')}: `
|
|
1867
|
+
+ `estimate ${est}, cap $${Number(dryMaxCost.cap).toFixed(4)})`);
|
|
1868
|
+
return { exitCode: 2, wouldStop: true, reasons };
|
|
1869
|
+
}
|
|
1870
|
+
const held = (costEstimate?.pairs || []).reduce((n, p) => n + (p.held || 0), 0);
|
|
1871
|
+
if (held > 0) reasons.push(`${held} key(s) held back — refused before, not sent again`);
|
|
1872
|
+
if (pluralGaps > 0) {
|
|
1873
|
+
reasons.push(`${pluralGaps} plural message(s) on disk lack a form the language uses for ordinary counts, and this run does not ask for them again`);
|
|
1874
|
+
}
|
|
1875
|
+
if (reasons.length === 0 && gapsAskedAgain > 0) {
|
|
1876
|
+
// Asked again: 0 when the model now supplies the forms, 2 when it does not.
|
|
1877
|
+
return { exitCode: 0, wouldStop: false, reasons: [`${gapsAskedAgain} plural message(s) are asked for again — the real run exits 2 if the answer lacks the forms too`] };
|
|
1878
|
+
}
|
|
1879
|
+
return { exitCode: reasons.length > 0 ? 2 : 0, wouldStop: false, reasons };
|
|
1880
|
+
}
|
|
1881
|
+
|
|
1882
|
+
/**
|
|
1883
|
+
* The line before the per-locale work: what the run is about to do. It
|
|
1884
|
+
* said "Translating 2 locale(s) with concurrency 50" on a dry run and on a
|
|
1885
|
+
* run with nothing to send, which reads as model calls about to happen
|
|
1886
|
+
* (Round 14, Next.js persona). "Translating" is said only when the
|
|
1887
|
+
* estimate shows something will be sent — and of how many locales; a dry
|
|
1888
|
+
* run and a run that sends nothing say they are checking. With no estimate
|
|
1889
|
+
* (it failed) what will be sent is not known, and the line says so.
|
|
1890
|
+
*
|
|
1891
|
+
* @param {{ dryRun: boolean, pairEntries: Array<[string, object]>, costEstimate: object|null,
|
|
1892
|
+
* contentByTarget: Record<string, { billedChars?: number }>, concurrency: number }} args
|
|
1893
|
+
* @returns {string}
|
|
1894
|
+
*/
|
|
1895
|
+
function describeLocaleWork({ dryRun, pairEntries, costEstimate, contentByTarget, concurrency }) {
|
|
1896
|
+
const n = pairEntries.length;
|
|
1897
|
+
if (dryRun) return `Checking ${n} locale(s) — a dry run: nothing is sent to a model, nothing is written`;
|
|
1898
|
+
if (!costEstimate) return `Translating up to ${n} locale(s) with concurrency ${concurrency} (no estimate: what is sent is not known in advance)`;
|
|
1899
|
+
const keysFor = new Map((costEstimate.pairs || []).map(p => [p.pair, p.keys || 0]));
|
|
1900
|
+
const sending = pairEntries.filter(([pairKey, pc]) => (keysFor.get(pairKey) || 0) > 0
|
|
1901
|
+
|| (contentByTarget?.[pc.target]?.billedChars || 0) > 0).length;
|
|
1902
|
+
if (sending === 0) return `Checking ${n} locale(s) — nothing to send to a model this run`;
|
|
1903
|
+
return sending === n
|
|
1904
|
+
? `Translating ${n} locale(s) with concurrency ${concurrency}`
|
|
1905
|
+
: `Translating ${sending} of ${n} locale(s) with concurrency ${concurrency} (the other ${n - sending}: nothing to send)`;
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1908
|
+
/** What `--prune` may remove (lib/plurals.js pluralExtraKeys). */
|
|
1909
|
+
const PRUNE_SCOPES = ['plural-extras'];
|
|
1910
|
+
|
|
1911
|
+
/**
|
|
1912
|
+
* --prune values → the set of removals asked for. Unknown values fail loud
|
|
1913
|
+
* (a typo must not read as "nothing to prune").
|
|
1914
|
+
*
|
|
1915
|
+
* @param {string[]|string|boolean|undefined} raw
|
|
1916
|
+
* @returns {Set<string>}
|
|
1917
|
+
*/
|
|
1918
|
+
function parsePrune(raw) {
|
|
1919
|
+
const out = new Set();
|
|
1920
|
+
if (raw === undefined) return out;
|
|
1921
|
+
const values = [].concat(raw);
|
|
1922
|
+
if (values.some(v => typeof v !== 'string' || v.trim() === '')) {
|
|
1923
|
+
throw new Error(`--prune needs what to remove: ${PRUNE_SCOPES.join(' | ')} (e.g. --prune plural-extras).`);
|
|
1924
|
+
}
|
|
1925
|
+
for (const v of values.flatMap(x => x.split(',')).map(x => x.trim()).filter(Boolean)) {
|
|
1926
|
+
if (!PRUNE_SCOPES.includes(v)) {
|
|
1927
|
+
throw new Error(`--prune ${v}: unknown. Use --prune plural-extras (removes i18next plural keys for forms a language does not have — nothing else).`);
|
|
1928
|
+
}
|
|
1929
|
+
out.add(v);
|
|
1930
|
+
}
|
|
1931
|
+
return out;
|
|
246
1932
|
}
|
|
247
1933
|
|
|
248
1934
|
/**
|
|
@@ -254,9 +1940,32 @@ async function runSync(options = {}) {
|
|
|
254
1940
|
const { dryRun = false, audit = false, cwd = process.cwd(), cliArgs = {} } = options;
|
|
255
1941
|
const config = resolveConfig(cliArgs, cwd);
|
|
256
1942
|
|
|
1943
|
+
// --show-prompt: shows the request a real run would send, so only in a
|
|
1944
|
+
// dry run (which sends nothing). A bare flag before another flag is read by
|
|
1945
|
+
// the argument parser as its value ("--show-prompt --dry"): say so.
|
|
1946
|
+
if (cliArgs['show-prompt']) {
|
|
1947
|
+
if (typeof cliArgs['show-prompt'] === 'string' && cliArgs['show-prompt'].startsWith('--')) {
|
|
1948
|
+
throw new Error(`--show-prompt took "${cliArgs['show-prompt']}" as the key to show. Put --show-prompt last, `
|
|
1949
|
+
+ 'or give it a key: sync --dry --show-prompt <key>.');
|
|
1950
|
+
}
|
|
1951
|
+
if (!dryRun) {
|
|
1952
|
+
throw new Error('--show-prompt shows the request a run would send, without sending it: add --dry '
|
|
1953
|
+
+ '(sync --dry --show-prompt [key]).');
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
|
|
257
1957
|
// --max-cost: parse eagerly so a malformed cap fails loud up front,
|
|
258
1958
|
// before anything (even read-only work) happens.
|
|
259
1959
|
const maxCost = parseMaxCost(cliArgs['max-cost']);
|
|
1960
|
+
// --prune <what>: removals a run makes only when asked; --redo gaps: the
|
|
1961
|
+
// plural messages a model left incomplete, asked again. Both act on
|
|
1962
|
+
// key-value locale files (i18next plural keys, ICU and gettext plurals).
|
|
1963
|
+
const prune = parsePrune(cliArgs.prune);
|
|
1964
|
+
const redoGaps = !!cliArgs['redo-gaps'];
|
|
1965
|
+
if (config.format === 'docusaurus' && (prune.size > 0 || redoGaps)) {
|
|
1966
|
+
throw new Error(`${redoGaps ? '--redo gaps' : '--prune plural-extras'} acts on plural messages in key-value locale files `
|
|
1967
|
+
+ '(i18next plural keys, ICU and gettext plurals); a Docusaurus project\'s translation files have none. Nothing was changed.');
|
|
1968
|
+
}
|
|
260
1969
|
|
|
261
1970
|
// Clear any translation failure recorded by a prior in-process sync (watch
|
|
262
1971
|
// mode) so getSetupHelp() reflects THIS run's failure, not a stale one.
|
|
@@ -271,9 +1980,16 @@ async function runSync(options = {}) {
|
|
|
271
1980
|
return runDocusaurusSync(options, config, cwd, resolveRuntime);
|
|
272
1981
|
}
|
|
273
1982
|
|
|
274
|
-
// Verify locales directory exists
|
|
1983
|
+
// Verify locales directory exists. The hint names what `init` would find
|
|
1984
|
+
// on disk (messages/en.json, public/locales/en/…) — a project that never
|
|
1985
|
+
// ran init otherwise gets only "not found" for the default ./locales.
|
|
275
1986
|
if (!fs.existsSync(config.localesDir)) {
|
|
276
|
-
|
|
1987
|
+
const { describeLocaleSetupHint } = await import('./commands/init.js');
|
|
1988
|
+
const hint = describeLocaleSetupHint(cwd, config.inputLocale);
|
|
1989
|
+
throw new Error(
|
|
1990
|
+
`Locales directory not found: ${config.localesDir}. `
|
|
1991
|
+
+ (hint || 'Create it or set "localesDir" in your config file (`champollion init` detects common layouts).'),
|
|
1992
|
+
);
|
|
277
1993
|
}
|
|
278
1994
|
|
|
279
1995
|
// --- Version banner ---
|
|
@@ -281,48 +1997,82 @@ async function runSync(options = {}) {
|
|
|
281
1997
|
const { version } = require('../package.json');
|
|
282
1998
|
output.banner(version);
|
|
283
1999
|
|
|
284
|
-
//
|
|
285
|
-
//
|
|
2000
|
+
// Locale layout — WHERE every locale's files are (lib/locale-layout.js):
|
|
2001
|
+
// flat <dir>/<code>.<ext>, dir <dir>/<code>/<ns>.<ext>, or a configured
|
|
2002
|
+
// localesPattern. The format rides on each file's REAL extension, so a
|
|
2003
|
+
// `.yml` project writes `.yml`. CLI flag beats config beats detection.
|
|
2004
|
+
const layout = discoverLocaleLayout(config, { cwd });
|
|
286
2005
|
const isAutoFormat = config.format === 'auto';
|
|
287
|
-
const format =
|
|
288
|
-
? detectFormatFromDir(config.localesDir)
|
|
289
|
-
: config.format;
|
|
290
|
-
const ext = getExtension(format);
|
|
2006
|
+
const format = layout.format;
|
|
291
2007
|
output.info(`Detected format: ${format} (${isAutoFormat ? 'auto' : 'config'})`);
|
|
2008
|
+
if (layout.kind !== 'flat') {
|
|
2009
|
+
output.info(`Locale layout: ${layout.kind} — ${layout.display}`);
|
|
2010
|
+
}
|
|
2011
|
+
for (const rel of layout.ignored) {
|
|
2012
|
+
output.warn(`Ignoring ${rel} — the source locale's files are ${format}; this file is a different format and is not synced.`);
|
|
2013
|
+
}
|
|
292
2014
|
|
|
293
|
-
//
|
|
2015
|
+
// Content directory (Docusaurus was dispatched above). Hugo is named only
|
|
2016
|
+
// on real Hugo evidence (lib/content.js detectContentSite) — a plain folder
|
|
2017
|
+
// of Markdown is a Markdown folder. The naming rule is the same for both.
|
|
294
2018
|
if (config.contentDir) {
|
|
295
|
-
|
|
296
|
-
|
|
2019
|
+
const site = detectContentSite(config.contentDir, cwd);
|
|
2020
|
+
const shownDir = path.relative(cwd, config.contentDir) || '.';
|
|
2021
|
+
if (site.site === 'hugo') {
|
|
2022
|
+
output.info(`Detected framework: Hugo (${site.evidence})`);
|
|
2023
|
+
output.info(`Content directory: ${shownDir} — each translation is written beside its source as <name>.<locale>.md (Hugo's translation-by-filename)`);
|
|
2024
|
+
} else {
|
|
2025
|
+
output.info(`Content directory: ${shownDir} — a folder of Markdown/MDX files (no Hugo site found); each translation is written beside its source as <name>.<locale>.md`);
|
|
2026
|
+
}
|
|
297
2027
|
}
|
|
298
2028
|
|
|
299
2029
|
const inputLocale = config.inputLocale;
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
//
|
|
312
|
-
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
// Defense-in-depth: remove any keys that could cause prototype pollution.
|
|
318
|
-
// Extremely unlikely in real locale files but important for a public package.
|
|
319
|
-
for (const key of Object.keys(sourceFlat)) {
|
|
320
|
-
if (isUnsafeKey(key)) {
|
|
321
|
-
delete sourceFlat[key];
|
|
2030
|
+
// One unit per source file (flat layouts: exactly one). Each carries its
|
|
2031
|
+
// flat map (JSON flattened, unsafe keys removed), YAML style and i18next
|
|
2032
|
+
// plural groups. Throws "Source locale not found: …" when missing.
|
|
2033
|
+
const units = loadSourceUnits(layout);
|
|
2034
|
+
// "en.json" for one file; "en/ (3 files, …)" for a folder of namespaces.
|
|
2035
|
+
const sourceLabel = (keyCount) => (layout.namespaced
|
|
2036
|
+
? `${layout.kind === 'dir' ? `${inputLocale}/` : layout.display} (${units.length} file(s), ${keyCount} keys)`
|
|
2037
|
+
: `${units[0].file.rel} (${keyCount} keys)`);
|
|
2038
|
+
|
|
2039
|
+
// The source in the SHARED key space: bare keys for single-file layouts
|
|
2040
|
+
// (so a flat project's lock file is exactly what it was), `<ns>::<key>`
|
|
2041
|
+
// for namespaced ones — two files may both define "title".
|
|
2042
|
+
const sourceLock = {};
|
|
2043
|
+
for (const unit of units) {
|
|
2044
|
+
for (const [key, value] of Object.entries(unit.flat)) {
|
|
2045
|
+
sourceLock[lockKey(layout, unit.ns, key)] = value;
|
|
322
2046
|
}
|
|
323
2047
|
}
|
|
2048
|
+
const sourceKeyCount = Object.keys(sourceLock).length;
|
|
2049
|
+
|
|
2050
|
+
const pluralFiles = units.filter(u => u.pluralGroups.size > 0);
|
|
2051
|
+
if (pluralFiles.length > 0) {
|
|
2052
|
+
const groupCount = pluralFiles.reduce((n, u) => n + u.pluralGroups.size, 0);
|
|
2053
|
+
// What that means for THIS run's targets, read from CLDR (never a fixed
|
|
2054
|
+
// example: "French adds _many" was printed for a Spanish-only project).
|
|
2055
|
+
const targets = new Set(Object.keys(config.resolvedLanguages || {}));
|
|
2056
|
+
for (const k of Object.keys(config.pairs || {})) { const t = parsePairKey(k).target; if (t) targets.add(t); }
|
|
2057
|
+
const changes = describePluralFormChanges(inputLocale, [...targets]);
|
|
2058
|
+
output.info(`i18next plural keys: ${groupCount} group(s) — each locale gets its own CLDR plural forms${changes ? `: ${changes}` : ''}.`);
|
|
2059
|
+
}
|
|
324
2060
|
|
|
325
|
-
|
|
2061
|
+
// Keys named for a redo must exist: a name that matches nothing fails the
|
|
2062
|
+
// run (exit 1) with the closest keys, instead of a silent "0 keys". With
|
|
2063
|
+
// some names matching, those are redone and the run then fails naming the
|
|
2064
|
+
// rest; with none, nothing runs (lib/named-keys.js — the Docusaurus path
|
|
2065
|
+
// applies the same rule).
|
|
2066
|
+
const unmatchedNamed = applyNamedKeyRule({
|
|
2067
|
+
cliArgs, config,
|
|
2068
|
+
known: () => {
|
|
2069
|
+
const targets = new Set(Object.keys(config.resolvedLanguages || {}));
|
|
2070
|
+
for (const k of Object.keys(config.pairs || {})) { const t = parsePairKey(k).target; if (t) targets.add(t); }
|
|
2071
|
+
return namedKeySpace({ layout, units, inputLocale, targets: [...targets] });
|
|
2072
|
+
},
|
|
2073
|
+
bare: (k) => bareNamedKey(layout, k),
|
|
2074
|
+
bareMatches: layout.namespaced,
|
|
2075
|
+
});
|
|
326
2076
|
|
|
327
2077
|
// --force: re-queue EVERY string key — the whole-locale rebuild verb.
|
|
328
2078
|
// Recovering from a bad version is exactly when someone needs this, and
|
|
@@ -332,16 +2082,48 @@ async function runSync(options = {}) {
|
|
|
332
2082
|
// a fully fresh re-bill). Set BEFORE the cost estimator so the preview
|
|
333
2083
|
// prices the full rebuild, and --max-cost can cap it.
|
|
334
2084
|
if (cliArgs.force) {
|
|
335
|
-
config.forceKeys = Object.keys(
|
|
336
|
-
output.info(
|
|
2085
|
+
config.forceKeys = Object.keys(sourceLock).filter(k => typeof sourceLock[k] === 'string');
|
|
2086
|
+
output.info(`${cliArgs.redo ? '--redo all' : '--force'}: re-queuing all ${config.forceKeys.length} source key(s)${cliArgs.pair ? ' for the selected pair(s)' : ''}`);
|
|
337
2087
|
}
|
|
338
2088
|
|
|
339
2089
|
// Load the hash manifest and detect which English values changed
|
|
340
2090
|
// since the last sync. On first run (no manifest), this returns []
|
|
341
2091
|
// and everything flows through the normal missing-key detection.
|
|
342
|
-
const
|
|
343
|
-
const
|
|
344
|
-
|
|
2092
|
+
const lock = readLock(cwd);
|
|
2093
|
+
const oldManifest = lock.source;
|
|
2094
|
+
// Per-locale record (lib/locale-state.js): written values, pending redo
|
|
2095
|
+
// keys, refusals. Mutated by the run; written with the manifest.
|
|
2096
|
+
const lockState = new LockState(lock.locales);
|
|
2097
|
+
// No lock, but translations already on disk (a fresh clone of a repo that
|
|
2098
|
+
// never committed it, or a deleted lock): sync cannot tell which of them
|
|
2099
|
+
// are out of date, so it fills missing keys and keeps every existing
|
|
2100
|
+
// translation as it is. That used to happen in silence — say so once.
|
|
2101
|
+
if (!fs.existsSync(path.join(cwd, LOCK_FILENAME))) {
|
|
2102
|
+
const translated = [];
|
|
2103
|
+
for (const code of Object.keys(config.resolvedLanguages || {})) {
|
|
2104
|
+
for (const unit of units) {
|
|
2105
|
+
let file;
|
|
2106
|
+
try { file = layout.fileFor(code, unit.ns); } catch { continue; }
|
|
2107
|
+
if (!file || !fs.existsSync(file.path)) continue;
|
|
2108
|
+
try {
|
|
2109
|
+
if (Object.keys(readLocaleFlat(file) || {}).length > 0) translated.push(file.rel);
|
|
2110
|
+
} catch { /* unreadable targets are reported where they are synced */ }
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
if (translated.length > 0) {
|
|
2114
|
+
output.warn(`No ${LOCK_FILENAME}, but ${translated.length} target file(s) already hold translations `
|
|
2115
|
+
+ `(${translated.slice(0, 3).join(', ')}${translated.length > 3 ? ', …' : ''}). Without it, sync cannot tell `
|
|
2116
|
+
+ 'which of them are out of date: it fills missing keys and keeps the rest as they are. If the source '
|
|
2117
|
+
+ 'changed since they were made, run `champollion sync --redo all` once (cached translations are free), '
|
|
2118
|
+
+ `then commit ${LOCK_FILENAME}.`);
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
const changedKeys = detectChangedKeys(sourceLock, oldManifest);
|
|
2122
|
+
const currentManifest = buildHashManifest(sourceLock);
|
|
2123
|
+
// Each unit diffs in its own (un-namespaced) key space.
|
|
2124
|
+
for (const unit of units) {
|
|
2125
|
+
unit.changedKeys = keysForNamespace(layout, changedKeys, unit.ns);
|
|
2126
|
+
}
|
|
345
2127
|
|
|
346
2128
|
// No-translate matcher — the ONE compiled instance for this run. The cost
|
|
347
2129
|
// estimator and every locale's diff share it, so the keys excluded from the
|
|
@@ -354,15 +2136,39 @@ async function runSync(options = {}) {
|
|
|
354
2136
|
// Resolve the pair graph via the shared helper.
|
|
355
2137
|
// Thread dryRun/audit into cliArgs so the preflight check can skip
|
|
356
2138
|
// for read-only operations that don't need an API key.
|
|
357
|
-
|
|
2139
|
+
// deferUnreachable: a model server that does not answer is judged after
|
|
2140
|
+
// the plan below (resolveDeferredProbes) — a run that sends it nothing
|
|
2141
|
+
// does not need it.
|
|
2142
|
+
const runtime = await resolveRuntime(config, cwd, { ...cliArgs, dryRun, audit, deferUnreachable: true });
|
|
2143
|
+
const { apiKey, resolvedPairs, pairEntries, deferredProbeFailures } = runtime;
|
|
2144
|
+
let { preflightFailures } = runtime;
|
|
358
2145
|
|
|
359
2146
|
// Provenance check — warn about uncleared licensing before sync starts.
|
|
360
2147
|
// This is informational only (does not block execution).
|
|
361
|
-
|
|
2148
|
+
// Fallback methods translate too, so their licensing is audited the same way.
|
|
2149
|
+
const auditedRoutes = new Map(resolvedPairs);
|
|
2150
|
+
for (const [key, pc] of resolvedPairs) {
|
|
2151
|
+
if (pc.fallback) auditedRoutes.set(`${key} (fallback)`, pc.fallback);
|
|
2152
|
+
}
|
|
2153
|
+
const provenanceAudit = auditProvenance(auditedRoutes);
|
|
2154
|
+
// A project's first sync has no lock file yet: that is when the
|
|
2155
|
+
// "runs a model you choose" fact is news. After that it repeated on every
|
|
2156
|
+
// run (Round 3, Next.js persona); `status` and `provenance` keep saying it.
|
|
2157
|
+
const firstSync = !fs.existsSync(path.join(cwd, LOCK_FILENAME));
|
|
362
2158
|
if (!provenanceAudit.allClear) {
|
|
363
2159
|
for (const blockedKey of provenanceAudit.blockedPairs) {
|
|
364
|
-
const blockedPair =
|
|
365
|
-
|
|
2160
|
+
const blockedPair = auditedRoutes.get(blockedKey);
|
|
2161
|
+
// A model the user runs themselves (local server, own api endpoint,
|
|
2162
|
+
// external plugin): its licence is theirs to know, not ours to verify.
|
|
2163
|
+
// That is a fact to state once, not a warning on every run (Round 2).
|
|
2164
|
+
if (['local', 'api', 'external'].includes(blockedPair.method)) {
|
|
2165
|
+
if (firstSync) {
|
|
2166
|
+
output.info(`${blockedKey}: "${blockedPair.method}" runs a model you choose — Champollion cannot check its licence; `
|
|
2167
|
+
+ 'make sure your use of it is allowed. (Said once; `champollion status` repeats it.)');
|
|
2168
|
+
}
|
|
2169
|
+
} else {
|
|
2170
|
+
output.warn(`${blockedKey}: Method "${blockedPair.method}" has unverified licensing. Run \`champollion provenance\` for details.`);
|
|
2171
|
+
}
|
|
366
2172
|
}
|
|
367
2173
|
}
|
|
368
2174
|
|
|
@@ -375,73 +2181,167 @@ async function runSync(options = {}) {
|
|
|
375
2181
|
if (audit) {
|
|
376
2182
|
output.info('Audit: scanning for untranslated values...');
|
|
377
2183
|
let total = 0;
|
|
2184
|
+
let gapTotal = 0;
|
|
2185
|
+
const pairKeyOf = (code) => (pairEntries.find(([, pc]) => pc.target === code) || [`${inputLocale}:${code}`])[0];
|
|
2186
|
+
const shownKeyName = (k) => String(k).replace(/\u0004/g, '\u2404');
|
|
378
2187
|
const auditLocales = [];
|
|
379
2188
|
const missingLocales = [];
|
|
380
2189
|
for (const [, pairConfig] of pairEntries) {
|
|
381
2190
|
const code = pairConfig.target;
|
|
382
|
-
const
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
2191
|
+
for (const unit of units) {
|
|
2192
|
+
const file = layout.fileFor(code, unit.ns);
|
|
2193
|
+
const filename = file.rel;
|
|
2194
|
+
if (!fs.existsSync(file.path)) {
|
|
2195
|
+
// A configured locale with NO file is 100% untranslated, not "fully
|
|
2196
|
+
// translated". Skipping it silently let an audit wired as a CI gate
|
|
2197
|
+
// pass with zero translation done — every source key counts as
|
|
2198
|
+
// untranslated and the run must exit non-zero.
|
|
2199
|
+
const expectedKeys = Object.keys(expectedForTarget(unit, inputLocale, code).flat);
|
|
2200
|
+
if (expectedKeys.length > 0) {
|
|
2201
|
+
missingLocales.push(filename);
|
|
2202
|
+
auditLocales.push({
|
|
2203
|
+
locale: code,
|
|
2204
|
+
file: filename,
|
|
2205
|
+
missing: true,
|
|
2206
|
+
untranslatedCount: expectedKeys.length,
|
|
2207
|
+
untranslatedKeys: expectedKeys.map(k => lockKey(layout, unit.ns, k)),
|
|
2208
|
+
});
|
|
2209
|
+
output.error(`${filename}: locale file missing — all ${expectedKeys.length} key(s) untranslated. Run \`champollion sync\` to create it.`);
|
|
2210
|
+
total += expectedKeys.length;
|
|
2211
|
+
}
|
|
2212
|
+
continue;
|
|
400
2213
|
}
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
2214
|
+
const flat = readLocaleFlat(file);
|
|
2215
|
+
// Untranslated = an [EN] fallback, OR a key the target should have
|
|
2216
|
+
// but does not (absent, or empty — an empty msgstr, a new msgid).
|
|
2217
|
+
// Only fallbacks used to count, so an audit wired as the CI gate
|
|
2218
|
+
// passed catalogs with keys missing (synthetic review, 2026-10-03).
|
|
2219
|
+
const expected = expectedForTarget(unit, inputLocale, code).flat;
|
|
2220
|
+
const fallbacks = Object.entries(flat)
|
|
2221
|
+
.filter(([, val]) => typeof val === 'string' && val.startsWith(config.fallbackPrefix));
|
|
2222
|
+
const absent = Object.keys(expected)
|
|
2223
|
+
.filter((k) => !(k in flat) || (typeof flat[k] === 'string' && flat[k].trim() === ''))
|
|
2224
|
+
.map((k) => [k, flat[k]]);
|
|
2225
|
+
const untranslated = [...fallbacks, ...absent];
|
|
2226
|
+
// Plural messages without a form the language uses for ordinary
|
|
2227
|
+
// counts — what `verify --strict` fails on — are not "fully
|
|
2228
|
+
// translated" either (Round 7, Django persona).
|
|
2229
|
+
const gone = new Set(untranslated.map(([k]) => k));
|
|
2230
|
+
const gaps = pluralGapsInFile({ file, expected, targetFlat: flat, locale: code }).filter(g => !gone.has(g.key));
|
|
2231
|
+
auditLocales.push({
|
|
2232
|
+
locale: code,
|
|
2233
|
+
file: filename,
|
|
2234
|
+
untranslatedCount: untranslated.length,
|
|
2235
|
+
untranslatedKeys: untranslated.map(([key]) => lockKey(layout, unit.ns, key)),
|
|
2236
|
+
...(gaps.length > 0 && {
|
|
2237
|
+
pluralGapCount: gaps.length,
|
|
2238
|
+
pluralGaps: gaps.map(g => ({ key: lockKey(layout, unit.ns, g.key), missing: g.missing })),
|
|
2239
|
+
}),
|
|
2240
|
+
});
|
|
2241
|
+
if (untranslated.length > 0) {
|
|
2242
|
+
output.raw(` ${filename}: ${untranslated.length} keys still need translation`);
|
|
2243
|
+
for (const [key] of untranslated) {
|
|
2244
|
+
output.raw(` - ${key}`);
|
|
2245
|
+
}
|
|
2246
|
+
total += untranslated.length;
|
|
2247
|
+
}
|
|
2248
|
+
if (gaps.length > 0) {
|
|
2249
|
+
output.raw(` ${filename}: ${gaps.length} plural message(s) without a form ${code} uses for ordinary counts — the "other" form is shown instead`);
|
|
2250
|
+
for (const g of gaps) output.raw(` - ${shownKeyName(lockKey(layout, unit.ns, g.key))} (${g.missing.join(', ')})`);
|
|
2251
|
+
const marked = gaps.some(g => g.marked) ? ` (or write the forms by hand and delete the "# champollion:" line above each entry)` : '';
|
|
2252
|
+
output.raw(` Repair: \`${redoCommand(gaps.map(g => g.key), { pair: pairKeyOf(code), ns: layout.namespaced ? unit.ns : '', fresh: true })}\`${marked}`);
|
|
2253
|
+
gapTotal += gaps.length;
|
|
417
2254
|
}
|
|
418
|
-
total += untranslated.length;
|
|
419
2255
|
}
|
|
420
2256
|
}
|
|
2257
|
+
// OUT OF DATE: a translation made from an older source text than the
|
|
2258
|
+
// current one (the lock's record — lib/locale-state.js). The file holds a
|
|
2259
|
+
// value, so the checks above pass it, but it says what the source USED
|
|
2260
|
+
// to say: a source edit whose re-translation failed left exactly this,
|
|
2261
|
+
// and audit used to call the locale "fully translated" (Round 4,
|
|
2262
|
+
// i18next persona).
|
|
2263
|
+
const tmForProof = loadTM(cwd);
|
|
2264
|
+
let staleTotal = 0;
|
|
2265
|
+
for (const [pairKey, pairConfig] of pairEntries) {
|
|
2266
|
+
const health = localeHealth({
|
|
2267
|
+
layout, units, inputLocale, code: pairConfig.target, localeState: lockState.peek(pairConfig.target),
|
|
2268
|
+
manifest: oldManifest, tm: tmForProof, pairConfig,
|
|
2269
|
+
helpers: { expectedForTarget, readLocaleFlat, lockKey, originKey, fallbackPrefix: config.fallbackPrefix },
|
|
2270
|
+
});
|
|
2271
|
+
if (health.stale.length === 0) continue;
|
|
2272
|
+
staleTotal += health.stale.length;
|
|
2273
|
+
const entry = auditLocales.find(l => l.locale === pairConfig.target && !l.missing);
|
|
2274
|
+
if (entry) entry.outOfDateKeys = [...(entry.outOfDateKeys || []), ...health.stale];
|
|
2275
|
+
else auditLocales.push({ locale: pairConfig.target, untranslatedCount: 0, untranslatedKeys: [], outOfDateKeys: health.stale });
|
|
2276
|
+
const heldStale = health.stale.filter(k => health.held.includes(k));
|
|
2277
|
+
output.raw(` ${pairConfig.target}: ${health.stale.length} translation(s) out of date — made from an older source text`);
|
|
2278
|
+
for (const key of health.stale) output.raw(` - ${key}`);
|
|
2279
|
+
output.raw(` Repair: \`champollion sync --pair ${pairKey}\` re-translates them`
|
|
2280
|
+
+ (heldStale.length > 0
|
|
2281
|
+
? `; ${heldStale.length} of them were refused before and are held back — name them: \`${redoCommand(heldStale.slice(0, 8), { pair: pairKey })}\``
|
|
2282
|
+
: '.'));
|
|
2283
|
+
}
|
|
2284
|
+
const gapPart = gapTotal > 0 ? `, ${gapTotal} plural message(s) missing everyday forms` : '';
|
|
421
2285
|
if (missingLocales.length > 0) {
|
|
422
|
-
output.raw(`\n Total: ${total} keys need translation (${missingLocales.length} locale file(s) missing: ${missingLocales.join(', ')}).`);
|
|
2286
|
+
output.raw(`\n Total: ${total} keys need translation${gapPart} (${missingLocales.length} locale file(s) missing: ${missingLocales.join(', ')}).`);
|
|
423
2287
|
} else {
|
|
424
|
-
output.raw(total === 0
|
|
425
|
-
? '\n All locale files are fully translated.'
|
|
426
|
-
: `\n Total: ${total} keys need translation.`);
|
|
2288
|
+
output.raw(total === 0 && staleTotal === 0 && gapTotal === 0
|
|
2289
|
+
? '\n All locale files are fully translated and up to date.'
|
|
2290
|
+
: `\n Total: ${total} keys need translation${staleTotal > 0 ? `, ${staleTotal} translation(s) out of date` : ''}${gapPart}.`);
|
|
427
2291
|
}
|
|
428
2292
|
// Machine-readable end-of-command summary — in --json mode the raw lines
|
|
429
2293
|
// above are suppressed, so the key list must ride the summary object.
|
|
430
2294
|
output.summary({
|
|
431
2295
|
command: 'audit',
|
|
432
2296
|
untranslatedCount: total,
|
|
2297
|
+
outOfDateCount: staleTotal,
|
|
2298
|
+
pluralGapCount: gapTotal,
|
|
433
2299
|
missingLocales,
|
|
434
2300
|
locales: auditLocales,
|
|
435
2301
|
});
|
|
436
|
-
return { untranslatedCount: total, missingLocaleCount: missingLocales.length };
|
|
2302
|
+
return { untranslatedCount: total, outOfDateCount: staleTotal, pluralGapCount: gapTotal, missingLocaleCount: missingLocales.length };
|
|
437
2303
|
}
|
|
438
2304
|
|
|
439
2305
|
// --- Sync mode ---
|
|
440
|
-
const methodSummary = pairEntries
|
|
441
|
-
|
|
2306
|
+
const methodSummary = pairEntries
|
|
2307
|
+
.map(([, p]) => `${p.target}:${p.method}${p.fallback ? ` (fallback: ${p.fallback.method})` : ''}`)
|
|
2308
|
+
.join(', ');
|
|
2309
|
+
output.info(`Source: ${sourceLabel(sourceKeyCount)}`);
|
|
442
2310
|
output.info(`Pairs: ${methodSummary}`);
|
|
2311
|
+
// --model / --method name the model or method for THIS run; the file is
|
|
2312
|
+
// not changed, and the next plain sync uses what it says (Round 10,
|
|
2313
|
+
// Next.js persona: the help called --model an "override" without saying
|
|
2314
|
+
// for how long, and the next sync then named the run's model as another).
|
|
2315
|
+
{
|
|
2316
|
+
const file = cliArgs.config ? path.basename(String(cliArgs.config)) : 'champollion.config.json';
|
|
2317
|
+
const oneRun = [];
|
|
2318
|
+
const fields = [];
|
|
2319
|
+
if (config._modelOverride && config._modelOverride !== config._fileModel) {
|
|
2320
|
+
oneRun.push(`--model ${config._modelOverride} (the config says ${config._fileModel ? `"model": "${config._fileModel}"` : 'no "model": each method uses its default'})`);
|
|
2321
|
+
fields.push('"model"');
|
|
2322
|
+
}
|
|
2323
|
+
if (config._methodOverride && config._methodOverride !== (config._fileDefaultMethod || 'llm')) {
|
|
2324
|
+
oneRun.push(`--method ${config._methodOverride} (the config says "defaultMethod": "${config._fileDefaultMethod || 'llm'}")`);
|
|
2325
|
+
fields.push('"defaultMethod"');
|
|
2326
|
+
}
|
|
2327
|
+
if (oneRun.length > 0) {
|
|
2328
|
+
output.info(`${oneRun.join(' and ')}: for this run only — ${file} is not changed, and a plain \`champollion sync\` uses it again. `
|
|
2329
|
+
+ `To switch for good, edit ${fields.join(' and ')} in ${file}.`);
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
// Named when there are few; otherwise the flag that names them (Round 9,
|
|
2333
|
+
// Next.js persona: "Changed: 1 key(s)" named nothing, and nothing said
|
|
2334
|
+
// that --list-keys would).
|
|
443
2335
|
if (changedKeys.length > 0) {
|
|
444
|
-
|
|
2336
|
+
const shownKeys = changedKeys.map(k => String(k).replace(/\u0004/g, '\u2404'));
|
|
2337
|
+
if (changedKeys.length <= 5) {
|
|
2338
|
+
output.info(`Changed: ${changedKeys.length} key(s) have updated source content: ${shownKeys.join(', ')}`);
|
|
2339
|
+
} else {
|
|
2340
|
+
const how = dryRun
|
|
2341
|
+
? (cliArgs['list-keys'] ? 'each file lists its queued keys below' : 'add --list-keys to name every queued key')
|
|
2342
|
+
: '`champollion sync --dry --list-keys` names every queued key';
|
|
2343
|
+
output.info(`Changed: ${changedKeys.length} key(s) have updated source content (${shownKeys.slice(0, 3).join(', ')}, …) — ${how}`);
|
|
2344
|
+
}
|
|
445
2345
|
}
|
|
446
2346
|
if (dryRun) output.info('Dry-run mode — no files will be modified.');
|
|
447
2347
|
|
|
@@ -454,48 +2354,203 @@ async function runSync(options = {}) {
|
|
|
454
2354
|
// Loaded BEFORE the cost estimate so the estimator partitions against the
|
|
455
2355
|
// exact TM this run will use — TM hits are $0, not fresh API calls.
|
|
456
2356
|
//
|
|
457
|
-
// --no-tm
|
|
458
|
-
//
|
|
459
|
-
//
|
|
2357
|
+
// --fresh (alias --no-tm): nothing is served from the cache, so every
|
|
2358
|
+
// queued key goes to the API and the estimator prices it — but the results
|
|
2359
|
+
// ARE stored, so the cache holds what this run paid for (lib/tm.js
|
|
2360
|
+
// setTMReads). It used to be a throwaway empty TM: a fresh re-translation
|
|
2361
|
+
// was never cached, and a later --redo served the older text again.
|
|
460
2362
|
const noTM = cliArgs['no-tm'] || false;
|
|
461
|
-
const tm =
|
|
2363
|
+
const tm = loadTM(cwd);
|
|
2364
|
+
if (noTM) setTMReads(tm, false);
|
|
462
2365
|
const tmInitialSize = tmSize(tm);
|
|
463
2366
|
if (noTM) {
|
|
464
|
-
output.info(
|
|
2367
|
+
output.info(`Translation Memory: not read this run (${cliArgs.fresh ? '--fresh' : '--no-tm'}) — everything queued is translated and billed, and the results are cached`);
|
|
465
2368
|
} else if (tmInitialSize > 0) {
|
|
466
2369
|
output.info(`Translation Memory: ${tmInitialSize} cached entries loaded`);
|
|
467
2370
|
}
|
|
2371
|
+
// Model carry-over (lib/tm.js) is on unless --fresh-on-model-change.
|
|
2372
|
+
const freshOnModelChange = !!cliArgs['fresh-on-model-change'];
|
|
2373
|
+
if (freshOnModelChange) setModelCarryover(tm, false);
|
|
2374
|
+
// Entries a pair (or its fallback) made before its coaching was part of
|
|
2375
|
+
// the cache key, with the coaching it has NOW: recorded so they are still
|
|
2376
|
+
// served, and the lock's writers read as today's key — an upgrade neither
|
|
2377
|
+
// re-translates nor reports "another coaching" (lib/tm.js
|
|
2378
|
+
// adoptLegacyCoachingKeys). A coaching the old version never sent (a
|
|
2379
|
+
// fallback's own coachingFile) gets no record: that is a change.
|
|
2380
|
+
adoptLegacyCoachingKeys(tm, resolvedPairs.values());
|
|
2381
|
+
for (const pc of resolvedPairs.values()) {
|
|
2382
|
+
const by = lockState.peek(pc.target).by || {};
|
|
2383
|
+
for (const [lk, mk] of Object.entries(by)) by[lk] = canonicalWriterKey(tm, pc.target, mk);
|
|
2384
|
+
}
|
|
2385
|
+
// A cache written before borrowed i18next plural forms had their own entry:
|
|
2386
|
+
// an entry that holds a borrowed form's text moves to that form's entry,
|
|
2387
|
+
// before the estimate prices anything (lib/tm-evict.js). A dry run does
|
|
2388
|
+
// it in memory only (the cache is not saved).
|
|
2389
|
+
for (const unit of units) {
|
|
2390
|
+
if (!unit.pluralGroups || unit.pluralGroups.size === 0) continue;
|
|
2391
|
+
for (const [, pc] of pairEntries) {
|
|
2392
|
+
const { expansion } = expectedForTarget(unit, inputLocale, pc.target);
|
|
2393
|
+
if (!expansion?.borrowed || Object.keys(expansion.borrowed).length === 0) continue;
|
|
2394
|
+
let file;
|
|
2395
|
+
try { file = layout.fileFor(pc.target, unit.ns); } catch { continue; }
|
|
2396
|
+
if (!file || !fs.existsSync(file.path)) continue;
|
|
2397
|
+
let targetFlat;
|
|
2398
|
+
try { targetFlat = readLocaleFlat(file); } catch { continue; }
|
|
2399
|
+
const moved = splitSharedPluralEntries(tm, { expansion, targetFlat, locale: pc.target });
|
|
2400
|
+
if (moved.length > 0) {
|
|
2401
|
+
output.info(`${file.rel}: ${moved.length} cached translation(s) of a plural form (${moved.slice(0, 3).join(', ')}${moved.length > 3 ? ', …' : ''}) `
|
|
2402
|
+
+ 'were stored under the form they are translated from — until this release the two shared one cache entry. '
|
|
2403
|
+
+ `Moved to their own entry${dryRun ? ' (in this dry run only)' : ''}; the form they borrow from (_other) is translated again the next time it is queued, `
|
|
2404
|
+
+ 'instead of being served the wrong form.');
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
// Keys NAMED for a redo (--redo keys: / --force-keys), apart from a bulk
|
|
2409
|
+
// --redo all: a named key replaces even a person's edit (lib/locale-state.js).
|
|
2410
|
+
const namedKeys = cliArgs.force ? splitKeyList(cliArgs['force-keys'] || '') : (config.forceKeys || []);
|
|
2411
|
+
// The texts the current source strings are cached under: the notice
|
|
2412
|
+
// counts only translations the run could actually reuse. (A project with
|
|
2413
|
+
// a contentDir also caches Markdown blocks in the same TM; those texts are
|
|
2414
|
+
// not listed here, so its notice keeps counting every entry.)
|
|
2415
|
+
const currentSourceTexts = config.contentDir ? null : [];
|
|
2416
|
+
for (const unit of currentSourceTexts ? units : []) {
|
|
2417
|
+
for (const [key, value] of Object.entries(unit.flat)) {
|
|
2418
|
+
if (typeof value === 'string') currentSourceTexts.push(tmSourceText(key, value));
|
|
2419
|
+
}
|
|
2420
|
+
// Borrowed plural forms are cached under their own text (tmTextFor).
|
|
2421
|
+
if (unit.pluralGroups && unit.pluralGroups.size > 0) {
|
|
2422
|
+
for (const [, pc] of pairEntries) {
|
|
2423
|
+
const { flat, expansion } = expectedForTarget(unit, inputLocale, pc.target);
|
|
2424
|
+
for (const k of Object.keys(expansion?.borrowed || {})) currentSourceTexts.push(tmTextFor(k, flat[k], expansion));
|
|
2425
|
+
}
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
// --fresh-on-model-change: said before the estimate (it changes what is
|
|
2429
|
+
// billed). The carry-over notice waits for the estimate: it is said only
|
|
2430
|
+
// when this run actually serves keys from the previous model's cache.
|
|
2431
|
+
let tmModelSwitch = noTM || !freshOnModelChange ? [] : warnModelSwitchStrandedTM(tm, resolvedPairs.values(), {
|
|
2432
|
+
fresh: true, redoAll: !!cliArgs.force, sourceTexts: currentSourceTexts,
|
|
2433
|
+
});
|
|
468
2434
|
|
|
469
2435
|
// --- Pre-sync cost estimation ---
|
|
470
2436
|
// Runs for EVERY engine (each method implements estimateCost, or honestly
|
|
471
2437
|
// reports "unknown"). Without --max-cost this stays non-blocking: failures
|
|
472
2438
|
// log a warning and the sync continues. With --max-cost the estimate is a
|
|
473
2439
|
// GATE: over-cap or unknowable estimates abort before any API call.
|
|
2440
|
+
// --files / --retranslate scope the CONTENT files (lib/file-scope.js).
|
|
2441
|
+
// Resolved before the estimate so a pattern that matches nothing fails
|
|
2442
|
+
// here, before anything is spent, and so the estimate prices exactly the
|
|
2443
|
+
// scoped work.
|
|
2444
|
+
// A pattern matches the path sync prints (relative to the contentDir) and
|
|
2445
|
+
// the path from the project root ("newsletter/2026-10.md").
|
|
2446
|
+
const fileScope = compileFileScope(cliArgs, {
|
|
2447
|
+
rootPrefix: config.contentDir ? path.relative(cwd, config.contentDir).split(path.sep).join('/') : '',
|
|
2448
|
+
});
|
|
2449
|
+
if (fileScope) {
|
|
2450
|
+
if (!config.contentDir) {
|
|
2451
|
+
throw new Error('--files / --retranslate select content files, but this project has no contentDir.');
|
|
2452
|
+
}
|
|
2453
|
+
for (const p of discoverContentFiles(config.contentDir, inputLocale)) {
|
|
2454
|
+
fileScope.includes(path.relative(config.contentDir, p));
|
|
2455
|
+
}
|
|
2456
|
+
fileScope.assertAllMatched();
|
|
2457
|
+
}
|
|
2458
|
+
const forceContent = !!cliArgs['force-content'];
|
|
2459
|
+
|
|
2460
|
+
// Said once per locale: why what goes to the model did not come from the
|
|
2461
|
+
// cache after a method change (the run as it sends; a dry run up front).
|
|
2462
|
+
const cacheScopeNoted = new Set();
|
|
2463
|
+
const noteCacheScope = (pairKey, pairConfig, code, sentTexts) => {
|
|
2464
|
+
if (noTM || cacheScopeNoted.has(code)) return;
|
|
2465
|
+
// Each text once (primary and fallback sends overlap), and never the
|
|
2466
|
+
// pair's own fallback: its entries are this run's, not "another method's"
|
|
2467
|
+
// (a pair with a fallback reported 6 of 7 keys as cached elsewhere).
|
|
2468
|
+
const texts = [...new Set(sentTexts)];
|
|
2469
|
+
const fallbackKey = pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null;
|
|
2470
|
+
// A key from before coaching was keyed is named as the setup it was
|
|
2471
|
+
// (lib/tm.js canonicalWriterKey) — its coaching included.
|
|
2472
|
+
const others = findOtherMethodEntries(tm, pairConfig, texts)
|
|
2473
|
+
.map(o => ({ ...o, methodKey: canonicalWriterKey(tm, code, o.methodKey) }))
|
|
2474
|
+
.filter(o => o.methodKey !== fallbackKey && o.methodKey !== tmMethodKey(pairConfig));
|
|
2475
|
+
if (others.length === 0) return;
|
|
2476
|
+
cacheScopeNoted.add(code);
|
|
2477
|
+
const held = others.reduce((n, o) => Math.max(n, o.count), 0);
|
|
2478
|
+
output.info(`${pairKey}: ${held} of the ${texts.length} key(s) ${dryRun ? 'a real run would send' : 'sent'} to the model have translations in the cache from `
|
|
2479
|
+
+ `${others.slice(0, 2).map(o => describeMethodKey(o.methodKey)).join('; ')} — not reused: the cache is kept per method, `
|
|
2480
|
+
+ 'register and coaching, so a change of method (or register, or coaching file) translates again. '
|
|
2481
|
+
+ '(A change of model alone reuses earlier translations — model carry-over — unless --fresh-on-model-change.)');
|
|
2482
|
+
};
|
|
2483
|
+
|
|
2484
|
+
// Per pair: the texts the run would send, and the keys it would serve
|
|
2485
|
+
// from an earlier model's translations (cost-report.js options.collect).
|
|
2486
|
+
const estimateDetail = {};
|
|
2487
|
+
// Per target: the Markdown the run would send (cost-report.js collectContent).
|
|
2488
|
+
const contentByTarget = {};
|
|
2489
|
+
// Model carry-over: said BEFORE the estimate's figure, only when this run
|
|
2490
|
+
// serves keys from the previous model's cache. A project with Markdown
|
|
2491
|
+
// pages also caches their blocks, which the key estimate does not
|
|
2492
|
+
// partition: while pages are pending, the notice counts what the cache
|
|
2493
|
+
// holds, as before.
|
|
2494
|
+
let carryoverSaid = false;
|
|
2495
|
+
const sayCarryover = (content) => {
|
|
2496
|
+
carryoverSaid = true;
|
|
2497
|
+
if (noTM || freshOnModelChange) return;
|
|
2498
|
+
const contentPending = (content?.pendingTranslations || 0) > 0;
|
|
2499
|
+
const served = {};
|
|
2500
|
+
for (const d of Object.values(estimateDetail)) served[d.target] = d.carriedFrom;
|
|
2501
|
+
tmModelSwitch = warnModelSwitchStrandedTM(tm, resolvedPairs.values(), {
|
|
2502
|
+
sourceTexts: currentSourceTexts, served: contentPending || content === undefined ? null : served,
|
|
2503
|
+
});
|
|
2504
|
+
};
|
|
474
2505
|
const costEstimate = await printCostEstimate(
|
|
475
|
-
pairEntries,
|
|
2506
|
+
pairEntries, units[0].flat, config, format, units[0].file.ext, changedKeys,
|
|
2507
|
+
{
|
|
2508
|
+
cwd, tm, noTranslate, fileScope, forceContent, layout, units,
|
|
2509
|
+
lockState, redo: { named: namedKeys, bulk: !!cliArgs.force, fresh: noTM, gaps: redoGaps }, collect: estimateDetail,
|
|
2510
|
+
collectContent: contentByTarget,
|
|
2511
|
+
beforeTable: ({ content }) => sayCarryover(content),
|
|
2512
|
+
}
|
|
476
2513
|
);
|
|
2514
|
+
// Machine-readable estimate BEFORE the gate (the summary comes too late
|
|
2515
|
+
// for an agent deciding whether to let the run proceed).
|
|
2516
|
+
if (costEstimate) output.event('cost', costEstimate);
|
|
2517
|
+
// The estimate failed before its table: the notice counts what the cache holds.
|
|
2518
|
+
if (!carryoverSaid) sayCarryover(undefined);
|
|
2519
|
+
// A dry run says why what it would send is not served from the cache after
|
|
2520
|
+
// a method change — the real run says it as it sends (Round 5, Next.js persona).
|
|
2521
|
+
if (dryRun) {
|
|
2522
|
+
for (const [pairKey, d] of Object.entries(estimateDetail)) {
|
|
2523
|
+
if (d.sendTexts.length > 0) noteCacheScope(pairKey, d.pairConfig, d.target, d.sendTexts);
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
|
|
2527
|
+
// A model server that did not answer at startup: now that the plan is
|
|
2528
|
+
// known, a pair that sends it nothing goes on (with a warning); one that
|
|
2529
|
+
// sends something stops here, before anything is sent — a dry run says the
|
|
2530
|
+
// real run would (Round 11, Django persona: a redo served entirely from
|
|
2531
|
+
// the cache failed because Ollama was not running).
|
|
2532
|
+
preflightFailures = [...preflightFailures, ...resolveDeferredProbes(deferredProbeFailures, { costEstimate, contentByTarget, cliArgs: { ...cliArgs, dryRun } })];
|
|
2533
|
+
|
|
2534
|
+
// Fallbacks are not in the estimate: they only translate what the primary
|
|
2535
|
+
// fails, which nobody knows before the run. Say so, and how the cap
|
|
2536
|
+
// treats them (lib/fallback.js createFallbackBudget).
|
|
2537
|
+
if (pairEntries.some(([, p]) => p.fallback)) {
|
|
2538
|
+
output.info(
|
|
2539
|
+
'Fallback methods are not in this estimate — they only translate what the primary method fails, '
|
|
2540
|
+
+ 'which is not known in advance.'
|
|
2541
|
+
+ (maxCost !== null ? ' Under --max-cost each fallback batch is priced before it runs and skipped if it would pass the cap.' : '')
|
|
2542
|
+
);
|
|
2543
|
+
}
|
|
477
2544
|
|
|
478
2545
|
// Enforce the cap only for real runs: a dry-run makes zero API calls, and
|
|
479
2546
|
// aborting it would block the exact preview a capped user needs to see.
|
|
2547
|
+
// A dry run says instead what the real run would do at the cap.
|
|
2548
|
+
let dryMaxCost = null;
|
|
480
2549
|
if (maxCost !== null && !dryRun) {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
);
|
|
486
|
-
}
|
|
487
|
-
if (costEstimate.hasUnknownCosts) {
|
|
488
|
-
return abortForMaxCost(
|
|
489
|
-
maxCost, null,
|
|
490
|
-
'Some pairs have unknown pricing, so the total cost cannot be bounded (unknown is not free).'
|
|
491
|
-
);
|
|
492
|
-
}
|
|
493
|
-
if (costEstimate.totalEstimatedCost > maxCost) {
|
|
494
|
-
return abortForMaxCost(
|
|
495
|
-
maxCost, costEstimate.totalEstimatedCost,
|
|
496
|
-
'Estimated translation cost exceeds the --max-cost cap.'
|
|
497
|
-
);
|
|
498
|
-
}
|
|
2550
|
+
const verdict = maxCostVerdict(maxCost, costEstimate);
|
|
2551
|
+
if (verdict.wouldStop) return abortForMaxCost(maxCost, verdict.estimatedCost, verdict.reason);
|
|
2552
|
+
} else if (maxCost !== null) {
|
|
2553
|
+
dryMaxCost = reportDryRunMaxCost(maxCost, costEstimate, { stopsEarlier: preflightStopReason(preflightFailures) });
|
|
499
2554
|
}
|
|
500
2555
|
|
|
501
2556
|
output.raw('');
|
|
@@ -513,311 +2568,454 @@ async function runSync(options = {}) {
|
|
|
513
2568
|
// Node.js is single-threaded so storeTM() property assignments can't
|
|
514
2569
|
// interleave between await points — fully safe under pMap concurrency.
|
|
515
2570
|
const jsonConcurrency = config.jsonConcurrency ?? DEFAULT_JSON_CONCURRENCY;
|
|
516
|
-
output.info(
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
// A refusal means NO file was written — count it as a failure so the run
|
|
526
|
-
// reports it and exits non-zero, instead of printing [OK] and exiting 0.
|
|
527
|
-
if (!isPathContained(filePath, config.localesDir)) {
|
|
528
|
-
output.error(`${filename} — refusing to write outside locales directory`);
|
|
529
|
-
// Nothing ran for this locale, so every changed key is unresolved here.
|
|
530
|
-
// Only changed keys matter for manifest retry-safety: missing keys
|
|
531
|
-
// re-fire via missing-key detection regardless of the manifest.
|
|
532
|
-
return { processed: 0, tmHits: 0, failed: 1, failedKeys: changedKeys, pairKey };
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
// If locale file doesn't exist yet, create it as empty
|
|
536
|
-
let data = {};
|
|
537
|
-
if (fs.existsSync(filePath)) {
|
|
538
|
-
data = readLocaleFile(filePath, format);
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
// For JSON, flatten the nested structure. TOML/YAML is already flat.
|
|
542
|
-
const targetFlat = format === 'json' ? flattenKeys(data) : { ...data };
|
|
543
|
-
// Source-echo requeue suppression: a target value equal to its source is
|
|
544
|
-
// only requeued when the TM does NOT confirm the echo came from the
|
|
545
|
-
// pipeline. lookupTM === sourceValue means a previous run translated this
|
|
546
|
-
// exact text to itself and the gate approved it — skip, don't re-bill.
|
|
547
|
-
// With --no-tm the TM is empty, so nothing is suppressed.
|
|
548
|
-
const tmKey = tmMethodKey(pairConfig);
|
|
549
|
-
const diff = diffLocale(
|
|
550
|
-
sourceFlat, targetFlat, config.fallbackPrefix, config.forceKeys, changedKeys,
|
|
551
|
-
(key, sourceValue) => lookupTM(tm, sourceValue, code, tmKey) === sourceValue,
|
|
552
|
-
noTranslate.active ? noTranslate.matches : null
|
|
553
|
-
);
|
|
554
|
-
|
|
555
|
-
if (diff.toProcess.length === 0 && diff.noTranslate.length === 0 && diff.extra.length === 0) {
|
|
556
|
-
output.ok(`${filename} — fully synced`);
|
|
557
|
-
return { processed: 0, tmHits: 0 };
|
|
558
|
-
}
|
|
2571
|
+
output.info(describeLocaleWork({ dryRun, pairEntries, costEstimate, contentByTarget, concurrency: jsonConcurrency }));
|
|
2572
|
+
// Flutter ARB targets this run is about to create: once written, each is
|
|
2573
|
+
// checked against the languages Flutter's own widget text covers (lib/flutter-locales.js).
|
|
2574
|
+
const newArbTargets = dryRun ? [] : pairEntries.map(([, pc]) => pc.target).filter((code) => {
|
|
2575
|
+
try {
|
|
2576
|
+
const f = layout.fileFor(code, units[0].ns);
|
|
2577
|
+
return f && f.format === 'arb' && !fs.existsSync(f.path);
|
|
2578
|
+
} catch { return false; }
|
|
2579
|
+
});
|
|
559
2580
|
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
2581
|
+
// Shared, read-only context for the per-file worker (syncLocaleFile).
|
|
2582
|
+
// --max-cost for fallback batches: the cap admitted the pre-run estimate;
|
|
2583
|
+
// each fallback batch must fit in what is left (lib/fallback.js).
|
|
2584
|
+
const fallbackBudget = createFallbackBudget({
|
|
2585
|
+
maxCost: dryRun ? null : maxCost,
|
|
2586
|
+
committed: costEstimate?.knownEstimatedCost ?? 0,
|
|
2587
|
+
cwd,
|
|
2588
|
+
});
|
|
2589
|
+
// How a pending key is labelled in the lock and in status: the redo that left it.
|
|
2590
|
+
const redoLabel = [
|
|
2591
|
+
cliArgs.force ? '--redo all' : (namedKeys.length > 0 ? '--redo keys:' : null),
|
|
2592
|
+
redoGaps ? '--redo gaps' : null,
|
|
2593
|
+
noTM ? '--fresh' : null,
|
|
2594
|
+
freshOnModelChange ? '--fresh-on-model-change' : null,
|
|
2595
|
+
].filter(Boolean).join(' ') || '--redo';
|
|
2596
|
+
// Per locale: what was pending before the run (a model switch completes
|
|
2597
|
+
// when its last pending key is translated).
|
|
2598
|
+
const pendingBefore = new Map(pairEntries.map(([, pc]) => [pc.target, Object.values(lockState.peek(pc.target).pending)]));
|
|
2599
|
+
// One different-inputs-same-output index per locale, shared by every file
|
|
2600
|
+
// of it — and by its content files after (lib/validate.js SharedOutputIndex).
|
|
2601
|
+
const sharedOutputIndexes = new Map();
|
|
2602
|
+
const ctx = {
|
|
2603
|
+
config, layout, inputLocale, dryRun, cliArgs, apiKey, tm, noTranslate, cwd,
|
|
2604
|
+
pluralFallbackReported: new Set(),
|
|
2605
|
+
fallbackBudget,
|
|
2606
|
+
lockState,
|
|
2607
|
+
namedKeys,
|
|
2608
|
+
redoLabel,
|
|
2609
|
+
replacedEdits: [],
|
|
2610
|
+
// --prune plural-extras: what was (or would be) removed, per file.
|
|
2611
|
+
prune,
|
|
2612
|
+
pruned: [],
|
|
2613
|
+
// --redo gaps, and every plural message this run asks again (pair → lock keys).
|
|
2614
|
+
redoGaps,
|
|
2615
|
+
gapsAsked: new Map(),
|
|
2616
|
+
gapsSeen: 0,
|
|
2617
|
+
sharedOutputsFor(code) {
|
|
2618
|
+
if (!sharedOutputIndexes.has(code)) {
|
|
2619
|
+
// Sentences an earlier sync caught a model repeating for different
|
|
2620
|
+
// source strings (refused from their first source on), and what the
|
|
2621
|
+
// locale already holds that this run leaves alone — every key-value
|
|
2622
|
+
// file and every Markdown page, as `verify` reads them — before
|
|
2623
|
+
// anything new is checked (lib/shared-output-seed.js).
|
|
2624
|
+
sharedOutputIndexes.set(code, projectSharedOutputIndex({
|
|
2625
|
+
config, cwd, code, tm, layout, units, localeState: lockState.peek(code), namedKeys, bulk: !!cliArgs.force,
|
|
2626
|
+
fileScope, forceContent,
|
|
2627
|
+
}));
|
|
605
2628
|
}
|
|
606
|
-
|
|
2629
|
+
return sharedOutputIndexes.get(code);
|
|
2630
|
+
},
|
|
2631
|
+
noteCacheScope,
|
|
2632
|
+
previewsShown: 0,
|
|
2633
|
+
notePreviewShown() { ctx.previewsShown++; },
|
|
2634
|
+
};
|
|
607
2635
|
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
2636
|
+
const localeResults = await pMap(pairEntries, async ([pairKey, pairConfig]) => {
|
|
2637
|
+
// A locale is one file (flat) or several (one per namespace). The files
|
|
2638
|
+
// of ONE locale run in sequence, never in parallel: the Translation
|
|
2639
|
+
// Memory is keyed by source TEXT, so a string the first file paid for
|
|
2640
|
+
// is a free cache hit in the next — parallel files would each pay.
|
|
2641
|
+
const agg = {
|
|
2642
|
+
processed: 0, tmHits: 0, sent: 0, retried: 0, pluralGaps: {}, copied: 0, keptWorkingScript: 0, failed: 0, failedKeys: [], pairKey,
|
|
2643
|
+
held: 0, heldKeys: [], kept: 0, keptKeys: [], replaced: 0, fates: { retry: [], 'pending-retry': [], held: [] },
|
|
2644
|
+
};
|
|
2645
|
+
let queuedKeys = null;
|
|
2646
|
+
let backendDown = false;
|
|
2647
|
+
let fallbackDown = false;
|
|
2648
|
+
// One fallback tally per pair, across its files → one [FALLBACK] line.
|
|
2649
|
+
const fallbackTally = pairConfig.fallback && !dryRun ? newFallbackReport(pairConfig.fallback) : null;
|
|
2650
|
+
for (const unit of units) {
|
|
2651
|
+
const r = await syncLocaleFile(ctx, unit, pairKey, pairConfig, { backendDown, fallbackDown });
|
|
2652
|
+
if (fallbackTally) addToTally(fallbackTally, r.fallback);
|
|
2653
|
+
agg.processed += r.processed || 0;
|
|
2654
|
+
agg.tmHits += r.tmHits || 0;
|
|
2655
|
+
agg.sent += r.sent || 0;
|
|
2656
|
+
agg.retried += r.retried || 0;
|
|
2657
|
+
Object.assign(agg.pluralGaps, r.pluralGaps || {});
|
|
2658
|
+
agg.copied += r.copied || 0;
|
|
2659
|
+
agg.keptWorkingScript += r.keptWorkingScript || 0;
|
|
2660
|
+
agg.failed += r.failed || 0;
|
|
2661
|
+
agg.failedKeys.push(...(r.failedKeys || []));
|
|
2662
|
+
agg.held += r.held || 0;
|
|
2663
|
+
agg.heldKeys.push(...(r.heldKeys || []));
|
|
2664
|
+
agg.kept += r.kept || 0;
|
|
2665
|
+
agg.keptKeys.push(...(r.keptKeys || []));
|
|
2666
|
+
agg.replaced += r.replaced || 0;
|
|
2667
|
+
for (const [mk, n] of Object.entries(r.otherMethods || {})) {
|
|
2668
|
+
agg.otherMethods = agg.otherMethods || {};
|
|
2669
|
+
agg.otherMethods[mk] = (agg.otherMethods[mk] || 0) + n;
|
|
635
2670
|
}
|
|
2671
|
+
for (const [mk, n] of Object.entries(r.otherMethodsForced || {})) {
|
|
2672
|
+
agg.otherMethodsForced = agg.otherMethodsForced || {};
|
|
2673
|
+
agg.otherMethodsForced[mk] = (agg.otherMethodsForced[mk] || 0) + n;
|
|
2674
|
+
}
|
|
2675
|
+
for (const [model, n] of Object.entries(r.otherModels || {})) {
|
|
2676
|
+
agg.otherModels = agg.otherModels || {};
|
|
2677
|
+
agg.otherModels[model] = (agg.otherModels[model] || 0) + n;
|
|
2678
|
+
}
|
|
2679
|
+
for (const [fate, keys] of Object.entries(r.fates || {})) agg.fates[fate].push(...keys);
|
|
2680
|
+
if (r.queuedKeys) {
|
|
2681
|
+
queuedKeys = queuedKeys || {
|
|
2682
|
+
missing: [], fallback: [], untranslated: [], changed: [], forced: [], gaps: [], noTranslate: [], pending: [], held: [], kept: [],
|
|
2683
|
+
};
|
|
2684
|
+
for (const [reason, keys] of Object.entries(r.queuedKeys)) queuedKeys[reason].push(...keys);
|
|
2685
|
+
}
|
|
2686
|
+
if (r.backendDown) backendDown = true;
|
|
2687
|
+
if (r.fallbackDown) fallbackDown = true;
|
|
2688
|
+
}
|
|
2689
|
+
if (queuedKeys) agg.queuedKeys = queuedKeys;
|
|
2690
|
+
if (fallbackTally) {
|
|
2691
|
+
printFallbackReport(pairKey, pairConfig.method, fallbackTally);
|
|
2692
|
+
warnFallbackMajority(pairKey, pairConfig.method, fallbackTally);
|
|
2693
|
+
agg.fallback = fallbackTally;
|
|
2694
|
+
}
|
|
2695
|
+
return agg;
|
|
2696
|
+
}, { concurrency: jsonConcurrency });
|
|
636
2697
|
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
onProgress: (completed, total) => {
|
|
646
|
-
output.progressBar(completed, total);
|
|
647
|
-
},
|
|
648
|
-
});
|
|
649
|
-
translated = result.translated;
|
|
650
|
-
localeTMHits += result.tmHitCount;
|
|
651
|
-
|
|
652
|
-
// Terminology enforcement: check if dictionary terms were applied.
|
|
653
|
-
// This only runs when the pair has coaching data with a dictionary.
|
|
654
|
-
if (translated && pairConfig.coachingData?.dictionary) {
|
|
655
|
-
const { violations } = verifyTerminology(translated, sourceFlat, pairConfig.coachingData.dictionary);
|
|
656
|
-
if (violations.length > 0) {
|
|
657
|
-
logTermViolations(violations, pairKey);
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
if (translated) {
|
|
662
|
-
localeFailed += result.failures.length;
|
|
663
|
-
output.progress(result.failures.length > 0
|
|
664
|
-
? ` [OK] (${result.failures.length} key(s) failed quality gate)`
|
|
665
|
-
: ' [OK]\n');
|
|
666
|
-
} else if (result.apiReturnedNull) {
|
|
667
|
-
// Method returned null — fail loud with actionable guidance.
|
|
668
|
-
output.progress(' [ERR]\n');
|
|
669
|
-
output.error(`${pairKey}: Translation method "${pairConfig.method}" returned no results.`);
|
|
670
|
-
const methodInstance = getMethod(pairConfig.method, pairConfig);
|
|
671
|
-
const helpLines = methodInstance.getSetupHelp();
|
|
672
|
-
for (const line of helpLines) {
|
|
673
|
-
output.error(line);
|
|
674
|
-
}
|
|
675
|
-
// Whole locale failed: every pending key must re-fire next sync.
|
|
676
|
-
return { processed: 0, tmHits: localeTMHits, copied: flushNoTranslateOnBail(), failed: diff.toProcess.length, failedKeys: diff.toProcess, pairKey };
|
|
677
|
-
} else if (result.failures.length > 0 && !translated) {
|
|
678
|
-
// All translations failed quality gate — fail loud
|
|
679
|
-
output.progress(' [ERR] all translations failed quality gate\n');
|
|
680
|
-
output.error(`${pairKey}: All translations were rejected by the quality gate.`);
|
|
681
|
-
output.error('Check your method configuration or review the gate failures above.');
|
|
682
|
-
return { processed: 0, tmHits: localeTMHits, copied: flushNoTranslateOnBail(), failed: diff.toProcess.length, failedKeys: diff.toProcess, pairKey };
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
// Post-translation script conversion — ONLY when this pair's script
|
|
687
|
-
// resolution asked for it (config `script:`). The old gate was a bare
|
|
688
|
-
// registry lookup, which converted every tlh/crk/… project into
|
|
689
|
-
// display scripts (PUA for the conlangs) whether or not their fonts
|
|
690
|
-
// could render them. See lib/scripts.js resolveTargetScript.
|
|
691
|
-
const scriptConverterKey = pairConfig.scriptResolution?.converterKey || null;
|
|
692
|
-
if (scriptConverterKey && translated && Object.keys(translated).length > 0) {
|
|
693
|
-
const info = getConverterInfo(scriptConverterKey);
|
|
694
|
-
output.info(`[SCRIPT] Converting ${info.from} → ${info.to} (${Object.keys(translated).length} keys)`);
|
|
695
|
-
}
|
|
2698
|
+
// --redo gaps with nothing to ask for: said, never a silent no-op.
|
|
2699
|
+
if (redoGaps && ctx.gapsSeen === 0) {
|
|
2700
|
+
output.info(`--redo gaps: no plural message in the ${cliArgs.pair ? 'selected pairs\'' : 'project\'s'} files lacks a form its language uses for ordinary counts — nothing to ask again.`);
|
|
2701
|
+
}
|
|
2702
|
+
// --prune plural-extras with nothing to remove: said too.
|
|
2703
|
+
if (prune.has('plural-extras') && ctx.pruned.length === 0) {
|
|
2704
|
+
output.info('--prune plural-extras: no plural key for a form its language does not have — nothing removed.');
|
|
2705
|
+
}
|
|
696
2706
|
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
// User-declared transliteration fallbacks first (validated at
|
|
706
|
-
// pair build), then the converter. If letters remain that the
|
|
707
|
-
// converter cannot map, the output would be an unreadable mix
|
|
708
|
-
// of both scripts — keep the WHOLE value in the working script
|
|
709
|
-
// instead, and say which letters and how to map them. Not a
|
|
710
|
-
// failure: unmappable proper nouns would fail identically on
|
|
711
|
-
// every retry, and a permanently red sync is the trap this
|
|
712
|
-
// release exists to close.
|
|
713
|
-
const prepared = applyScriptFallback(value, pairConfig.scriptFallback);
|
|
714
|
-
const { converted, unmapped } = convertScript(prepared, scriptConverterKey);
|
|
715
|
-
if (unmapped.length === 0) {
|
|
716
|
-
value = converted;
|
|
717
|
-
} else {
|
|
718
|
-
const hint = unmapped.map(l => `"${l}": "?"`).join(', ');
|
|
719
|
-
output.warn(
|
|
720
|
-
`${pairKey}: key "${key}" kept in ${getConverterInfo(scriptConverterKey).from} — `
|
|
721
|
-
+ `letter(s) the converter cannot map: ${unmapped.join(', ')}. `
|
|
722
|
-
+ `To transliterate them, add "scriptFallback": { ${hint} } for ${pairConfig.target}.`
|
|
723
|
-
);
|
|
724
|
-
localeKeptWorkingScript++;
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
} else if (typeof sourceValue === 'string') {
|
|
728
|
-
// Key not in translated result (gate rejection or partial API
|
|
729
|
-
// response) — skip it, don't write garbage. Record the key so its
|
|
730
|
-
// old manifest hash is restored and the retry actually happens.
|
|
731
|
-
output.warn(`${pairKey}: key "${key}" not translated — skipping (will retry next sync)`);
|
|
732
|
-
localeFailedKeys.push(key);
|
|
733
|
-
continue;
|
|
734
|
-
} else {
|
|
735
|
-
value = sourceValue;
|
|
736
|
-
}
|
|
2707
|
+
// A new Flutter locale Flutter's own Material/Cupertino text does not
|
|
2708
|
+
// cover needs a fallback delegate in the app (Round 11, hospital persona).
|
|
2709
|
+
const arbWritten = newArbTargets.filter((code) => {
|
|
2710
|
+
try { return fs.existsSync(layout.fileFor(code, units[0].ns).path); } catch { return false; }
|
|
2711
|
+
});
|
|
2712
|
+
if (arbWritten.length > 0) {
|
|
2713
|
+
for (const { level, text } of flutterLocaleLines(arbWritten)) output[level](text);
|
|
2714
|
+
}
|
|
737
2715
|
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
2716
|
+
// What the CONFIGURED setup wrote, when only this run's --method/--model
|
|
2717
|
+
// differ from it: kept by design, said once below (configuredPairKeys).
|
|
2718
|
+
// A DRY run keeps the full note with the redo and its price: that is where
|
|
2719
|
+
// a switch is tried and priced before it is committed (Rounds 7-9, Next.js
|
|
2720
|
+
// persona: `sync --dry --model m2`). A real run with the flags is a
|
|
2721
|
+
// deliberate per-run choice — the CI guide's one-run hosted model — and
|
|
2722
|
+
// repeating a paid redo on every such run is noise (Round 12, Django).
|
|
2723
|
+
const configuredKeys = dryRun ? null : configuredPairKeys(config, cwd, cliArgs);
|
|
2724
|
+
const keptFromConfig = [];
|
|
2725
|
+
|
|
2726
|
+
// Files that still hold an earlier model's text: one line per pair, unless
|
|
2727
|
+
// the model-change notice above already said it for that locale (it names
|
|
2728
|
+
// the keys this run served from the previous model's cache).
|
|
2729
|
+
if (!(cliArgs.force && freshOnModelChange)) {
|
|
2730
|
+
const affected = [];
|
|
2731
|
+
pairEntries.forEach(([pairKey, pc], i) => {
|
|
2732
|
+
let counts = localeResults[i]?.otherModels;
|
|
2733
|
+
if (!counts || Object.keys(counts).length === 0) return;
|
|
2734
|
+
const own = configuredKeys?.get(pairKey);
|
|
2735
|
+
if (own) {
|
|
2736
|
+
// The same method, register and coaching as this run: only --model
|
|
2737
|
+
// differs, and the configured model's text is the project's own.
|
|
2738
|
+
const [om, omodel, or, oc] = own.split('|');
|
|
2739
|
+
const [cm, , cr, cc] = tmMethodKey(pc).split('|');
|
|
2740
|
+
const label = omodel || '(none)';
|
|
2741
|
+
if (om === cm && or === cr && oc === cc && counts[label]) {
|
|
2742
|
+
keptFromConfig.push({ pairKey, target: pc.target, n: counts[label], setup: flagSetupOf(own) });
|
|
2743
|
+
counts = { ...counts };
|
|
2744
|
+
delete counts[label];
|
|
2745
|
+
if (Object.keys(counts).length === 0) return;
|
|
743
2746
|
}
|
|
2747
|
+
}
|
|
2748
|
+
const said = (tmModelSwitch || []).some(row => row.target === pc.target && (freshOnModelChange || (row.servedThisRun || 0) > 0));
|
|
2749
|
+
if (said) return;
|
|
2750
|
+
const total = Object.values(counts).reduce((a, b) => a + b, 0);
|
|
2751
|
+
const from = Object.entries(counts).sort((a, b) => b[1] - a[1]).map(([m, n]) => `${m} (${n})`).join(', ');
|
|
2752
|
+
const model = tmMethodKey(pc).split('|')[1] || pc.method;
|
|
2753
|
+
// `--model X` on this run: X is the model for THIS run, while the
|
|
2754
|
+
// config still names the old one (Round 7, Next.js persona).
|
|
2755
|
+
const which = cliArgs.model ? `the model for this run (--model ${model})` : `the configured model (${model})`;
|
|
2756
|
+
affected.push({ pairKey, pc, total, models: Object.keys(counts), line: `${total} translation(s) in the files were written by ${from}, not ${which}` });
|
|
2757
|
+
});
|
|
2758
|
+
// The redo is the same for every pair (the flags are the run's): ONE
|
|
2759
|
+
// command — project-wide when every pair this run covered is affected
|
|
2760
|
+
// and no --pair narrowed it, else the affected pairs together. One pair:
|
|
2761
|
+
// its own command, as before (Round 8: one command per pair, where the
|
|
2762
|
+
// docs show one).
|
|
2763
|
+
if (affected.length > 0) {
|
|
2764
|
+
const scope = affected.length > 1 && affected.length === pairEntries.length && !cliArgs.pair
|
|
2765
|
+
? ''
|
|
2766
|
+
: ` --pair ${affected.map(a => a.pairKey).join(',')}`;
|
|
2767
|
+
const command = `champollion sync${scope}${cliArgs.model ? ` --model ${shellWord(String(cliArgs.model))}` : ''} --redo all --fresh-on-model-change`;
|
|
2768
|
+
// What that redo costs, so the note is enough to decide on (Round 9,
|
|
2769
|
+
// Next.js persona: it took a second dry run to learn the price).
|
|
2770
|
+
const keys = affected.reduce((n, a) => n + a.total, 0);
|
|
2771
|
+
const price = await priceOfSends(affected.map(a => [a.total, a.pc]), { cwd });
|
|
2772
|
+
// The other direction too: text written by a one-off `--model` run is
|
|
2773
|
+
// kept by making that model the configured one (Round 10, Next.js
|
|
2774
|
+
// persona: the note offered only to replace it).
|
|
2775
|
+
const others = [...new Set(affected.flatMap(a => a.models))];
|
|
2776
|
+
const file = cliArgs.config ? path.basename(String(cliArgs.config)) : 'champollion.config.json';
|
|
2777
|
+
const keep = !cliArgs.model && others.length === 1 && others[0] !== '(none)' && affected.every(a => a.models.length === 1)
|
|
2778
|
+
? `. To keep ${others[0]}'s text instead and use ${others[0]} from now on, set "model": "${others[0]}" in ${file} (nothing is sent).`
|
|
2779
|
+
: '';
|
|
2780
|
+
const tail = ' — a model change alone re-translates nothing. To re-translate them with it: '
|
|
2781
|
+
+ `\`${command}\` (sends up to ${keys} key(s) an earlier model wrote — ${price}; what this model already translated comes from the cache)${keep}`;
|
|
2782
|
+
if (affected.length === 1) {
|
|
2783
|
+
output.info(`${affected[0].pairKey}: ${affected[0].line}${tail}`);
|
|
2784
|
+
} else {
|
|
2785
|
+
for (const a of affected) output.info(`${a.pairKey}: ${a.line}.`);
|
|
2786
|
+
output.info(`${affected.length} pairs keep an earlier model's text${tail}`);
|
|
2787
|
+
}
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
744
2790
|
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
2791
|
+
// Files that hold another METHOD's text (or another register's, or another
|
|
2792
|
+
// coaching file's): one line per pair, with the redo and what it costs.
|
|
2793
|
+
//
|
|
2794
|
+
// What the pair's FALLBACK wrote as it was set up before (another model,
|
|
2795
|
+
// register or coaching of the fallback) is said apart: the redo sends it
|
|
2796
|
+
// to the pair's method first and the fallback gets what that refuses — so
|
|
2797
|
+
// the line names both, and prices both (Round 10, school persona: a coaching
|
|
2798
|
+
// file added to the fallback changed nothing, and nothing said why).
|
|
2799
|
+
const byEarlierFallback = (pc, counts) => {
|
|
2800
|
+
const own = {};
|
|
2801
|
+
const fb = {};
|
|
2802
|
+
for (const [mk, n] of Object.entries(counts || {})) (fallbackWriterOf(pc, mk) === 'earlier' ? fb : own)[mk] = n;
|
|
2803
|
+
return { own, fb };
|
|
2804
|
+
};
|
|
2805
|
+
const listFrom = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1]).map(([mk, n]) => `${describeMethodKey(mk)} (${n})`).join(', ');
|
|
2806
|
+
const priceOf = async (n, cfg) => { try { return costLabel(await estimateCost(n, cfg, { cwd })); } catch { return 'cost unknown'; } };
|
|
2807
|
+
// Pairs whose files hold another method's text: said together below, with
|
|
2808
|
+
// ONE redo for all of them and its total (Round 13, Next.js persona: a
|
|
2809
|
+
// dry run after a method change gave one command per language, each priced
|
|
2810
|
+
// on its own, and no total).
|
|
2811
|
+
const methodAffected = [];
|
|
2812
|
+
for (const [i, [pairKey, pc]] of pairEntries.entries()) {
|
|
2813
|
+
const forcedAll = byEarlierFallback(pc, localeResults[i]?.otherMethodsForced);
|
|
2814
|
+
const countsAll = byEarlierFallback(pc, localeResults[i]?.otherMethods);
|
|
2815
|
+
// The fallback's earlier setup: a dry run of the redo says what it would
|
|
2816
|
+
// send; any other run says what the files hold and the redo that changes it.
|
|
2817
|
+
for (const [counts, forcedNote] of [[forcedAll.fb, true], [countsAll.fb, false]]) {
|
|
2818
|
+
const n = Object.values(counts).reduce((a, b) => a + b, 0);
|
|
2819
|
+
if (n === 0 || (forcedNote && !dryRun)) continue;
|
|
2820
|
+
const nowKey = tmMethodKey(pc.fallback);
|
|
2821
|
+
const nowFb = describeMethodKey(nowKey);
|
|
2822
|
+
// Only another MODEL of the fallback: its translations are reused
|
|
2823
|
+
// (model carry-over, lib/tm.js) unless the redo turns that off.
|
|
2824
|
+
const [fm, , fr, fc] = nowKey.split('|');
|
|
2825
|
+
const modelOnly = Object.keys(counts).some((mk) => { const p = mk.split('|'); return p[0] === fm && p[2] === fr && p[3] === fc; });
|
|
2826
|
+
const fresh = modelOnly && !freshOnModelChange ? ' --fresh-on-model-change' : '';
|
|
2827
|
+
const prices = `${pc.method}: ${await priceOf(n, pc)}; ${pc.fallback.method}: ${await priceOf(n, pc.fallback)}`;
|
|
2828
|
+
output.info(forcedNote
|
|
2829
|
+
? `${pairKey}: this redo would re-translate ${n} translation(s) the fallback wrote as it was set up before — ${listFrom(counts)} — `
|
|
2830
|
+
+ `sending them to ${pc.method} first and what it refuses to the fallback as it is now (${nowFb})${fresh ? ' — those an earlier model of the fallback wrote are served from its cache unless --fresh-on-model-change' : ''} `
|
|
2831
|
+
+ `(${prices}; the estimate above prices the pair's method only).`
|
|
2832
|
+
: `${pairKey}: ${n} translation(s) in the files were written by the fallback as it was set up before — ${listFrom(counts)} — `
|
|
2833
|
+
+ `not as it is now (${nowFb}). ${dryRun ? 'Nothing would change' : 'Nothing changed'}: a change of the fallback's model, register or coaching `
|
|
2834
|
+
+ `re-translates nothing on its own${modelOnly ? ' (a model change alone reuses its earlier translations)' : ''}. `
|
|
2835
|
+
+ `To re-translate them: \`champollion sync --pair ${pairKey} --redo all${fresh}\` — sends up to ${n} key(s) `
|
|
2836
|
+
+ `to ${pc.method} first, and what it refuses to the fallback (${prices}).`);
|
|
2837
|
+
}
|
|
2838
|
+
// A dry run of a redo that forces them: what it WOULD send, and the price
|
|
2839
|
+
// — never "nothing would change" over the command being previewed.
|
|
2840
|
+
const forced = forcedAll.own;
|
|
2841
|
+
if (dryRun && forced && Object.keys(forced).length > 0) {
|
|
2842
|
+
const n = Object.values(forced).reduce((a, b) => a + b, 0);
|
|
2843
|
+
const from = Object.entries(forced).sort((a, b) => b[1] - a[1]).map(([mk, c]) => `${describeMethodKey(mk)} (${c})`).join(', ');
|
|
2844
|
+
let price = 'cost unknown';
|
|
2845
|
+
try { price = costLabel(await estimateCost(n, pc, { cwd })); } catch { /* unknown */ }
|
|
2846
|
+
output.info(`${pairKey}: this redo would re-translate ${n} translation(s) written by ${from} — sends them to `
|
|
2847
|
+
+ `${describeMethodKey(tmMethodKey(pc))} unless the cache already holds its translation (${price}; the estimate above prices the whole run).`);
|
|
2848
|
+
}
|
|
2849
|
+
let counts = countsAll.own;
|
|
2850
|
+
if (!counts || Object.keys(counts).length === 0) continue;
|
|
2851
|
+
const own = configuredKeys?.get(pairKey);
|
|
2852
|
+
if (own && counts[own]) {
|
|
2853
|
+
keptFromConfig.push({ pairKey, target: pc.target, n: counts[own], setup: flagSetupOf(own) });
|
|
2854
|
+
counts = { ...counts };
|
|
2855
|
+
delete counts[own];
|
|
2856
|
+
if (Object.keys(counts).length === 0) continue;
|
|
2857
|
+
}
|
|
2858
|
+
const total = Object.values(counts).reduce((a, b) => a + b, 0);
|
|
2859
|
+
const from = Object.entries(counts).sort((a, b) => b[1] - a[1]).map(([mk, n]) => `${describeMethodKey(mk)} (${n})`).join(', ');
|
|
2860
|
+
methodAffected.push({ pairKey, pc, total, from, now: describeMethodKey(tmMethodKey(pc)) });
|
|
2861
|
+
}
|
|
2862
|
+
if (methodAffected.length > 0) {
|
|
2863
|
+
const byFlag = cliArgs.method || cliArgs.model ? ' (as this run sets it)' : '';
|
|
2864
|
+
const flags = [
|
|
2865
|
+
cliArgs.method ? `--method ${shellWord(String(cliArgs.method))}` : null,
|
|
2866
|
+
cliArgs.model ? `--model ${shellWord(String(cliArgs.model))}` : null,
|
|
2867
|
+
].filter(Boolean).join(' ');
|
|
2868
|
+
const nothing = `${dryRun ? 'Nothing would change' : 'Nothing changed'}: a change of method, register or coaching re-translates nothing on its own — `
|
|
2869
|
+
+ 'the files keep the other text.';
|
|
2870
|
+
if (methodAffected.length === 1) {
|
|
2871
|
+
const [{ pairKey, pc, total, from, now }] = methodAffected;
|
|
2872
|
+
output.info(`${pairKey}: ${total} translation(s) in the files were written by ${from}, not by ${now}${byFlag}. `
|
|
2873
|
+
+ `${nothing} To have ${pc.method} translate them: `
|
|
2874
|
+
+ `\`champollion sync --pair ${pairKey}${flags ? ` ${flags}` : ''} --redo all\` — sends ${total} key(s) to ${pc.method} (${await priceOf(total, pc)}).`);
|
|
2875
|
+
} else {
|
|
2876
|
+
// Each pair with its own count and price, then one command for all of
|
|
2877
|
+
// them: project-wide when every pair this run covered is affected and
|
|
2878
|
+
// no --pair narrowed it, else those pairs together.
|
|
2879
|
+
for (const a of methodAffected) {
|
|
2880
|
+
output.info(`${a.pairKey}: ${a.total} translation(s) in the files were written by ${a.from}, not by ${a.now}${byFlag} `
|
|
2881
|
+
+ `(re-translating them: ${a.total} key(s) to ${a.pc.method}, ${await priceOf(a.total, a.pc)}).`);
|
|
775
2882
|
}
|
|
2883
|
+
const scope = methodAffected.length === pairEntries.length && !cliArgs.pair
|
|
2884
|
+
? ''
|
|
2885
|
+
: ` --pair ${methodAffected.map(a => a.pairKey).join(',')}`;
|
|
2886
|
+
const keys = methodAffected.reduce((n, a) => n + a.total, 0);
|
|
2887
|
+
const methods = [...new Set(methodAffected.map(a => a.pc.method))];
|
|
2888
|
+
const price = await priceOfSends(methodAffected.map(a => [a.total, a.pc]), { cwd });
|
|
2889
|
+
output.info(`${methodAffected.length} pairs keep another method's text. ${nothing} `
|
|
2890
|
+
+ `To have ${methods.length === 1 ? methods[0] : 'each pair\'s method'} translate them, all at once: `
|
|
2891
|
+
+ `\`champollion sync${scope}${flags ? ` ${flags}` : ''} --redo all\` — sends ${keys} key(s) `
|
|
2892
|
+
+ `(${methodAffected.map(a => `${a.total} for ${a.pairKey}`).join(', ')}) — total: ${price}.`);
|
|
776
2893
|
}
|
|
2894
|
+
}
|
|
777
2895
|
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
2896
|
+
// Text the configured setup wrote, while this run's flags name another:
|
|
2897
|
+
// ONE quiet line for the whole run, no redo offered — the flags are for
|
|
2898
|
+
// this run (a CI job's hosted model over a project set up for a local
|
|
2899
|
+
// one). A change of the config itself still gets the full note above.
|
|
2900
|
+
if (keptFromConfig.length > 0) {
|
|
2901
|
+
const n = keptFromConfig.reduce((a, k) => a + k.n, 0);
|
|
2902
|
+
const flags = [
|
|
2903
|
+
cliArgs.method ? `--method ${shellWord(String(cliArgs.method))}` : null,
|
|
2904
|
+
cliArgs.model ? `--model ${shellWord(String(cliArgs.model))}` : null,
|
|
2905
|
+
].filter(Boolean);
|
|
2906
|
+
const setups = [...new Set(keptFromConfig.map(k => k.setup))].join('; ');
|
|
2907
|
+
output.info(`${flags.join(' ')} ${flags.length > 1 ? 'apply' : 'applies'} to this run only: the `
|
|
2908
|
+
+ `${n} translation${n === 1 ? '' : 's'} in the files the configured setup (${setups}) wrote `
|
|
2909
|
+
+ `(${translationsBreakdown(keptFromConfig.map(k => ({ target: k.target, n: k.n })))}) ${n === 1 ? 'is' : 'are'} kept, `
|
|
2910
|
+
+ `and nothing is redone — the flags translate new and changed keys only`
|
|
2911
|
+
+ `${[...ctx.gapsAsked.values()].some(k => k.length > 0) ? ', and the plural messages another setup left incomplete (above)' : ''}.`);
|
|
2912
|
+
}
|
|
2913
|
+
|
|
2914
|
+
// `--dry --show-prompt` that previewed nothing: say why, never silence.
|
|
2915
|
+
if (dryRun && cliArgs['show-prompt'] && ctx.previewsShown === 0) {
|
|
2916
|
+
const named = typeof cliArgs['show-prompt'] === 'string' ? cliArgs['show-prompt'] : null;
|
|
2917
|
+
output.info(named
|
|
2918
|
+
? `--show-prompt ${JSON.stringify(named)}: no key by that name in the source files of the selected pair(s) `
|
|
2919
|
+
+ `(name it as sync lists it${layout.namespaced ? ', with its file: <ns>::<key>' : ''}; a gettext context as msgctxt␄msgid; `
|
|
2920
|
+
+ 'a comma inside a key in a list as \\,). Nothing was sent.'
|
|
2921
|
+
: 'Nothing to preview: no file would send anything to the model (every key is up to date or served from the cache). '
|
|
2922
|
+
+ 'Name a key to see its request anyway: `champollion sync --dry --show-prompt <key>`.');
|
|
2923
|
+
}
|
|
801
2924
|
|
|
802
2925
|
// Aggregate results across all locales
|
|
803
2926
|
const failedKeySet = new Set();
|
|
2927
|
+
let totalSent = 0;
|
|
2928
|
+
let totalRetried = 0;
|
|
2929
|
+
let totalHeld = 0;
|
|
2930
|
+
let totalKept = 0;
|
|
804
2931
|
for (const r of localeResults) {
|
|
2932
|
+
totalHeld += r.held || 0;
|
|
2933
|
+
totalKept += r.kept || 0;
|
|
805
2934
|
totalProcessed += r.processed;
|
|
806
2935
|
totalTMHits += r.tmHits;
|
|
2936
|
+
totalSent += r.sent || 0;
|
|
2937
|
+
totalRetried += r.retried || 0;
|
|
807
2938
|
totalFailed += (r.failed || 0);
|
|
808
2939
|
totalCopied += (r.copied || 0);
|
|
809
2940
|
totalKeptWorkingScript += (r.keptWorkingScript || 0);
|
|
810
|
-
if (r.failed > 0 && r.pairKey) {
|
|
811
|
-
failedPairs.push({ pair: r.pairKey, count: r.failed });
|
|
2941
|
+
if ((r.failed > 0 || r.held > 0) && r.pairKey) {
|
|
2942
|
+
failedPairs.push({ pair: r.pairKey, count: r.failed, held: r.held || 0 });
|
|
812
2943
|
}
|
|
813
2944
|
for (const key of r.failedKeys || []) failedKeySet.add(key);
|
|
814
2945
|
}
|
|
815
2946
|
|
|
816
2947
|
// Summary. Never print [OK] when keys failed — a green success marker on a
|
|
817
2948
|
// partially-failed sync misleads CI and agents into thinking all is well.
|
|
818
|
-
|
|
2949
|
+
// What this run asked a model/API for, and what came free from the cache
|
|
2950
|
+
// (Round 3: nothing said it outright). A dry run reports the estimate's
|
|
2951
|
+
// partition — what the real run would send.
|
|
2952
|
+
const work = dryRun
|
|
2953
|
+
? {
|
|
2954
|
+
sent: (costEstimate?.pairs || []).reduce((n, p) => n + (p.keys || 0), 0),
|
|
2955
|
+
cached: (costEstimate?.pairs || []).reduce((n, p) => n + (p.tmHits || 0), 0),
|
|
2956
|
+
known: !!costEstimate,
|
|
2957
|
+
}
|
|
2958
|
+
: { sent: totalSent, retried: totalRetried, cached: totalTMHits, known: true };
|
|
2959
|
+
// Plural messages without an everyday form: the ones this run wrote (sync
|
|
2960
|
+
// warned per file), and the ones still on disk from an earlier sync — said
|
|
2961
|
+
// here with their repair, and counted, so every run exits 2 while they
|
|
2962
|
+
// remain (pluralGapsOnDisk).
|
|
2963
|
+
//
|
|
2964
|
+
// A dry run reads the same files, and counts them the same way: they stay
|
|
2965
|
+
// after the real run (it exits 2 over them), except the ones that run asks
|
|
2966
|
+
// for again — said per file above (Round 13, Django persona: `sync --dry
|
|
2967
|
+
// --json` said totalPluralGaps 0 over files a real sync exits 2 on).
|
|
2968
|
+
const gapsOnDisk = pluralGapsOnDisk({ layout, units, inputLocale, pairEntries });
|
|
2969
|
+
let gapsAskedAgain = 0;
|
|
2970
|
+
pairEntries.forEach(([pairKey, pc], i) => {
|
|
2971
|
+
const r = localeResults[i];
|
|
2972
|
+
if (!r) return;
|
|
2973
|
+
const askedNow = new Set(ctx.gapsAsked.get(pairKey) || []);
|
|
2974
|
+
if (dryRun) gapsAskedAgain += askedNow.size;
|
|
2975
|
+
for (const { unit, file, gaps } of gapsOnDisk.get(pairKey) || []) {
|
|
2976
|
+
const earlier = gaps.filter(g => !(r.pluralGaps && lockKey(layout, unit.ns, g.key) in r.pluralGaps)
|
|
2977
|
+
&& !(dryRun && askedNow.has(lockKey(layout, unit.ns, g.key))));
|
|
2978
|
+
if (earlier.length === 0) continue;
|
|
2979
|
+
reportPluralGapsOnDisk({ pairKey, code: pc.target, file, unit, gaps: earlier, layout, dryRun });
|
|
2980
|
+
r.pluralGaps = r.pluralGaps || {};
|
|
2981
|
+
for (const g of earlier) r.pluralGaps[lockKey(layout, unit.ns, g.key)] = g.missing;
|
|
2982
|
+
}
|
|
2983
|
+
});
|
|
2984
|
+
const totalPluralGaps = localeResults.reduce((n, r) => n + Object.keys(r?.pluralGaps || {}).length, 0);
|
|
2985
|
+
// The one repair command when every gap is in one file (else each file's
|
|
2986
|
+
// own, printed above), and whether a catalog marks them.
|
|
2987
|
+
const gapFiles = [...gapsOnDisk.entries()].flatMap(([pairKey, files]) => files.map(f => ({ pairKey, ...f })));
|
|
2988
|
+
const pluralRepair = gapFiles.length === 1
|
|
2989
|
+
? redoCommand(gapFiles[0].gaps.map(g => g.key), { pair: gapFiles[0].pairKey, ns: layout.namespaced ? gapFiles[0].unit.ns : '', fresh: true })
|
|
2990
|
+
: null;
|
|
2991
|
+
const summary = formatSyncSummary(dryRun, totalProcessed, totalFailed, totalCopied, work, totalHeld, {
|
|
2992
|
+
contentPending: (costEstimate?.content?.pendingTranslations || 0) > 0,
|
|
2993
|
+
pluralGaps: totalPluralGaps,
|
|
2994
|
+
pluralRepair,
|
|
2995
|
+
pluralMarked: gapFiles.some(f => f.gaps.some(g => g.marked)),
|
|
2996
|
+
});
|
|
819
2997
|
if (summary.ok) output.ok(summary.message);
|
|
820
2998
|
else output.warn(summary.message);
|
|
2999
|
+
// Said again at the end, where a dry run's reader looks for the verdict.
|
|
3000
|
+
if (dryRun && preflightFailures.length > 0) {
|
|
3001
|
+
// Each reason is a sentence of its own ("No OpenRouter API key (…)."):
|
|
3002
|
+
// joined without its full stop, so the list never reads ".;" (Round 8).
|
|
3003
|
+
output.warn(`A real run would stop before translating: ${preflightFailures.map(f => `${f.pair}: ${String(f.reason).replace(/[.;\s]+$/, '')}`).join('; ')}.`);
|
|
3004
|
+
}
|
|
3005
|
+
// The --max-cost verdict, said once — here, beside the preflight's (lib/cost-report.js reportDryRunMaxCost).
|
|
3006
|
+
if (dryMaxCost) output[dryMaxCost.level](dryMaxCost.message);
|
|
3007
|
+
// The exit code the real run would end with, as far as a preview can know
|
|
3008
|
+
// it: the dry run itself exits 0 (Round 11), and said this run's partial
|
|
3009
|
+
// verdict nowhere (Round 13, Django persona). What only the real run finds
|
|
3010
|
+
// — a refusal by the quality gate, a failed verification — can still turn
|
|
3011
|
+
// a 0 into a 2; the reasons a preview knows are named.
|
|
3012
|
+
const realRun = dryRun
|
|
3013
|
+
? predictRealRun({ preflightFailures, dryMaxCost, unmatched: unmatchedNamed.length, pluralGaps: totalPluralGaps, gapsAskedAgain, costEstimate })
|
|
3014
|
+
: null;
|
|
3015
|
+
if (realRun && !realRun.wouldStop && realRun.exitCode !== 0) {
|
|
3016
|
+
output.warn(`A real sync would exit ${realRun.exitCode}${realRun.exitCode === 2 ? ' (partial)' : ''}: ${realRun.reasons.join('; ')}. `
|
|
3017
|
+
+ dryRunCiHint('realRun.exitCode'));
|
|
3018
|
+
}
|
|
821
3019
|
|
|
822
3020
|
// Keys kept in the working script (unmapped letters) are informational —
|
|
823
3021
|
// valid translations, just not converted. Surface the count once so the
|
|
@@ -836,56 +3034,130 @@ async function runSync(options = {}) {
|
|
|
836
3034
|
if (failedPairs.length > 0) {
|
|
837
3035
|
output.raw('');
|
|
838
3036
|
output.warn('Failure summary:');
|
|
839
|
-
for (const { pair, count } of failedPairs) {
|
|
840
|
-
|
|
3037
|
+
for (const { pair, count, held } of failedPairs) {
|
|
3038
|
+
const parts = [count > 0 && `${count} key(s) not translated`, held > 0 && `${held} held back`].filter(Boolean);
|
|
3039
|
+
output.warn(` ${pair}: ${parts.join(', ')}`);
|
|
3040
|
+
}
|
|
3041
|
+
output.warn(`Total: ${[totalFailed > 0 && `${totalFailed} key(s) not translated`, totalHeld > 0 && `${totalHeld} held back`]
|
|
3042
|
+
.filter(Boolean).join(', ')} across ${failedPairs.length} locale(s).`);
|
|
3043
|
+
// What the next sync does with them — said per outcome, so it is true
|
|
3044
|
+
// (lib/locale-state.js has the rules).
|
|
3045
|
+
const byFate = (fate) => localeResults.flatMap(r => (r.fates?.[fate] || []).map(k => ({ pair: r.pairKey, key: k })));
|
|
3046
|
+
const retry = byFate('retry');
|
|
3047
|
+
const pendingNext = byFate('pending-retry');
|
|
3048
|
+
const heldNext = byFate('held');
|
|
3049
|
+
if (retry.length > 0) {
|
|
3050
|
+
output.warn(` ${retry.length} key(s) got no usable answer (missing from the response, or the method failed) — the next sync asks for them again.`);
|
|
841
3051
|
}
|
|
842
|
-
|
|
843
|
-
|
|
3052
|
+
if (pendingNext.length > 0) {
|
|
3053
|
+
output.warn(` ${pendingNext.length} key(s) this redo could not finish are recorded as pending in ${LOCK_FILENAME} — the next plain `
|
|
3054
|
+
+ '`champollion sync` asks the model for them once more (`champollion status` lists them). If it refuses them again, they are held back.');
|
|
3055
|
+
}
|
|
3056
|
+
for (const line of describeHeldNext(heldNext, (keys, pair) => redoCommand(keys, { pair }))) output.warn(line);
|
|
3057
|
+
}
|
|
3058
|
+
if (totalKept > 0 && !dryRun) {
|
|
3059
|
+
output.info(`Kept ${totalKept} hand-edited value(s) — a bulk redo never replaces a person's text; \`--redo keys:<key>\` replaces one.`);
|
|
844
3060
|
}
|
|
845
3061
|
|
|
846
3062
|
// Write the updated hash manifest so the next sync knows
|
|
847
3063
|
// what state the translations are based on.
|
|
848
3064
|
// Skip in dry-run mode — don't mark stale keys as resolved.
|
|
849
3065
|
if (!dryRun) {
|
|
850
|
-
//
|
|
851
|
-
//
|
|
852
|
-
//
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
//
|
|
864
|
-
//
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
3066
|
+
// The locale files are already written: whatever goes wrong below, the
|
|
3067
|
+
// translations this run paid for must reach the cache (Round 3: a crash
|
|
3068
|
+
// here wrote the files and lost tm.json).
|
|
3069
|
+
try {
|
|
3070
|
+
// ── Manifest retry-safety ─────────────────────────────────────
|
|
3071
|
+
// A key that failed in ANY locale must keep its OLD hash: persisting the
|
|
3072
|
+
// NEW hash would mark the changed source as resolved, and the failed key
|
|
3073
|
+
// would never be re-detected as 'changed' (the locale file still has the
|
|
3074
|
+
// stale translation, so missing-key detection can't catch it either).
|
|
3075
|
+
//
|
|
3076
|
+
// CONSERVATIVE by design: one restore is global, so the key re-fires for
|
|
3077
|
+
// ALL locales next sync — but the re-fire is TM-served (zero API cost)
|
|
3078
|
+
// for locales that already succeeded, so the only real work is the retry
|
|
3079
|
+
// that actually failed. Keys with no prior hash are dropped from the
|
|
3080
|
+
// manifest entirely; they were never written, so missing-key detection
|
|
3081
|
+
// re-fires them regardless.
|
|
3082
|
+
for (const key of failedKeySet) {
|
|
3083
|
+
// Own-property check: `in` walks the prototype chain, so a key
|
|
3084
|
+
// literally named "toString"/"valueOf" would mis-resolve.
|
|
3085
|
+
if (Object.prototype.hasOwnProperty.call(oldManifest, key)) {
|
|
3086
|
+
currentManifest[key] = oldManifest[key];
|
|
3087
|
+
} else {
|
|
3088
|
+
delete currentManifest[key];
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
3091
|
+
// Per-locale record: drop locales the config no longer has and keys a
|
|
3092
|
+
// locale no longer expects (locales this run did not process keep theirs).
|
|
3093
|
+
const configured = new Set(Object.keys(config.resolvedLanguages || {}));
|
|
3094
|
+
for (const k of Object.keys(config.pairs || {})) {
|
|
3095
|
+
const t = parsePairKey(k).target;
|
|
3096
|
+
if (t) configured.add(t);
|
|
3097
|
+
}
|
|
3098
|
+
for (const [, pc] of pairEntries) configured.add(pc.target);
|
|
3099
|
+
const processed = new Set(pairEntries.map(([, pc]) => pc.target));
|
|
3100
|
+
const expectedByLocale = new Map();
|
|
3101
|
+
for (const code of configured) {
|
|
3102
|
+
if (!processed.has(code)) { expectedByLocale.set(code, null); continue; }
|
|
3103
|
+
const keys = new Set();
|
|
3104
|
+
for (const unit of units) {
|
|
3105
|
+
for (const k of Object.keys(expectedForTarget(unit, inputLocale, code).flat)) keys.add(lockKey(layout, unit.ns, k));
|
|
3106
|
+
}
|
|
3107
|
+
expectedByLocale.set(code, keys);
|
|
3108
|
+
}
|
|
3109
|
+
lockState.prune(expectedByLocale);
|
|
3110
|
+
writeManifest(cwd, currentManifest, lockState.toJSON());
|
|
3111
|
+
// A person's wording this run replaced: kept, in a tracked file.
|
|
3112
|
+
const recorded = recordReplacedEdits(cwd, ctx.replacedEdits);
|
|
3113
|
+
if (recorded) {
|
|
3114
|
+
output.warn(`Recorded ${ctx.replacedEdits.length} replaced hand edit(s) in ${recorded} (commit it with the lock — `
|
|
3115
|
+
+ 'it is the only copy of that wording).');
|
|
3116
|
+
}
|
|
3117
|
+
|
|
3118
|
+
// A model switch left keys pending and this run finished them: the
|
|
3119
|
+
// switch is done for that locale (the "Model changed" notice stops).
|
|
3120
|
+
pairEntries.forEach(([, pc], i) => {
|
|
3121
|
+
const before = pendingBefore.get(pc.target) || [];
|
|
3122
|
+
const nowPending = Object.keys(lockState.peek(pc.target).pending).length;
|
|
3123
|
+
if (before.some(r => String(r).includes('fresh-on-model-change')) && nowPending === 0
|
|
3124
|
+
&& (localeResults[i]?.failed || 0) === 0 && (localeResults[i]?.held || 0) === 0) {
|
|
3125
|
+
tm._meta = tm._meta || {};
|
|
3126
|
+
tm._meta.switchedTo = tm._meta.switchedTo || {};
|
|
3127
|
+
tm._meta.switchedTo[pc.target] = tmMethodKey(pc);
|
|
3128
|
+
}
|
|
3129
|
+
});
|
|
3130
|
+
|
|
3131
|
+
// A locale re-translated in full under the current model (--redo all
|
|
3132
|
+
// --fresh-on-model-change, no failures) has switched: record it so the
|
|
3133
|
+
// "Model changed" notice stops for it. The notice compares cached-entry
|
|
3134
|
+
// counts per model, and leftovers (strings since deleted) kept it firing
|
|
3135
|
+
// after the switch was done (Round 2, Next.js persona).
|
|
3136
|
+
if (cliArgs.force && freshOnModelChange) {
|
|
3137
|
+
tm._meta = tm._meta || {};
|
|
3138
|
+
tm._meta.switchedTo = tm._meta.switchedTo || {};
|
|
3139
|
+
pairEntries.forEach(([, pc], i) => {
|
|
3140
|
+
if ((localeResults[i]?.failed || 0) === 0) tm._meta.switchedTo[pc.target] = tmMethodKey(pc);
|
|
3141
|
+
});
|
|
869
3142
|
}
|
|
870
|
-
}
|
|
871
|
-
writeManifest(cwd, currentManifest);
|
|
872
3143
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
3144
|
+
} finally {
|
|
3145
|
+
// Persist TM if it was mutated during this sync (stores OR evictions —
|
|
3146
|
+
// a size check would miss eviction-only runs and same-key replacements).
|
|
3147
|
+
// Under --fresh/--no-tm too: what was paid for is cached.
|
|
3148
|
+
if (isTMDirty(tm)) {
|
|
3149
|
+
saveTM(cwd, tm);
|
|
3150
|
+
output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
|
|
3151
|
+
}
|
|
881
3152
|
}
|
|
882
3153
|
}
|
|
883
3154
|
|
|
884
|
-
// Content sync — translate
|
|
3155
|
+
// Content sync — translate the contentDir Markdown files if configured.
|
|
885
3156
|
// Uses the same resolved pair graph as key-value sync, ensuring method
|
|
886
3157
|
// dispatch is consistent across both translation modes.
|
|
3158
|
+
let content = null;
|
|
887
3159
|
if (config.contentDir) {
|
|
888
|
-
await runContentSync({
|
|
3160
|
+
content = await runContentSync({
|
|
889
3161
|
contentDir: config.contentDir,
|
|
890
3162
|
sourceLocale: inputLocale,
|
|
891
3163
|
pairs: resolvedPairs,
|
|
@@ -893,11 +3165,87 @@ async function runSync(options = {}) {
|
|
|
893
3165
|
apiKey,
|
|
894
3166
|
dryRun,
|
|
895
3167
|
noTM,
|
|
3168
|
+
freshOnModelChange,
|
|
3169
|
+
forceContent,
|
|
3170
|
+
fileScope,
|
|
896
3171
|
cwd,
|
|
897
3172
|
concurrency: config.contentConcurrency || 12,
|
|
3173
|
+
fallbackBudget,
|
|
3174
|
+
sharedOutputsFor: ctx.sharedOutputsFor,
|
|
898
3175
|
});
|
|
899
3176
|
}
|
|
900
3177
|
|
|
3178
|
+
// One text answering several different source strings (lib/validate.js
|
|
3179
|
+
// SharedOutputIndex): refused where it came back this run, but earlier
|
|
3180
|
+
// members of the group were already written — name them, with the redo.
|
|
3181
|
+
//
|
|
3182
|
+
// Two things outlive the run (Round 6, school persona: the printed repair
|
|
3183
|
+
// failed, and `--force-content` then re-served the same sentence from the
|
|
3184
|
+
// cache with no warning):
|
|
3185
|
+
// - the cache entries that produced the written members are EVICTED, so
|
|
3186
|
+
// any redo — with or without --fresh — asks the model again instead of
|
|
3187
|
+
// re-serving the sentence for free;
|
|
3188
|
+
// - the sentence is remembered per locale (TM _meta.memorized), so when
|
|
3189
|
+
// the model answers a repair with it again, it is refused from its
|
|
3190
|
+
// first source on (lib/validate.js markMemorized) and the key goes to
|
|
3191
|
+
// the pair's fallback.
|
|
3192
|
+
// The key-value TM object was saved before the content lane loaded its
|
|
3193
|
+
// own; both are on disk now, so this works on a fresh read and the
|
|
3194
|
+
// verification below gets that same object (a stale one would drop the
|
|
3195
|
+
// content lane's entries if verify saved it).
|
|
3196
|
+
let tmAfter = tm;
|
|
3197
|
+
if (!dryRun && sharedOutputIndexes.size > 0) {
|
|
3198
|
+
const flaggedNow = [...sharedOutputIndexes].filter(([, index]) => index.flagged.size > 0);
|
|
3199
|
+
if (flaggedNow.length > 0) {
|
|
3200
|
+
tmAfter = config.contentDir ? loadTM(cwd) : tm;
|
|
3201
|
+
const evictor = createTMEvictor(tmAfter);
|
|
3202
|
+
let evicted = 0;
|
|
3203
|
+
for (const [code, index] of flaggedNow) {
|
|
3204
|
+
const memo = new Set((tmAfter._meta?.memorized?.[code]) || []);
|
|
3205
|
+
for (const g of index.flagged.values()) {
|
|
3206
|
+
memo.add(g.value);
|
|
3207
|
+
// Members accepted (and written) before the repeat showed.
|
|
3208
|
+
const members = [...(index.byOutput.get(SharedOutputIndex.outputForm(g.value))?.values() || [])];
|
|
3209
|
+
const written = members.map(v => v.key);
|
|
3210
|
+
for (const member of members) {
|
|
3211
|
+
// A sentence of a longer value: what was cached is the whole value.
|
|
3212
|
+
const m = member.whole ? { ...member, ...member.whole } : member;
|
|
3213
|
+
if (typeof m.source !== 'string' || typeof m.value !== 'string') continue;
|
|
3214
|
+
const texts = new Set([m.source]);
|
|
3215
|
+
if (!m.key.startsWith('content:')) {
|
|
3216
|
+
const bare = layout.namespaced && m.key.includes(NS_SEPARATOR) ? m.key.slice(m.key.indexOf(NS_SEPARATOR) + NS_SEPARATOR.length) : m.key;
|
|
3217
|
+
texts.add(tmSourceText(bare, m.source));
|
|
3218
|
+
}
|
|
3219
|
+
for (const t of texts) evicted += evictor.evictProducing(t, code, m.value);
|
|
3220
|
+
}
|
|
3221
|
+
if (written.length === 0) continue;
|
|
3222
|
+
const pairKey = pairEntries.find(([, pc]) => pc.target === code)?.[0] || `${inputLocale}:${code}`;
|
|
3223
|
+
const shown = g.value.length > 60 ? `${g.value.slice(0, 57)}...` : g.value;
|
|
3224
|
+
const keys = written.filter(k => !k.startsWith('content:'));
|
|
3225
|
+
// The path sync prints for a content file (relative to the
|
|
3226
|
+
// contentDir) — what `--redo files:` matches (lib/file-scope.js).
|
|
3227
|
+
const files = [...new Set(written.filter(k => k.startsWith('content:')).map(k => k.slice(8).replace(/(#\d+| front matter:.*)$/, '')))];
|
|
3228
|
+
const redo = [
|
|
3229
|
+
keys.length > 0 && `\`${redoCommand(keys.slice(0, 8), { pair: pairKey })}\``,
|
|
3230
|
+
...files.slice(0, 3).map(f => `\`${contentRedoCommand(f, { pair: pairKey })}\``),
|
|
3231
|
+
].filter(Boolean).join(', ');
|
|
3232
|
+
const fb = pairEntries.find(([, pc]) => pc.target === code)?.[1]?.fallback;
|
|
3233
|
+
output.warn(`${pairKey}: ${JSON.stringify(shown)} came back for ${g.sources.length} different source string(s) — a model `
|
|
3234
|
+
+ `repeating a memorized sentence. Refused once the repeat showed, but it was already written (earlier in this run, or by an earlier sync) for `
|
|
3235
|
+
+ `${written.map(k => k.replace(/^content:/, '')).slice(0, 6).join(', ')}${written.length > 6 ? ', …' : ''}. `
|
|
3236
|
+
+ 'Removed from the cache, and remembered: if the model answers with it again, it is refused. '
|
|
3237
|
+
+ `Check those and ask again: ${redo}${fb ? ` (what ${pairEntries.find(([, pc]) => pc.target === code)[1].method} refuses goes to the fallback, ${fb.method})` : ' — a "fallback" method on the pair takes what the model can only answer this way'}.`);
|
|
3238
|
+
}
|
|
3239
|
+
tmAfter._meta = tmAfter._meta || {};
|
|
3240
|
+
tmAfter._meta.memorized = tmAfter._meta.memorized || {};
|
|
3241
|
+
tmAfter._meta.memorized[code] = [...memo];
|
|
3242
|
+
}
|
|
3243
|
+
saveTM(cwd, tmAfter);
|
|
3244
|
+
if (evicted > 0) output.info(`[TM] Removed ${evicted} cached translation(s) holding a memorized sentence.`);
|
|
3245
|
+
}
|
|
3246
|
+
}
|
|
3247
|
+
if (tmAfter === tm && config.contentDir && content && !dryRun) tmAfter = loadTM(cwd);
|
|
3248
|
+
|
|
901
3249
|
// ── Post-sync verification ──────────────────────────────────────
|
|
902
3250
|
// Re-read written locale files from disk and confirm translations
|
|
903
3251
|
// are actually present and correct. Catches the gap between the CLI
|
|
@@ -907,27 +3255,78 @@ async function runSync(options = {}) {
|
|
|
907
3255
|
// Verification errors feed the exit code: a sync that wrote files but
|
|
908
3256
|
// left [EN] markers, missing keys, or wrong-script values is NOT a clean
|
|
909
3257
|
// pass, and a CI gate must see that.
|
|
3258
|
+
//
|
|
3259
|
+
// Scope: `sync --pair en:fr` verifies the pair(s) that ran — reporting
|
|
3260
|
+
// Spanish errors on a French-only run is noise about work nobody asked for.
|
|
3261
|
+
//
|
|
3262
|
+
// Damage verify finds (ICU structure, placeholders, hollowed values) that
|
|
3263
|
+
// THIS run's Translation Memory produced is evicted from it, so the next
|
|
3264
|
+
// sync — or the `--force-keys` the report names — re-translates instead
|
|
3265
|
+
// of re-serving it for free (lib/tm-evict.js). Not under --no-tm.
|
|
910
3266
|
let verifyErrors = 0;
|
|
911
3267
|
let verifyWarnings = 0;
|
|
912
|
-
|
|
913
|
-
|
|
3268
|
+
let tmEvicted = 0;
|
|
3269
|
+
const verifyRan = !dryRun && !audit && !cliArgs['no-verify'];
|
|
3270
|
+
if (verifyRan) {
|
|
3271
|
+
// Translation is done: verify consults the cache (TM-confirmed echoes are
|
|
3272
|
+
// not findings) even after a --fresh run, and evicts what it rejects.
|
|
3273
|
+
setTMReads(tmAfter, true);
|
|
3274
|
+
// What this run left undone: verify's closing line names it instead of
|
|
3275
|
+
// an [OK] over a run that exits 2 (lib/verify.js printSummary).
|
|
3276
|
+
const contentLeft = content ? (content.heldBack || 0) + (content.refused || 0) : 0;
|
|
3277
|
+
const incomplete = [
|
|
3278
|
+
totalFailed > 0 && `${totalFailed} key(s) not translated`,
|
|
3279
|
+
totalHeld > 0 && `${totalHeld} key(s) held back`,
|
|
3280
|
+
totalPluralGaps > 0 && `${totalPluralGaps} plural message(s) without a form the language uses for ordinary counts`,
|
|
3281
|
+
content && content.failed > 0 && `${content.failed} content file(s) not translated`,
|
|
3282
|
+
contentLeft > 0 && `${contentLeft} Markdown block(s) or field(s) not translated`,
|
|
3283
|
+
].filter(Boolean).join(', ');
|
|
3284
|
+
const v = await verifyLocales(config, cwd, {
|
|
3285
|
+
afterSync: true,
|
|
3286
|
+
incomplete: incomplete || null,
|
|
3287
|
+
noTranslate,
|
|
3288
|
+
locales: cliArgs.pair ? pairEntries.map(([, p]) => p.target) : null,
|
|
3289
|
+
tm: tmAfter,
|
|
3290
|
+
noTM,
|
|
3291
|
+
});
|
|
914
3292
|
verifyErrors = v.errors;
|
|
915
3293
|
verifyWarnings = v.warnings;
|
|
3294
|
+
tmEvicted = v.tmEvicted || 0;
|
|
916
3295
|
}
|
|
917
3296
|
|
|
918
3297
|
// Per-locale structured results (zip the parallel pMap output back to its
|
|
919
3298
|
// pair keys) — used by the --json summary so agents don't regex log lines.
|
|
920
3299
|
const localeSummary = pairEntries.map(([pairKey, pairConfig], i) => {
|
|
921
3300
|
const r = localeResults[i] || {};
|
|
3301
|
+
// A dry run sends nothing: what each locale WOULD send and serve from the
|
|
3302
|
+
// cache is the estimate's partition for its pair — the same numbers the
|
|
3303
|
+
// run-wide sentToModel sums, so the locales add up to it (Round 9,
|
|
3304
|
+
// i18next persona: every locale said 0 under a total of 4).
|
|
3305
|
+
const planned = dryRun ? (costEstimate?.pairs || []).find(p => p.pair === pairKey) : null;
|
|
922
3306
|
return {
|
|
923
3307
|
pair: pairKey,
|
|
924
3308
|
target: pairConfig.target,
|
|
925
3309
|
processed: r.processed || 0,
|
|
926
3310
|
failed: r.failed || 0,
|
|
927
|
-
tmHits: r.tmHits || 0,
|
|
3311
|
+
tmHits: dryRun ? (planned?.tmHits || 0) : (r.tmHits || 0),
|
|
3312
|
+
sentToModel: dryRun ? (planned?.keys || 0) : (r.sent || 0),
|
|
3313
|
+
// Plural messages written without forms the language uses for
|
|
3314
|
+
// ordinary counts: key → missing CLDR categories (sync warned).
|
|
3315
|
+
...(r.pluralGaps && Object.keys(r.pluralGaps).length > 0 && { pluralGaps: r.pluralGaps }),
|
|
928
3316
|
copied: r.copied || 0,
|
|
929
3317
|
keptWorkingScript: r.keptWorkingScript || 0,
|
|
3318
|
+
// Refused before by this method: not sent, not billed (lib/locale-state.js).
|
|
3319
|
+
held: r.held || 0,
|
|
3320
|
+
...(r.heldKeys && r.heldKeys.length > 0 && { heldKeys: r.heldKeys }),
|
|
3321
|
+
// Hand-edited values a bulk redo kept, and ones this run replaced (recorded).
|
|
3322
|
+
kept: r.kept || 0,
|
|
3323
|
+
...(r.keptKeys && r.keptKeys.length > 0 && { keptKeys: r.keptKeys }),
|
|
3324
|
+
replacedEdits: r.replaced || 0,
|
|
3325
|
+
// Next-sync fate of each key left untranslated: retry | pending-retry | held.
|
|
3326
|
+
...(r.fates && Object.values(r.fates).some(v => v.length > 0) && { nextSync: r.fates }),
|
|
930
3327
|
...(r.queuedKeys && { queuedKeys: r.queuedKeys }),
|
|
3328
|
+
// What the pair's fallback method did (pairs with a fallback, real runs).
|
|
3329
|
+
...(r.fallback && { fallback: fallbackSummary(r.fallback) }),
|
|
931
3330
|
};
|
|
932
3331
|
});
|
|
933
3332
|
|
|
@@ -940,7 +3339,20 @@ async function runSync(options = {}) {
|
|
|
940
3339
|
dryRun,
|
|
941
3340
|
totalProcessed,
|
|
942
3341
|
totalFailed,
|
|
943
|
-
|
|
3342
|
+
totalHeld,
|
|
3343
|
+
// Plural messages without a form the language uses for ordinary counts:
|
|
3344
|
+
// after the run — in a dry run, those on disk the real run does not ask
|
|
3345
|
+
// for again (it exits 2 over them; realRun says so).
|
|
3346
|
+
totalPluralGaps,
|
|
3347
|
+
...(dryRun && { pluralGapsAskedAgain: gapsAskedAgain }),
|
|
3348
|
+
// --prune plural-extras: each key removed (a dry run: would remove).
|
|
3349
|
+
...(prune.size > 0 && { pruned: ctx.pruned }),
|
|
3350
|
+
totalKept,
|
|
3351
|
+
// A dry run: what the real run would serve from the cache (the estimate).
|
|
3352
|
+
tmHits: dryRun ? work.cached : totalTMHits,
|
|
3353
|
+
// Keys sent to the method (TM misses) and re-sent with gate feedback.
|
|
3354
|
+
sentToModel: dryRun ? work.sent : totalSent,
|
|
3355
|
+
resentWithFeedback: dryRun ? 0 : totalRetried,
|
|
944
3356
|
// No-translate keys copied verbatim: never sent to a backend, never
|
|
945
3357
|
// gated, never billed. Counted apart from totalProcessed so an agent
|
|
946
3358
|
// reading this can tell translation work from passthrough.
|
|
@@ -949,21 +3361,58 @@ async function runSync(options = {}) {
|
|
|
949
3361
|
// could not map some of their letters (see scriptFallback).
|
|
950
3362
|
totalKeptWorkingScript,
|
|
951
3363
|
noTranslate: { patterns: noTranslate.patterns, autoDetectUrls: noTranslate.urls },
|
|
952
|
-
|
|
3364
|
+
// Not run in a dry run (nothing was written) or under --no-verify: its
|
|
3365
|
+
// counts are null then, never a 0 that reads as "verified".
|
|
3366
|
+
verify: verifyRan
|
|
3367
|
+
? { ran: true, errors: verifyErrors, warnings: verifyWarnings, tmEvicted }
|
|
3368
|
+
: { ran: false, errors: null, warnings: null, tmEvicted: 0 },
|
|
953
3369
|
failedPairs,
|
|
954
3370
|
locales: localeSummary,
|
|
955
3371
|
// Pre-run cost estimate (null when estimation failed) — the human table
|
|
956
3372
|
// is output.raw and therefore invisible in --json, so the structured
|
|
957
3373
|
// estimate must ride the summary for agents/CI.
|
|
958
3374
|
costEstimate,
|
|
3375
|
+
// Dry runs: whether the real run's preflight would pass (missing keys).
|
|
3376
|
+
...(dryRun && { preflight: { ready: preflightFailures.length === 0, failures: preflightFailures } }),
|
|
3377
|
+
// Dry runs: the exit code the real run would end with, as far as a
|
|
3378
|
+
// preview can know (predictRealRun): wouldStop = before translating.
|
|
3379
|
+
...(realRun && { realRun }),
|
|
3380
|
+
// Dry runs with --max-cost: whether the real run would stop at the cap
|
|
3381
|
+
// (it would exit 2 before any API call); the dry run itself exits 0.
|
|
3382
|
+
...(dryMaxCost && { maxCost: {
|
|
3383
|
+
cap: dryMaxCost.cap, estimatedCost: dryMaxCost.estimatedCost, wouldStop: dryMaxCost.wouldStop,
|
|
3384
|
+
...(dryMaxCost.wouldStop && { exitCode: 2, reason: dryMaxCost.reason }),
|
|
3385
|
+
// The preflight would stop the real run first (exit 1), before the cap.
|
|
3386
|
+
...(dryMaxCost.stopsEarlier && { exitCode: 1, stopsEarlier: dryMaxCost.stopsEarlier }),
|
|
3387
|
+
} }),
|
|
3388
|
+
tmModelSwitch,
|
|
3389
|
+
// Content files (contentDir): translated / failed / kept-edit / held counts and lists.
|
|
3390
|
+
content,
|
|
3391
|
+
// Keys named for a redo that match no key (the run exits 1), each with
|
|
3392
|
+
// the closest keys that exist.
|
|
3393
|
+
...(unmatchedNamed.length > 0 && { unmatchedKeys: unmatchedKeysSummary(unmatchedNamed) }),
|
|
959
3394
|
});
|
|
960
3395
|
|
|
961
3396
|
// Return result for exit code determination.
|
|
962
3397
|
// totalFailed > 0 means some keys couldn't be translated; verifyErrors > 0
|
|
963
3398
|
// means written files didn't pass verification. The caller maps these to a
|
|
964
3399
|
// non-zero exit code (see lib/commands/sync.js computeExitCode).
|
|
965
|
-
|
|
3400
|
+
// Named keys that matched nothing: said again at the end, where a reader
|
|
3401
|
+
// (or CI) looks for the verdict — and the run fails.
|
|
3402
|
+
reportUnmatchedKeys(unmatchedNamed, cliArgs);
|
|
3403
|
+
return {
|
|
3404
|
+
totalProcessed, totalFailed, totalHeld, totalKept, totalCopied, totalKeptWorkingScript, failedPairs, verifyErrors, verifyWarnings,
|
|
3405
|
+
// Plural messages left without a form the language uses (exit 2).
|
|
3406
|
+
totalPluralGaps: dryRun ? 0 : totalPluralGaps,
|
|
3407
|
+
contentTranslated: content ? content.translated : 0,
|
|
3408
|
+
contentFailed: content ? content.failed : 0,
|
|
3409
|
+
// Markdown blocks/fields refused before and held back, or refused this
|
|
3410
|
+
// run (lib/content-refusals.js): not translated — not a clean pass.
|
|
3411
|
+
contentHeldBack: content ? content.heldBack || 0 : 0,
|
|
3412
|
+
contentRefused: content ? content.refused || 0 : 0,
|
|
3413
|
+
unmatchedKeys: unmatchedNamed.map(m => m.name),
|
|
3414
|
+
};
|
|
966
3415
|
}
|
|
967
3416
|
|
|
968
|
-
export { runSync, runContentSync, resolveRuntime, loadApiKey, formatSyncSummary };
|
|
3417
|
+
export { runSync, runContentSync, resolveRuntime, loadApiKey, formatSyncSummary, reportGeneratedPluralKeys, reportPluralGaps };
|
|
969
3418
|
|