champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
package/lib/content-sync.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Content sync — translates
|
|
2
|
+
* Content sync — translates the Markdown/MDX files of a contentDir (a Hugo
|
|
3
|
+
* site's content/ or any folder of Markdown), each translation written beside
|
|
4
|
+
* its source as <name>.<locale>.md (lib/content.js getTargetContentPath).
|
|
3
5
|
*
|
|
4
6
|
* WHY THIS EXISTS: This was extracted from sync.js to reduce the
|
|
5
7
|
* god-module's line count and give content translation its own
|
|
@@ -35,15 +37,31 @@
|
|
|
35
37
|
* results that pass the existing checks (non-null API result; structural
|
|
36
38
|
* block-batch validation; body placeholder-corruption check on the
|
|
37
39
|
* REASSEMBLED body) are stored, and --no-tm bypasses the cache entirely.
|
|
40
|
+
*
|
|
41
|
+
* EDITS MADE BY HAND (lib/content-review.js): every file this lane writes is
|
|
42
|
+
* also recorded in the content lock ("written:<relPath>:<locale>"), block by
|
|
43
|
+
* block. Before a changed source re-translates a file, the file on disk is
|
|
44
|
+
* compared with that record: paragraphs and front-matter fields a person
|
|
45
|
+
* changed are KEPT when the source text they translate is unchanged (never
|
|
46
|
+
* sent to the API, never cached as machine output), and the run says so. An
|
|
47
|
+
* edited paragraph whose own source changed is re-translated with its edited
|
|
48
|
+
* wording printed; edits that cannot be matched paragraph by paragraph leave
|
|
49
|
+
* the whole file as is until it is updated by hand or named for a redo.
|
|
38
50
|
*/
|
|
39
51
|
|
|
40
52
|
import fs from 'node:fs';
|
|
41
53
|
import path from 'node:path';
|
|
42
54
|
import crypto from 'node:crypto';
|
|
43
55
|
import { translateBatch, translateRawContent, getMethod } from './translate.js';
|
|
44
|
-
import { checkContentPreservation } from './validate.js';
|
|
45
|
-
import { loadTM, saveTM, lookupTM,
|
|
56
|
+
import { checkContentPreservation, contentGateFault } from './validate.js';
|
|
57
|
+
import { loadTM, saveTM, lookupTM, peekTM, storeTM, evictTM, isTMDirty, tmSize, tmMethodKey, partitionByTM, setModelCarryover, setTMReads, bypassTMFor, cacheKey, describeTMChanges, adoptLegacyCoachingKeys } from './tm.js';
|
|
58
|
+
import {
|
|
59
|
+
writtenRecordKey, parseWrittenRecord, buildWrittenRecord, buildAdoptedRecord, buildBootstrapRecord,
|
|
60
|
+
assessExistingTarget, matchEditedBlocks, matchEditedFields, describeEdits,
|
|
61
|
+
} from './content-review.js';
|
|
46
62
|
import { output } from './output.js';
|
|
63
|
+
import { missingKeyAdvice } from './missing-key.js';
|
|
64
|
+
import { billableContentChars, translatableBlockSources } from './content-estimate.js';
|
|
47
65
|
import { DEFAULT_REGISTERS } from './registers.js';
|
|
48
66
|
import { isPathContained } from './security.js';
|
|
49
67
|
import { pMap } from './concurrent.js';
|
|
@@ -63,12 +81,23 @@ import {
|
|
|
63
81
|
splitBlocks, buildBlockBatchPrompt, parseBlockBatchResponse,
|
|
64
82
|
translateBlockBatchResilient, assertSegmentationMode,
|
|
65
83
|
} from './segment.js';
|
|
84
|
+
import {
|
|
85
|
+
lookupContentTM, serveFieldsFromFallbackCache, translateFieldsWithFallback,
|
|
86
|
+
translateBlocksWithFallback, translatePageWithFallback,
|
|
87
|
+
newFallbackReport, addToTally, fallbackSummary, printFallbackReport, warnFallbackMajority,
|
|
88
|
+
refuseRepeatedCachedPage,
|
|
89
|
+
} from './fallback.js';
|
|
90
|
+
import {
|
|
91
|
+
readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
|
|
92
|
+
describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME,
|
|
93
|
+
} from './content-refusals.js';
|
|
94
|
+
import { contentRedoCommand } from './verify.js';
|
|
66
95
|
|
|
67
96
|
/**
|
|
68
|
-
* Run content sync — translate
|
|
97
|
+
* Run content sync — translate the contentDir's Markdown/MDX files.
|
|
69
98
|
*
|
|
70
99
|
* @param {object} options
|
|
71
|
-
* @param {string} options.contentDir - Path to
|
|
100
|
+
* @param {string} options.contentDir - Path to the content directory (any folder of Markdown/MDX)
|
|
72
101
|
* @param {string} options.sourceLocale - Source language code
|
|
73
102
|
* @param {Map<string, object>} options.pairs - Resolved pair graph (pairKey → pairConfig)
|
|
74
103
|
* @param {string[]|null} options.translatableFields - Front matter fields to translate
|
|
@@ -77,8 +106,13 @@ import {
|
|
|
77
106
|
* (gemini, openai, anthropic, deepl, …) resolve their own env keys inside
|
|
78
107
|
* their method classes — a missing OpenRouter key must NOT block them.
|
|
79
108
|
* @param {boolean} options.dryRun - Whether to write files
|
|
109
|
+
* @param {boolean} [options.freshOnModelChange] - --fresh-on-model-change: serve
|
|
110
|
+
* only exact-model TM hits (default reuses other models' translations)
|
|
80
111
|
* @param {boolean} [options.noTM] - Bypass the Translation Memory (--no-tm):
|
|
81
112
|
* every segment goes to the API and nothing is cached.
|
|
113
|
+
* @param {object|null} [options.fallbackBudget] - lib/fallback.js
|
|
114
|
+
* createFallbackBudget(): the --max-cost guard for pairs' fallback methods
|
|
115
|
+
* (null = no cap).
|
|
82
116
|
*/
|
|
83
117
|
async function runContentSync(options) {
|
|
84
118
|
const {
|
|
@@ -89,12 +123,22 @@ async function runContentSync(options) {
|
|
|
89
123
|
apiKey,
|
|
90
124
|
dryRun = false,
|
|
91
125
|
noTM = false,
|
|
126
|
+
freshOnModelChange = false,
|
|
127
|
+
// --force-content: re-process up-to-date files (TM serves unchanged
|
|
128
|
+
// text, so this rebuilds from cache rather than re-billing).
|
|
129
|
+
forceContent = false,
|
|
130
|
+
// --files / --retranslate (lib/file-scope.js); null = every file.
|
|
131
|
+
fileScope = null,
|
|
92
132
|
cwd = process.cwd(),
|
|
93
133
|
concurrency = 48,
|
|
94
134
|
// The honest-fallback marker (config.js default). Written in front of a
|
|
95
135
|
// block's SOURCE text when the model drops its segment twice — visible,
|
|
96
136
|
// never silent, never cached (see translateBlockBatchResilient).
|
|
97
137
|
fallbackPrefix = '[EN] ',
|
|
138
|
+
fallbackBudget = null,
|
|
139
|
+
// (code) => the locale's different-inputs-same-output index, shared with
|
|
140
|
+
// the key-value sync of the same run (lib/validate.js SharedOutputIndex).
|
|
141
|
+
sharedOutputsFor = null,
|
|
98
142
|
} = options;
|
|
99
143
|
|
|
100
144
|
if (!fs.existsSync(contentDir)) {
|
|
@@ -110,6 +154,11 @@ async function runContentSync(options) {
|
|
|
110
154
|
|
|
111
155
|
const fieldsList = translatableFields || DEFAULT_TRANSLATABLE_FIELDS;
|
|
112
156
|
|
|
157
|
+
// --redo files:<glob> (= --files + --force-content) and --retranslate name
|
|
158
|
+
// files: the one way a run replaces edits made by hand to a translation.
|
|
159
|
+
// A bare --force-content / --redo content keeps them.
|
|
160
|
+
const namedByFiles = Boolean(fileScope && fileScope.limitsFiles);
|
|
161
|
+
|
|
113
162
|
// Warn at most once per source file about translatable-looking front matter
|
|
114
163
|
// we can't reach (arrays / nested blocks). Never silently drop them. The
|
|
115
164
|
// check + add are synchronous (no await between), so this is race-free.
|
|
@@ -129,10 +178,18 @@ async function runContentSync(options) {
|
|
|
129
178
|
//
|
|
130
179
|
// Safe under pMap concurrency: storeTM is a synchronous property assignment
|
|
131
180
|
// and Node is single-threaded, so writes can't interleave between awaits.
|
|
132
|
-
|
|
181
|
+
// --fresh/--no-tm: serve nothing from the cache, but cache what is paid for
|
|
182
|
+
// (lib/tm.js setTMReads) — same as the key-value path in sync.js.
|
|
183
|
+
const tm = loadTM(cwd);
|
|
184
|
+
if (noTM) setTMReads(tm, false);
|
|
185
|
+
if (freshOnModelChange) setModelCarryover(tm, false); // see lib/tm.js "Model carry-over"
|
|
186
|
+
// Blocks a pair cached before its coaching was keyed, with today's
|
|
187
|
+
// coaching, stay served (lib/tm.js adoptLegacyCoachingKeys — sync records
|
|
188
|
+
// it first; a dry run, which saves nothing, records it here too).
|
|
189
|
+
adoptLegacyCoachingKeys(tm, pairs.values());
|
|
133
190
|
const tmInitialSize = tmSize(tm);
|
|
134
191
|
if (noTM) {
|
|
135
|
-
output.info('Content sync: Translation Memory
|
|
192
|
+
output.info('Content sync: Translation Memory not read this run — content is translated fresh, and cached');
|
|
136
193
|
} else if (tmInitialSize > 0) {
|
|
137
194
|
output.info(`Content sync: Translation Memory loaded (${tmInitialSize} cached entries)`);
|
|
138
195
|
}
|
|
@@ -171,6 +228,25 @@ async function runContentSync(options) {
|
|
|
171
228
|
assertSegmentationMode(pairConfig.contentSegmentation, `pair "${pairKey}"`);
|
|
172
229
|
}
|
|
173
230
|
|
|
231
|
+
// Per-pair tally of what each pair's fallback method did (lib/fallback.js)
|
|
232
|
+
// — one [FALLBACK] line per pair at the end, and the --json summary.
|
|
233
|
+
const fallbackTallies = new Map();
|
|
234
|
+
for (const [pairKey, pairConfig] of pairEntries) {
|
|
235
|
+
if (pairConfig.fallback) fallbackTallies.set(pairKey, newFallbackReport(pairConfig.fallback));
|
|
236
|
+
}
|
|
237
|
+
const tallyFallback = (pairKey, report, page = null) => {
|
|
238
|
+
const tally = fallbackTallies.get(pairKey);
|
|
239
|
+
if (!tally) return;
|
|
240
|
+
addToTally(tally, report);
|
|
241
|
+
if (page && report && (report.accepted > 0 || report.cached > 0)) notePage(pairKey, page);
|
|
242
|
+
};
|
|
243
|
+
// A page some of whose text the fallback produced (its answer or its
|
|
244
|
+
// cache): named in the [FALLBACK] line.
|
|
245
|
+
const notePage = (pairKey, page) => {
|
|
246
|
+
const tally = fallbackTallies.get(pairKey);
|
|
247
|
+
if (tally && !tally.produced.includes(page)) tally.produced.push(page);
|
|
248
|
+
};
|
|
249
|
+
|
|
174
250
|
output.info(`Content sync: ${sourceFiles.length} source file(s) × ${pairEntries.length} language(s), concurrency: ${concurrency}`);
|
|
175
251
|
if (dryRun) output.info('Dry-run mode — no content files will be written.');
|
|
176
252
|
|
|
@@ -179,54 +255,114 @@ async function runContentSync(options) {
|
|
|
179
255
|
let skipped = 0;
|
|
180
256
|
|
|
181
257
|
const syncStartTime = Date.now();
|
|
258
|
+
const failedItems = []; // { file, locale, error } — listed at the end
|
|
259
|
+
// Files left as is because edits made by hand could not be merged with a
|
|
260
|
+
// source change — { file, target, locale, reason }; listed at the end.
|
|
261
|
+
const heldItems = [];
|
|
262
|
+
let keptEditFiles = 0;
|
|
263
|
+
let fatalError = null;
|
|
264
|
+
// Blocks and front-matter fields the quality gate refused, per translation
|
|
265
|
+
// (lib/content-refusals.js): what each run held back and what it learned.
|
|
266
|
+
// Applied to the lock at the end, whatever each page's outcome.
|
|
267
|
+
const refusalStates = new Map(); // manifestKey → { prior, refused, filled, held, … }
|
|
268
|
+
let heldBackSeen = 0; // blocks/fields held back (a dry run: would be)
|
|
182
269
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
const remainingMs = msPerFile * (totalFiles - fileIdx);
|
|
195
|
-
const remainingMin = Math.ceil(remainingMs / 60000);
|
|
196
|
-
etaStr = remainingMin > 1 ? ` (~${remainingMin} min remaining)` : '';
|
|
270
|
+
// A written-record that does not parse must not silently turn into "no
|
|
271
|
+
// edits": say so, then fall back to the pre-record behaviour for that file.
|
|
272
|
+
const readRecord = (recordKey, label) => {
|
|
273
|
+
try {
|
|
274
|
+
return parseWrittenRecord(contentManifest[recordKey]);
|
|
275
|
+
} catch (err) {
|
|
276
|
+
output.warn(
|
|
277
|
+
`${CONTENT_LOCK_FILENAME}: the record of ${label} is unreadable (${err.message}) — ` +
|
|
278
|
+
'edits made by hand to that translation cannot be recognised this run. It is rewritten the next time sync writes the file.'
|
|
279
|
+
);
|
|
280
|
+
return null;
|
|
197
281
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
// Read source file once — shared across all locale translations
|
|
201
|
-
const raw = fs.readFileSync(sourcePath, 'utf-8');
|
|
202
|
-
const currentSourceHash = hashFileContent(sourcePath);
|
|
282
|
+
};
|
|
203
283
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
const
|
|
207
|
-
const
|
|
284
|
+
try {
|
|
285
|
+
for (let fileIdx = 0; fileIdx < sourceFiles.length; fileIdx++) {
|
|
286
|
+
const sourcePath = sourceFiles[fileIdx];
|
|
287
|
+
const relPath = path.relative(contentDir, sourcePath);
|
|
288
|
+
if (fileScope && !fileScope.includes(relPath)) continue;
|
|
289
|
+
const retranslate = fileScope ? fileScope.retranslates(relPath) : false;
|
|
290
|
+
const fileNum = fileIdx + 1;
|
|
291
|
+
const totalFiles = sourceFiles.length;
|
|
208
292
|
|
|
209
|
-
|
|
293
|
+
// ETA calculation
|
|
294
|
+
let etaStr = '';
|
|
295
|
+
if (fileIdx > 0) {
|
|
296
|
+
const elapsedMs = Date.now() - syncStartTime;
|
|
297
|
+
const msPerFile = elapsedMs / fileIdx;
|
|
298
|
+
const remainingMs = msPerFile * (totalFiles - fileIdx);
|
|
299
|
+
const remainingMin = Math.ceil(remainingMs / 60000);
|
|
300
|
+
etaStr = remainingMin > 1 ? ` (~${remainingMin} min remaining)` : '';
|
|
301
|
+
}
|
|
302
|
+
output.info(`[${fileNum}/${totalFiles}] ${relPath}${etaStr}`);
|
|
210
303
|
|
|
211
|
-
//
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
304
|
+
// Read and parse the source once — shared across all locale translations
|
|
305
|
+
// (parsing is stateless; nothing below mutates the parsed parts).
|
|
306
|
+
const raw = fs.readFileSync(sourcePath, 'utf-8');
|
|
307
|
+
const currentSourceHash = hashFileContent(sourcePath);
|
|
308
|
+
const { frontMatter, rawFrontMatter, body, hasFrontMatter, frontMatterFormat } = parseContentFile(raw);
|
|
309
|
+
// The translatable front-matter values — the per-field TM unit, and
|
|
310
|
+
// what the written-record fingerprints.
|
|
311
|
+
const sourceFields = {};
|
|
312
|
+
if (hasFrontMatter) {
|
|
313
|
+
for (const field of fieldsList) {
|
|
314
|
+
if (frontMatter[field] && typeof frontMatter[field] === 'string') sourceFields[field] = frontMatter[field];
|
|
315
|
+
}
|
|
215
316
|
}
|
|
216
317
|
|
|
217
|
-
//
|
|
218
|
-
const
|
|
318
|
+
// Parallelize across locales for this file
|
|
319
|
+
const translateOne = async ([pairKey, pairConfig]) => {
|
|
320
|
+
const code = pairConfig.target;
|
|
321
|
+
const result = { translated: false, fallback: false, skipped: false, retranslated: false, held: false, keptEdits: false };
|
|
219
322
|
|
|
220
|
-
|
|
323
|
+
const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
|
|
324
|
+
|
|
325
|
+
// Security: verify target path stays within content directory
|
|
326
|
+
if (!isPathContained(targetPath, contentDir)) {
|
|
327
|
+
output.error(`${code} — refusing to write outside content directory`);
|
|
328
|
+
return result;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// Change detection for existing files
|
|
332
|
+
const manifestKey = `${relPath}:${code}`;
|
|
333
|
+
// What sync last left on disk for this translation — how edits a
|
|
334
|
+
// person made to it are told apart from ours (lib/content-review.js).
|
|
335
|
+
const recordKey = writtenRecordKey(manifestKey);
|
|
336
|
+
const targetRel = path.relative(contentDir, targetPath);
|
|
337
|
+
const segMode = pairConfig.contentSegmentation || 'block';
|
|
338
|
+
const targetExists = fs.existsSync(targetPath);
|
|
221
339
|
const storedHash = contentManifest[manifestKey];
|
|
222
|
-
|
|
340
|
+
const sourceCurrent = Boolean(storedHash) && storedHash === currentSourceHash;
|
|
341
|
+
let edits = null; // a person's paragraphs and fields to keep (state 'edited')
|
|
342
|
+
|
|
343
|
+
if (!retranslate && targetExists && sourceCurrent && !forceContent) {
|
|
344
|
+
// Up to date — not touched. A translation no record exists for
|
|
345
|
+
// (written by an older version) is recorded now, so edits made to
|
|
346
|
+
// it from here on are recognised when its source changes.
|
|
347
|
+
if (contentManifest[recordKey] === undefined) {
|
|
348
|
+
const tmKeys = [tmMethodKey(pairConfig), pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null].filter(Boolean);
|
|
349
|
+
updatedManifest[recordKey] = buildBootstrapRecord({
|
|
350
|
+
sourceBody: body,
|
|
351
|
+
sourceFields,
|
|
352
|
+
written: fs.readFileSync(targetPath, 'utf-8'),
|
|
353
|
+
// Exact cached machine translations of a source text (read
|
|
354
|
+
// directly: a comparison, not a served hit).
|
|
355
|
+
machineFor: (text) => tmKeys
|
|
356
|
+
.map(k => tm[cacheKey(text, code, k)])
|
|
357
|
+
.filter(e => e && typeof e.t === 'string')
|
|
358
|
+
.map(e => e.t),
|
|
359
|
+
});
|
|
360
|
+
}
|
|
223
361
|
result.skipped = true;
|
|
224
362
|
return result;
|
|
225
363
|
}
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
result.retranslated = true;
|
|
229
|
-
} else if (!storedHash) {
|
|
364
|
+
|
|
365
|
+
if (!retranslate && targetExists && !storedHash) {
|
|
230
366
|
// No stored hash — check if this file was generated by a prior
|
|
231
367
|
// champollion run (contains [EN] fallback markers) or is a genuine
|
|
232
368
|
// hand-translated file that should be preserved.
|
|
@@ -238,340 +374,807 @@ async function runContentSync(options) {
|
|
|
238
374
|
const isLegacyFallback = existingContent.includes('[EN] ');
|
|
239
375
|
if (!isLegacyFallback) {
|
|
240
376
|
updatedManifest[manifestKey] = currentSourceHash;
|
|
377
|
+
// Recorded as a person's file: each paragraph is kept when the
|
|
378
|
+
// source later changes elsewhere (before, the first source edit
|
|
379
|
+
// replaced the whole hand translation).
|
|
380
|
+
updatedManifest[recordKey] = buildAdoptedRecord({ sourceBody: body, sourceFields, written: existingContent });
|
|
241
381
|
result.skipped = true;
|
|
382
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'kept-hand-translated' });
|
|
242
383
|
return result;
|
|
243
384
|
}
|
|
244
|
-
output.info(`${code} — replacing [EN] fallback`);
|
|
245
385
|
}
|
|
246
|
-
}
|
|
247
386
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
387
|
+
if (targetExists) {
|
|
388
|
+
// Before rewriting a translation: what did a person change in it?
|
|
389
|
+
const record = readRecord(recordKey, targetRel);
|
|
390
|
+
if (record) {
|
|
391
|
+
const targetRaw = fs.readFileSync(targetPath, 'utf-8');
|
|
392
|
+
const verdict = assessExistingTarget({
|
|
393
|
+
targetRaw,
|
|
394
|
+
record,
|
|
395
|
+
segMode,
|
|
396
|
+
replaceEdits: retranslate || (forceContent && namedByFiles),
|
|
397
|
+
sourceCurrent,
|
|
398
|
+
});
|
|
399
|
+
if (verdict.action === 'keep') {
|
|
400
|
+
// --force-content on an up-to-date file whose edits cannot be
|
|
401
|
+
// merged: nothing is out of date, so nothing is replaced.
|
|
402
|
+
output.info(
|
|
403
|
+
`${code} — kept ${targetRel} as is: it was edited by hand (${verdict.reason}). ` +
|
|
404
|
+
`To replace it with machine translation: champollion sync --redo files:${relPath}`
|
|
405
|
+
);
|
|
406
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'kept-edits' });
|
|
407
|
+
result.skipped = true;
|
|
408
|
+
return result;
|
|
409
|
+
}
|
|
410
|
+
if (verdict.action === 'accept') {
|
|
411
|
+
// Held on an earlier run and edited since: the person brought
|
|
412
|
+
// it up to date. It is current now — and theirs.
|
|
413
|
+
updatedManifest[manifestKey] = currentSourceHash;
|
|
414
|
+
updatedManifest[recordKey] = buildAdoptedRecord({ sourceBody: body, sourceFields, written: targetRaw });
|
|
415
|
+
output.info(`${code} — ${dryRun ? 'would take' : 'took'} your updated ${targetRel} as up to date with ${relPath}`);
|
|
416
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'accepted-edits' });
|
|
417
|
+
result.skipped = true;
|
|
418
|
+
return result;
|
|
419
|
+
}
|
|
420
|
+
if (verdict.action === 'hold') {
|
|
421
|
+
// Lock NOT advanced: this warning repeats every run until the
|
|
422
|
+
// file is brought up to date by hand or named for a redo.
|
|
423
|
+
updatedManifest[recordKey] = { ...contentManifest[recordKey], held: verdict.heldHash };
|
|
424
|
+
output.warn(
|
|
425
|
+
`${targetRel} was left as is: it was edited by hand (${verdict.reason}), so the changes to ` +
|
|
426
|
+
`${relPath} cannot be merged into it paragraph by paragraph. Bring it up to date by hand ` +
|
|
427
|
+
'(the next sync then takes it as current), or replace it with machine translation: ' +
|
|
428
|
+
`champollion sync --redo files:${relPath}`
|
|
429
|
+
);
|
|
430
|
+
heldItems.push({ file: relPath, target: targetRel, locale: code, reason: verdict.reason });
|
|
431
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'held-edits' });
|
|
432
|
+
result.held = true;
|
|
433
|
+
return result;
|
|
434
|
+
}
|
|
435
|
+
if (verdict.replaced) {
|
|
436
|
+
const r = verdict.replaced;
|
|
437
|
+
const what = r.state === 'edited'
|
|
438
|
+
? describeEdits({ paragraphs: r.ownedBlocks, fields: Object.keys(r.fields).filter(f => r.fields[f].owned) })
|
|
439
|
+
: 'the file';
|
|
440
|
+
output.info(
|
|
441
|
+
`${code} — ${dryRun ? 'would replace' : 'replacing'} the edits made by hand to ${what} of ${targetRel} ` +
|
|
442
|
+
`(you named it with ${retranslate ? '--retranslate' : '--redo files: / --files'})`
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
edits = verdict.edits;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
254
448
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
449
|
+
if (retranslate) {
|
|
450
|
+
// --retranslate: the operator named this file — skip the lock and
|
|
451
|
+
// the adoption rule, and make its text miss the TM (below).
|
|
452
|
+
result.retranslated = targetExists;
|
|
453
|
+
if (!dryRun) output.info(`${code} — re-translating (--retranslate)`);
|
|
454
|
+
} else if (targetExists) {
|
|
455
|
+
if (sourceCurrent) {
|
|
456
|
+
output.info(`${code} — re-processing (--force-content; cached text is reused)`);
|
|
457
|
+
result.retranslated = true;
|
|
458
|
+
} else if (storedHash) {
|
|
459
|
+
output.info(`${code} — source updated, re-translating`);
|
|
460
|
+
result.retranslated = true;
|
|
461
|
+
} else {
|
|
462
|
+
output.info(`${code} — replacing [EN] fallback`);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
266
465
|
|
|
267
|
-
|
|
268
|
-
|
|
466
|
+
if (dryRun) {
|
|
467
|
+
const keeping = edits
|
|
468
|
+
? ` (keeping the edits made by hand to ${describeEdits({ paragraphs: edits.ownedBlocks, fields: Object.keys(edits.fields).filter(f => edits.fields[f].owned) })})`
|
|
469
|
+
: '';
|
|
470
|
+
// What the quality gate refused before is held back by the real run
|
|
471
|
+
// (lib/content-refusals.js) — said here, not "would create".
|
|
472
|
+
const preview = previewHeld({
|
|
473
|
+
tm, code, pairConfig, fields: sourceFields, body, segMode,
|
|
474
|
+
holds: contentHolds(readRefusals(contentManifest, manifestKey), pairConfig, { redo: retranslate || forceContent || noTM }),
|
|
475
|
+
});
|
|
476
|
+
if (preview.pageHeld) {
|
|
477
|
+
output.info(`Would hold back ${targetRel}: ${preview.names.join(', ')} refused before — nothing sent, not written. `
|
|
478
|
+
+ `Ask again: \`${contentRedoCommand(relPath, { pair: pairKey })}\``);
|
|
479
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'would-hold', held: preview.names.length });
|
|
480
|
+
result.heldBack = preview.names.length;
|
|
481
|
+
return result;
|
|
482
|
+
}
|
|
483
|
+
const holding = preview.names.length > 0
|
|
484
|
+
? ` (holding back ${preview.names.join(', ')}: refused before — not sent; \`${contentRedoCommand(relPath, { pair: pairKey })}\` asks again)`
|
|
485
|
+
: '';
|
|
486
|
+
output.info(`Would ${targetExists ? 'update' : 'create'}: ${targetRel}${keeping}${holding}`);
|
|
487
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'would-translate', ...(preview.names.length > 0 && { held: preview.names.length }) });
|
|
488
|
+
result.translated = true;
|
|
489
|
+
return result;
|
|
490
|
+
}
|
|
269
491
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
492
|
+
// Key gate — require only what the pair's resolved method actually
|
|
493
|
+
// needs (see readinessCache above). Fail loud with the method's own
|
|
494
|
+
// reason instead of blaming OPENROUTER_API_KEY unconditionally.
|
|
495
|
+
const readiness = await checkPairReadiness(pairKey, pairConfig);
|
|
496
|
+
if (!readiness.ready) {
|
|
497
|
+
const err = new Error(
|
|
498
|
+
`Content sync for ${code}: method "${pairConfig.method || 'llm'}" cannot run — no API key or unmet prerequisite.\n` +
|
|
499
|
+
` ${readiness.reason}\n` +
|
|
500
|
+
missingKeyAdvice({ reasons: [readiness.reason], setupHelp: [' Set the key it names in .env.local (or your environment) to translate content.'] }).join('\n')
|
|
279
501
|
);
|
|
502
|
+
err.fatal = true; // every file would fail the same way — stop now
|
|
503
|
+
throw err;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
// The source was parsed once per file above (frontMatter, body, …).
|
|
507
|
+
if (retranslate) {
|
|
508
|
+
// Fresh translation for every field, the body and each block of
|
|
509
|
+
// this file — new results still replace the cached ones.
|
|
510
|
+
const fields = hasFrontMatter
|
|
511
|
+
? fieldsList.map(f => frontMatter[f]).filter(v => typeof v === 'string')
|
|
512
|
+
: [];
|
|
513
|
+
bypassTMFor(tm, code, [...fields, body, ...translatableBlockSources(body)]);
|
|
280
514
|
}
|
|
281
|
-
}
|
|
282
515
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
516
|
+
// Refused before (lib/content-refusals.js): blocks and fields the gate
|
|
517
|
+
// refused this pair's method for are not sent to it again — unless this
|
|
518
|
+
// page is named for a redo (--redo files: / --redo content /
|
|
519
|
+
// --retranslate / --fresh). Recorded below whatever the page's outcome.
|
|
520
|
+
const refusal = {
|
|
521
|
+
file: relPath, locale: code, pageHeld: false,
|
|
522
|
+
prior: readRefusals(contentManifest, manifestKey),
|
|
523
|
+
refused: [], filled: new Set(), held: [],
|
|
524
|
+
fields: sourceFields,
|
|
525
|
+
blockSources: segMode === 'block' && body.trim() ? translatableBlockSources(body) : [],
|
|
526
|
+
pageSource: segMode === 'page' && body.trim() ? body : null,
|
|
527
|
+
};
|
|
528
|
+
refusalStates.set(manifestKey, refusal);
|
|
529
|
+
const holds = contentHolds(refusal.prior, pairConfig, { redo: retranslate || forceContent || noTM });
|
|
530
|
+
// A front-matter field held back fails its page, as a refused one does:
|
|
531
|
+
// nothing of the page is sent, nothing written, the lock not advanced.
|
|
532
|
+
const holdPage = (names) => {
|
|
533
|
+
refusal.held.push(...names);
|
|
534
|
+
refusal.pageHeld = true;
|
|
535
|
+
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, pageHeld: true, handWritten: true, fallbackPrefix }));
|
|
536
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'held-refused', held: names.length });
|
|
537
|
+
result.heldBack = names.length;
|
|
538
|
+
return result;
|
|
539
|
+
};
|
|
540
|
+
// A page translated whole that the gate refused before (and that the
|
|
541
|
+
// cache does not hold): held before anything of it is sent — its
|
|
542
|
+
// front matter included (a pure look; the body read below serves).
|
|
543
|
+
if (refusal.pageSource && holds.page(body) === 'held'
|
|
544
|
+
&& ![tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])]
|
|
545
|
+
.some(k => peekTM(tm, body, code, k) !== null)) {
|
|
546
|
+
return holdPage([PAGE_NAME]);
|
|
547
|
+
}
|
|
286
548
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
549
|
+
// Never silently drop translatable-looking nested/array front matter
|
|
550
|
+
// (e.g. `related:` lists) — surface it once per source file.
|
|
551
|
+
if (hasFrontMatter && !warnedFrontMatter.has(sourcePath)) {
|
|
552
|
+
warnedFrontMatter.add(sourcePath);
|
|
553
|
+
const skipped = findUntranslatableNestedFields(rawFrontMatter);
|
|
554
|
+
if (skipped.length > 0) {
|
|
555
|
+
output.warn(
|
|
556
|
+
`${relPath}: front matter field(s) [${skipped.join(', ')}] are arrays/nested — ` +
|
|
557
|
+
`left untranslated. Flatten them to top-level strings to translate, or translate by hand.`
|
|
558
|
+
);
|
|
296
559
|
}
|
|
297
560
|
}
|
|
298
561
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
562
|
+
// This page's cached content through the repeat check, in one batch
|
|
563
|
+
// and before any of it is served (lib/fallback.js): a text cached for
|
|
564
|
+
// several different source strings is evicted, so the lookups below
|
|
565
|
+
// miss it and it is translated again, checked the same way. What a
|
|
566
|
+
// person's edits keep is not served from the cache, so not checked.
|
|
567
|
+
{
|
|
568
|
+
const keptFields = hasFrontMatter && edits && edits.ownedFields > 0
|
|
569
|
+
? matchEditedFields(edits.fields, sourceFields).keep : {};
|
|
570
|
+
const keptParagraphs = edits && edits.blocks && edits.ownedBlocks > 0
|
|
571
|
+
? matchEditedBlocks(edits.blocks, body.trim() ? translatableBlockSources(body) : []).keep : new Map();
|
|
572
|
+
const repeated = refuseRepeatedCachedPage({
|
|
573
|
+
label: `content:${relPath}`,
|
|
574
|
+
fields: hasFrontMatter ? Object.fromEntries(Object.entries(sourceFields).filter(([f]) => !(f in keptFields))) : {},
|
|
575
|
+
body,
|
|
576
|
+
skipBlocks: new Set(keptParagraphs.keys()),
|
|
577
|
+
wholeBody: keptParagraphs.size === 0,
|
|
578
|
+
tm, code, pairConfig,
|
|
579
|
+
sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
|
|
580
|
+
});
|
|
581
|
+
if (repeated.length > 0) {
|
|
582
|
+
output.warn(
|
|
583
|
+
`${relPath} → ${code}: cached ${repeated.join(', ')} held the same text as other, different source strings ` +
|
|
584
|
+
'(a memorized sentence, not a translation) — removed from the cache and translated again.'
|
|
585
|
+
);
|
|
312
586
|
}
|
|
313
587
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
` source: ${JSON.stringify(sourceValue)}\n` +
|
|
341
|
-
` got: ${JSON.stringify(value)}\n` +
|
|
342
|
-
' Nothing was written or cached. If this is a low-coverage target language,\n' +
|
|
343
|
-
' the model has no vocabulary for this string — fix the prompt or the pair.'
|
|
344
|
-
);
|
|
345
|
-
}
|
|
588
|
+
|
|
589
|
+
// TM entries are keyed on the full method key (method|model|register|
|
|
590
|
+
// coaching). A method/register/coaching change re-translates; a MODEL
|
|
591
|
+
// change reuses the previous model's entries unless
|
|
592
|
+
// --fresh-on-model-change (see "Model carry-over" in lib/tm.js).
|
|
593
|
+
const tmKey = tmMethodKey(pairConfig);
|
|
594
|
+
|
|
595
|
+
// Translate front matter fields — TM first, API for the misses.
|
|
596
|
+
// Each field is cached on its own source text, exactly like a
|
|
597
|
+
// key-value sync key: a title edit re-pays only the title.
|
|
598
|
+
const translatedFields = {};
|
|
599
|
+
// Edits made by hand that this write keeps, and the ones whose
|
|
600
|
+
// source text changed under them (re-translated; wording printed).
|
|
601
|
+
const keptFieldNames = new Set();
|
|
602
|
+
const keptBlocks = new Set(); // source block positions
|
|
603
|
+
const supersededNotes = [];
|
|
604
|
+
if (hasFrontMatter) {
|
|
605
|
+
const fieldsToTranslate = { ...sourceFields };
|
|
606
|
+
if (edits && edits.ownedFields > 0) {
|
|
607
|
+
const { keep, superseded } = matchEditedFields(edits.fields, sourceFields);
|
|
608
|
+
for (const [field, text] of Object.entries(keep)) {
|
|
609
|
+
// A person's value for an unchanged source value: written as
|
|
610
|
+
// is, never sent to the API, never cached as machine output.
|
|
611
|
+
translatedFields[field] = text;
|
|
612
|
+
keptFieldNames.add(field);
|
|
613
|
+
delete fieldsToTranslate[field];
|
|
346
614
|
}
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
storeTM(tm, fieldsToTranslate[field], code, tmKey, value);
|
|
353
|
-
}
|
|
615
|
+
for (const s of superseded) {
|
|
616
|
+
supersededNotes.push(
|
|
617
|
+
`${targetRel}: front matter "${s.field}" had been edited by hand, but its source value has changed, ` +
|
|
618
|
+
`so it was re-translated. The edited wording, to re-apply if it still fits: ${JSON.stringify(s.text)}`
|
|
619
|
+
);
|
|
354
620
|
}
|
|
355
|
-
output.raw(' [OK]');
|
|
356
|
-
} else {
|
|
357
|
-
// Front matter translation failed — loud error, skip this file
|
|
358
|
-
output.raw(' [ERR]');
|
|
359
|
-
throw new Error(
|
|
360
|
-
`Content sync for ${code}: front matter translation returned no results.\n` +
|
|
361
|
-
' Check your API key and method configuration.'
|
|
362
|
-
);
|
|
363
621
|
}
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
622
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
tmSegmentHits +=
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
const
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
//
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
if (
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
623
|
+
const { hits: fmHits, misses: fmMisses } = partitionByTM(
|
|
624
|
+
tm, fieldsToTranslate, Object.keys(fieldsToTranslate), code, tmKey
|
|
625
|
+
);
|
|
626
|
+
// Validate cached hits BEFORE serving. A cache is a time machine: an
|
|
627
|
+
// entry stored before the content-preservation gate existed re-serves
|
|
628
|
+
// exactly what that gate now rejects — hollowed titles sat here and
|
|
629
|
+
// were re-served forever, because TM hits skipped every gate. A hit
|
|
630
|
+
// that fails today's gate is evicted and re-billed as a miss.
|
|
631
|
+
for (const [field, cachedValue] of Object.entries(fmHits)) {
|
|
632
|
+
if (contentGateFault(fieldsToTranslate[field], cachedValue, pairConfig)) {
|
|
633
|
+
evictTM(tm, fieldsToTranslate[field], code, tmKey);
|
|
634
|
+
delete fmHits[field];
|
|
635
|
+
fmMisses.push(field);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
Object.assign(translatedFields, fmHits);
|
|
639
|
+
tmSegmentHits += Object.keys(fmHits).length;
|
|
640
|
+
// Then the fallback's cache: fields it translated on an earlier run
|
|
641
|
+
// are reused, not re-sent to the primary (lib/fallback.js ladder).
|
|
642
|
+
const fbCache = serveFieldsFromFallbackCache(tm, fieldsToTranslate, fmMisses, code, pairConfig);
|
|
643
|
+
Object.assign(translatedFields, fbCache.hits);
|
|
644
|
+
tmSegmentHits += Object.keys(fbCache.hits).length;
|
|
645
|
+
if (Object.keys(fbCache.hits).length > 0) notePage(pairKey, relPath);
|
|
646
|
+
const fieldsToSend = fbCache.misses;
|
|
647
|
+
// Filled without a call: the cache, a person's value.
|
|
648
|
+
for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
|
|
649
|
+
// Refused before: held back (the page with it), or the fallback's only.
|
|
650
|
+
const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
|
|
651
|
+
if (heldFields.length > 0) return holdPage(heldFields.map(f => `front matter "${f}"`));
|
|
652
|
+
const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
|
|
653
|
+
|
|
654
|
+
if (fieldsToSend.length > 0) {
|
|
655
|
+
output.progress(` [SYNC] ${code} front matter (${pairConfig.method})...`);
|
|
656
|
+
// The pair's method, then its fallback (if any) for every field
|
|
657
|
+
// it returned nothing for or hollowed. Content-preservation gate
|
|
658
|
+
// on every value: front matter (title, description, summary)
|
|
659
|
+
// once went from the API straight to disk AND into the TM with no
|
|
660
|
+
// validation — which is how a hollowed page title was written
|
|
661
|
+
// silently and then cached. Nothing is cached until no field is
|
|
662
|
+
// left failing; a failing field fails the file (nothing written,
|
|
663
|
+
// manifest not advanced, retried).
|
|
664
|
+
const fm = await translateFieldsWithFallback({
|
|
665
|
+
fields: fieldsToTranslate,
|
|
666
|
+
misses: fieldsToSend,
|
|
406
667
|
pairConfig,
|
|
668
|
+
budget: fallbackBudget,
|
|
669
|
+
sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
|
|
670
|
+
label: `content:${relPath} front matter`,
|
|
671
|
+
translate: (keys, cfg) => translateBatch(
|
|
672
|
+
keys, fieldsToTranslate, cfg,
|
|
673
|
+
{ apiKey, cwd, model: cfg.model, batchSize: cfg.batchSize || 30 },
|
|
674
|
+
),
|
|
675
|
+
fallbackOnly: fallbackOnlyFields,
|
|
407
676
|
});
|
|
408
|
-
|
|
409
|
-
|
|
677
|
+
tallyFallback(pairKey, fm.report, relPath);
|
|
678
|
+
// Refused fields are remembered (held back from the method that
|
|
679
|
+
// refused them), whatever happens to the page below.
|
|
680
|
+
for (const [field, methods] of Object.entries(fm.refusedBy || {})) {
|
|
681
|
+
refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
|
|
682
|
+
}
|
|
683
|
+
if (fm.hollowed.length > 0) {
|
|
684
|
+
const h = fm.hollowed[0];
|
|
685
|
+
output.raw(' [ERR]');
|
|
686
|
+
// A memorized sentence: asking the same model again returns it
|
|
687
|
+
// again — say so, and what does help (Round 6, school persona).
|
|
688
|
+
const advice = h.sharedOutput
|
|
689
|
+
? (pairConfig.fallback
|
|
690
|
+
? ` ${pairConfig.method} can only answer this string with that sentence, and the fallback did not translate it either.\n`
|
|
691
|
+
+ ` Write "${h.field}" in ${targetRel} by hand (a hand-written field is kept), or try another fallback.`
|
|
692
|
+
: ` ${pairConfig.method} can only answer this string with that sentence — asking it again returns the same.\n`
|
|
693
|
+
+ ' Add a "fallback" method to the pair (it is asked for what the primary\'s answer is refused for),\n'
|
|
694
|
+
+ ` or write "${h.field}" in ${targetRel} by hand (a hand-written field is kept).`)
|
|
695
|
+
: ' If this is a low-coverage target language,\n'
|
|
696
|
+
+ ' the model has no vocabulary for this string — fix the prompt or the pair.';
|
|
697
|
+
const err = new Error(
|
|
698
|
+
`Content sync for ${code}: front matter "${h.field}" — ${h.reason}.\n` +
|
|
699
|
+
` source: ${JSON.stringify(fieldsToTranslate[h.field])}\n` +
|
|
700
|
+
` got: ${JSON.stringify(h.value)}\n` +
|
|
701
|
+
(h.fallbackReason ? ` the fallback (${pairConfig.fallback.method}) failed it too: ${h.fallbackReason}\n` : '') +
|
|
702
|
+
' Nothing was written or cached.\n' + advice + '\n' +
|
|
703
|
+
` ${describeNewHold({ file: relPath, pairKey, pairConfig, count: fm.hollowed.length })}`
|
|
704
|
+
);
|
|
705
|
+
err.heldNext = true;
|
|
706
|
+
throw err;
|
|
707
|
+
}
|
|
708
|
+
if (fm.noResults) {
|
|
709
|
+
// Front matter translation failed — loud error, skip this file
|
|
410
710
|
output.raw(' [ERR]');
|
|
411
711
|
throw new Error(
|
|
412
|
-
`Content sync
|
|
712
|
+
`Content sync for ${code}: front matter translation returned no results`
|
|
713
|
+
+ `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
|
|
413
714
|
' Check your API key and method configuration.'
|
|
414
715
|
);
|
|
415
716
|
}
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
//
|
|
420
|
-
//
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
// restored text is stable and self-contained. Cached values are
|
|
427
|
-
// likewise restored — they must never carry another file's
|
|
428
|
-
// placeholder ids.
|
|
429
|
-
const rendered = segments.map(seg => ({
|
|
430
|
-
seg,
|
|
431
|
-
source: restoreBlocks(seg.text, blocks),
|
|
432
|
-
out: null,
|
|
433
|
-
}));
|
|
434
|
-
|
|
435
|
-
const missed = [];
|
|
436
|
-
for (const r of rendered) {
|
|
437
|
-
if (r.seg.type !== 'translatable') {
|
|
438
|
-
// Separators + passthrough blocks: copied verbatim, never billed.
|
|
439
|
-
r.out = r.source;
|
|
440
|
-
continue;
|
|
441
|
-
}
|
|
442
|
-
const cached = lookupTMValidated(tm, r.source, code, tmKey,
|
|
443
|
-
(src, c) => !checkContentPreservation(src, c));
|
|
444
|
-
if (cached !== null) {
|
|
445
|
-
r.out = cached;
|
|
446
|
-
tmSegmentHits += 1;
|
|
447
|
-
} else {
|
|
448
|
-
missed.push(r);
|
|
449
|
-
}
|
|
717
|
+
// Cache only gate-passing values, each under the TM key of the
|
|
718
|
+
// method that produced it.
|
|
719
|
+
for (const st of fm.stores) storeTM(tm, st.text, code, st.tmKey, st.value);
|
|
720
|
+
// A field only the fallback may be asked for that it did not
|
|
721
|
+
// translate (skipped by --max-cost, or no answer): still refused by
|
|
722
|
+
// the pair's method, so the page is held, as with no fallback.
|
|
723
|
+
const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
|
|
724
|
+
if (unfilled.length > 0) {
|
|
725
|
+
output.raw(' [HELD]');
|
|
726
|
+
return holdPage(unfilled.map(f => `front matter "${f}"`));
|
|
450
727
|
}
|
|
728
|
+
Object.assign(translatedFields, fm.translated);
|
|
729
|
+
for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
|
|
730
|
+
output.raw(' [OK]');
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// Translate body — whole-body TM first (a reverted or duplicate body
|
|
735
|
+
// is free), then block-level TM + ONE batched API call for the missed
|
|
736
|
+
// blocks ('block' mode, the default), or the whole-page prompt
|
|
737
|
+
// ('page' mode). A structural failure fails the file whole; a segment
|
|
738
|
+
// the model drops TWICE degrades to the honest '[EN] ' fallback for
|
|
739
|
+
// just that block (never cached, lock not advanced — re-fires next
|
|
740
|
+
// sync as a TM-cheap retry). Mirrors docusaurus-sync.js.
|
|
741
|
+
let translatedBody = body;
|
|
742
|
+
let bodyUsedFallback = false;
|
|
743
|
+
// Paragraphs a person edited whose source paragraph is unchanged:
|
|
744
|
+
// kept word for word (block mode only — assessExistingTarget never
|
|
745
|
+
// lets a 'page'-mode file with edited paragraphs get this far). The
|
|
746
|
+
// rest are reported, even when the source body is now empty.
|
|
747
|
+
let keepPlan = null;
|
|
748
|
+
if (edits && edits.blocks && edits.ownedBlocks > 0) {
|
|
749
|
+
keepPlan = matchEditedBlocks(edits.blocks, body.trim() ? translatableBlockSources(body) : []);
|
|
750
|
+
for (const s of keepPlan.superseded) {
|
|
751
|
+
supersededNotes.push(
|
|
752
|
+
`${targetRel}: paragraph ${s.paragraph} had been edited by hand, but the source paragraph it ` +
|
|
753
|
+
'translates has changed, so it was re-translated. The edited wording, to re-apply if it still fits:\n' +
|
|
754
|
+
s.text.split('\n').map(line => ` ${line}`).join('\n')
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
if (body.trim()) {
|
|
759
|
+
// Read-time validation: a whole-body entry cached by a gateless
|
|
760
|
+
// pipeline must not be re-served once the gate exists (evicts on fail).
|
|
761
|
+
// The pair's cache, then its fallback's (lib/fallback.js ladder).
|
|
762
|
+
// Not when a person's paragraphs are kept: a whole-body hit would
|
|
763
|
+
// put the machine wording back over them.
|
|
764
|
+
const cachedBody = keepPlan && keepPlan.keep.size > 0
|
|
765
|
+
? null
|
|
766
|
+
: lookupContentTM(tm, body, code, pairConfig);
|
|
767
|
+
if (cachedBody !== null) {
|
|
768
|
+
translatedBody = cachedBody.text;
|
|
769
|
+
tmSegmentHits += 1;
|
|
770
|
+
for (const src of refusal.blockSources) refusal.filled.add(blockUnit(src));
|
|
771
|
+
if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
|
|
772
|
+
} else {
|
|
773
|
+
const { protectedBody, blocks } = protectBlocks(body);
|
|
774
|
+
const promptOptions = {
|
|
775
|
+
sourceLanguageName: DEFAULT_REGISTERS[sourceLocale]?.name || sourceLocale,
|
|
776
|
+
promptContext: pairConfig.promptContext || null,
|
|
777
|
+
protectedTerms: pairConfig.protectedTerms || [],
|
|
778
|
+
};
|
|
779
|
+
|
|
780
|
+
// Block stores are deferred until the reassembled body passes the
|
|
781
|
+
// orphaned-placeholder check — the TM must never hold a value that
|
|
782
|
+
// would fail the gate on re-serve. Each carries the TM key of the
|
|
783
|
+
// method that produced it (the pair's, or its fallback's).
|
|
784
|
+
const pendingBlockStores = [];
|
|
785
|
+
let apiCalled = false;
|
|
786
|
+
// The whole body is cached under ONE method's key, so only when
|
|
787
|
+
// one method produced all of it (a body mixing the primary's and
|
|
788
|
+
// the fallback's blocks is cached block by block only).
|
|
789
|
+
let wholeBodyKey = tmKey;
|
|
790
|
+
let mixedProducers = false;
|
|
791
|
+
// Page mode: whose whole page the checks below refuse (the lane
|
|
792
|
+
// remembers it — lib/content-refusals.js "page").
|
|
793
|
+
let pageRefusedBy = null;
|
|
451
794
|
|
|
452
|
-
if (
|
|
453
|
-
|
|
795
|
+
if (segMode === 'page') {
|
|
796
|
+
// Refused before by this method (and the cache does not hold
|
|
797
|
+
// it): not sent again (lib/content-refusals.js).
|
|
798
|
+
const pageHold = holds.page(body);
|
|
799
|
+
if (pageHold === 'held') return holdPage([PAGE_NAME]);
|
|
800
|
+
// Whole-page prompt — the pre-segmentation single-call behavior.
|
|
801
|
+
output.progress(` [SYNC] ${code} body (${pairConfig.method})...`);
|
|
454
802
|
apiCalled = true;
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
buildPrompt: (texts) => buildBlockBatchPrompt(
|
|
470
|
-
texts, pairConfig, { ...promptOptions, pageTitle }),
|
|
471
|
-
callModel: (prompt) => translateRawContent(prompt, {
|
|
472
|
-
apiKey,
|
|
473
|
-
cwd,
|
|
474
|
-
pairConfig,
|
|
475
|
-
}),
|
|
476
|
-
fallbackPrefix,
|
|
803
|
+
const runPage = (cfg) => translateRawContent(buildContentPrompt(protectedBody, cfg, promptOptions), {
|
|
804
|
+
apiKey,
|
|
805
|
+
cwd,
|
|
806
|
+
pairConfig: cfg,
|
|
807
|
+
});
|
|
808
|
+
let bodyResult;
|
|
809
|
+
if (pairConfig.fallback) {
|
|
810
|
+
// The fallback translates the page when the primary returns
|
|
811
|
+
// nothing, damages a placeholder or hollows it. When both
|
|
812
|
+
// fail, the primary's page goes to the checks below, which
|
|
813
|
+
// fail the file exactly as without a fallback.
|
|
814
|
+
// Refused before by the pair's method: only the fallback is asked.
|
|
815
|
+
const page = await translatePageWithFallback({
|
|
816
|
+
body, blocks, pairConfig, runPage, budget: fallbackBudget, fallbackOnly: pageHold === 'fallback-only',
|
|
477
817
|
});
|
|
478
|
-
|
|
818
|
+
tallyFallback(pairKey, page.report, relPath);
|
|
819
|
+
if (page.refusedBy.length > 0) {
|
|
820
|
+
refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: page.refusedBy });
|
|
821
|
+
}
|
|
822
|
+
if (pageHold === 'fallback-only' && page.body === null) {
|
|
823
|
+
// The fallback did not translate it: the pair's method
|
|
824
|
+
// refused it before, so it is not asked — refused now by
|
|
825
|
+
// the fallback too, or held (no answer, or skipped by
|
|
826
|
+
// --max-cost).
|
|
827
|
+
if (page.refusedBy.length === 0) return holdPage([PAGE_NAME]);
|
|
828
|
+
output.raw(' [ERR]');
|
|
829
|
+
const err = new Error(
|
|
830
|
+
`Content sync body for ${code}: the fallback (${pairConfig.fallback.method}) failed the whole-page checks too.\n` +
|
|
831
|
+
` Nothing was written or cached. ${describeNewHold({ file: relPath, pairKey, pairConfig, count: 1 })}`
|
|
832
|
+
);
|
|
833
|
+
err.heldNext = true;
|
|
834
|
+
throw err;
|
|
835
|
+
}
|
|
836
|
+
bodyResult = page.body ?? page.primaryBody;
|
|
837
|
+
if (page.body !== null) wholeBodyKey = page.tmKey;
|
|
838
|
+
else if (page.refusedBy.length > 0) pageRefusedBy = page.refusedBy;
|
|
839
|
+
} else {
|
|
840
|
+
const raw = await runPage(pairConfig);
|
|
841
|
+
bodyResult = raw ? restoreBlocks(raw, blocks) : null;
|
|
842
|
+
if (bodyResult) pageRefusedBy = [tmKey];
|
|
843
|
+
}
|
|
844
|
+
if (!bodyResult) {
|
|
845
|
+
// Body translation returned null — loud error
|
|
479
846
|
output.raw(' [ERR]');
|
|
480
|
-
throw
|
|
847
|
+
throw new Error(
|
|
848
|
+
`Content sync body for ${code}: translation returned no results`
|
|
849
|
+
+ `${pairConfig.fallback ? ` (from the primary, ${pairConfig.method}, or its fallback, ${pairConfig.fallback.method})` : ''}.\n` +
|
|
850
|
+
' Check your API key and method configuration.'
|
|
851
|
+
);
|
|
852
|
+
}
|
|
853
|
+
translatedBody = bodyResult;
|
|
854
|
+
} else {
|
|
855
|
+
// Block mode: segment the PROTECTED body (placeholders are
|
|
856
|
+
// single tokens, so they can never be split), serve blocks from
|
|
857
|
+
// the TM, and batch the misses into one API call.
|
|
858
|
+
const segments = splitBlocks(protectedBody);
|
|
859
|
+
|
|
860
|
+
// TM keys use each block's RESTORED source text: placeholder
|
|
861
|
+
// numbering is positional per file, so the protected text of an
|
|
862
|
+
// identical paragraph differs across files/edits, while the
|
|
863
|
+
// restored text is stable and self-contained. Cached values are
|
|
864
|
+
// likewise restored — they must never carry another file's
|
|
865
|
+
// placeholder ids.
|
|
866
|
+
const rendered = segments.map(seg => ({
|
|
867
|
+
seg,
|
|
868
|
+
source: restoreBlocks(seg.text, blocks),
|
|
869
|
+
out: null,
|
|
870
|
+
}));
|
|
871
|
+
|
|
872
|
+
const missed = [];
|
|
873
|
+
const heldBlocks = []; // refused before: not sent, '[EN] ' kept
|
|
874
|
+
let position = 0; // index among translatable blocks (= translatableBlockSources order)
|
|
875
|
+
for (const r of rendered) {
|
|
876
|
+
if (r.seg.type !== 'translatable') {
|
|
877
|
+
// Separators + passthrough blocks: copied verbatim, never billed.
|
|
878
|
+
r.out = r.source;
|
|
879
|
+
continue;
|
|
880
|
+
}
|
|
881
|
+
const j = position++;
|
|
882
|
+
// Its name in the repeat check (content:<file>#<n>, as verify names it).
|
|
883
|
+
r.pos = j;
|
|
884
|
+
if (keepPlan && keepPlan.keep.has(j)) {
|
|
885
|
+
// A person's paragraph: written as is, not billed, not cached.
|
|
886
|
+
r.out = keepPlan.keep.get(j);
|
|
887
|
+
keptBlocks.add(j);
|
|
888
|
+
refusal.filled.add(blockUnit(r.source));
|
|
889
|
+
continue;
|
|
890
|
+
}
|
|
891
|
+
const cached = lookupContentTM(tm, r.source, code, pairConfig);
|
|
892
|
+
if (cached !== null) {
|
|
893
|
+
r.out = cached.text;
|
|
894
|
+
tmSegmentHits += 1;
|
|
895
|
+
if (cached.fromFallback) { mixedProducers = true; notePage(pairKey, relPath); }
|
|
896
|
+
refusal.filled.add(blockUnit(r.source));
|
|
897
|
+
continue;
|
|
898
|
+
}
|
|
899
|
+
const hold = holds.block(r.source);
|
|
900
|
+
if (hold === 'held') {
|
|
901
|
+
// Refused before by this method (and its fallback): the
|
|
902
|
+
// honest last resort stays, nothing is sent or billed.
|
|
903
|
+
r.out = restoreBlocks(fallbackPrefix + r.seg.text, blocks);
|
|
904
|
+
heldBlocks.push(r);
|
|
905
|
+
continue;
|
|
906
|
+
}
|
|
907
|
+
r.fallbackOnly = hold === 'fallback-only';
|
|
908
|
+
missed.push(r);
|
|
481
909
|
}
|
|
482
|
-
|
|
483
|
-
const fellBackSet = new Set(fellBack);
|
|
484
|
-
if (fellBack.length > 0) {
|
|
910
|
+
if (heldBlocks.length > 0) {
|
|
485
911
|
bodyUsedFallback = true;
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
`'${fallbackPrefix}'-prefixed source. Not cached, lock not advanced: ` +
|
|
491
|
-
`the next sync retries just those block(s).`
|
|
492
|
-
);
|
|
912
|
+
const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
|
|
913
|
+
refusal.held.push(...names);
|
|
914
|
+
result.heldBack = (result.heldBack || 0) + names.length;
|
|
915
|
+
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true, fallbackPrefix }));
|
|
493
916
|
}
|
|
494
|
-
|
|
495
|
-
|
|
917
|
+
|
|
918
|
+
if (missed.length > 0) {
|
|
919
|
+
output.progress(` [SYNC] ${code} body (${missed.length} block(s), ${pairConfig.method})...`);
|
|
920
|
+
apiCalled = true;
|
|
921
|
+
// Terminology context for the block-batch prompt: the page's
|
|
922
|
+
// title (front matter first, else the first H1 in the body).
|
|
923
|
+
const pageTitle =
|
|
924
|
+
(hasFrontMatter && typeof frontMatter.title === 'string' ? frontMatter.title : null)
|
|
925
|
+
|| (body.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? null);
|
|
926
|
+
// Self-repair ladder (translateBlockBatchResilient): full
|
|
927
|
+
// batch → one missing-segments-only retry → (with a fallback
|
|
928
|
+
// method: one batch through it for every block the primary
|
|
929
|
+
// dropped or damaged — lib/fallback.js) → honest
|
|
930
|
+
// '[EN] '-prefixed source for anything still missing. A
|
|
931
|
+
// duplicate/unknown marker (untrustworthy mapping) or an
|
|
932
|
+
// empty first response still fails the file whole when no
|
|
933
|
+
// fallback rescues it.
|
|
934
|
+
let batchOutcome;
|
|
935
|
+
try {
|
|
936
|
+
batchOutcome = await translateBlocksWithFallback({
|
|
937
|
+
missed,
|
|
938
|
+
blocks,
|
|
939
|
+
pairConfig,
|
|
940
|
+
fallbackPrefix,
|
|
941
|
+
budget: fallbackBudget,
|
|
942
|
+
sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
|
|
943
|
+
label: `content:${relPath}`,
|
|
944
|
+
runBatch: (texts, cfg) => translateBlockBatchResilient({
|
|
945
|
+
texts,
|
|
946
|
+
buildPrompt: (t) => buildBlockBatchPrompt(t, cfg, { ...promptOptions, pageTitle }),
|
|
947
|
+
callModel: (prompt) => translateRawContent(prompt, {
|
|
948
|
+
apiKey,
|
|
949
|
+
cwd,
|
|
950
|
+
pairConfig: cfg,
|
|
951
|
+
}),
|
|
952
|
+
fallbackPrefix,
|
|
953
|
+
}),
|
|
954
|
+
fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
|
|
955
|
+
});
|
|
956
|
+
} catch (batchErr) {
|
|
957
|
+
output.raw(' [ERR]');
|
|
958
|
+
throw batchErr;
|
|
959
|
+
}
|
|
960
|
+
tallyFallback(pairKey, batchOutcome.report, relPath);
|
|
961
|
+
if (batchOutcome.fromFallback > 0) mixedProducers = true;
|
|
962
|
+
const { fellBack } = batchOutcome;
|
|
963
|
+
// Refused blocks are remembered; translated ones drop a record.
|
|
964
|
+
const fellSet = new Set(fellBack);
|
|
965
|
+
const newlyHeld = [];
|
|
966
|
+
missed.forEach((r, i) => {
|
|
967
|
+
if (!fellSet.has(i)) { refusal.filled.add(blockUnit(r.source)); return; }
|
|
968
|
+
const methods = batchOutcome.refusedBy?.[i] || [];
|
|
969
|
+
if (methods.length > 0) {
|
|
970
|
+
refusal.refused.push({ unit: blockUnit(r.source), source: r.source, methods });
|
|
971
|
+
newlyHeld.push(i);
|
|
972
|
+
}
|
|
973
|
+
});
|
|
974
|
+
if ((batchOutcome.sharedOutput || []).length > 0) {
|
|
975
|
+
output.warn(
|
|
976
|
+
`Content sync body for ${code}: ${batchOutcome.sharedOutput.length} block(s) of ${relPath} came back as the same text ` +
|
|
977
|
+
'the model gave for other, different source strings (a memorized sentence, not a translation) — refused' +
|
|
978
|
+
(pairConfig.fallback
|
|
979
|
+
? `; sent to the fallback (${pairConfig.fallback.method}).`
|
|
980
|
+
: `. ${pairConfig.method} can only answer them with that sentence: add a "fallback" method to the pair, ` +
|
|
981
|
+
'or write those paragraphs by hand (a hand-written paragraph is kept).')
|
|
982
|
+
);
|
|
983
|
+
}
|
|
984
|
+
// Blocks the quality gate refused — the key-value gate's checks
|
|
985
|
+
// (length inflation, echo, truncation, script), per block.
|
|
986
|
+
const refusedByGate = batchOutcome.refused || [];
|
|
987
|
+
if (refusedByGate.length > 0) {
|
|
988
|
+
output.warn(
|
|
989
|
+
`Content sync body for ${code}: ${refusedByGate.length} block(s) of ${relPath} refused by the quality gate — `
|
|
990
|
+
+ refusedByGate.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
|
|
991
|
+
+ `${refusedByGate.length > 3 ? '; …' : ''}`
|
|
992
|
+
+ (pairConfig.fallback
|
|
993
|
+
? '.'
|
|
994
|
+
: `. Add a "fallback" method to the pair (it is asked for what ${pairConfig.method}'s answer is refused for), `
|
|
995
|
+
+ 'or write those paragraphs by hand (a hand-written paragraph is kept).')
|
|
996
|
+
);
|
|
997
|
+
}
|
|
998
|
+
if (fellBack.length > 0) {
|
|
999
|
+
bodyUsedFallback = true;
|
|
1000
|
+
output.raw(' [EN]');
|
|
1001
|
+
// Refused as a memorized sentence (said just above) is not
|
|
1002
|
+
// "missing from the response".
|
|
1003
|
+
const refusedShared = (batchOutcome.sharedOutput || []).length + refusedByGate.length;
|
|
1004
|
+
const why = batchOutcome.report?.attempted > 0
|
|
1005
|
+
? `neither the primary (${pairConfig.method}) nor its fallback (${pairConfig.fallback.method}) translated safely`
|
|
1006
|
+
: refusedShared >= fellBack.length
|
|
1007
|
+
? 'refused (above)'
|
|
1008
|
+
: refusedShared > 0
|
|
1009
|
+
? `refused (${refusedShared}, above) or missing from the model response after a retry`
|
|
1010
|
+
: 'missing from the model response after a retry';
|
|
1011
|
+
output.warn(
|
|
1012
|
+
`Content sync body for ${code}: ${fellBack.length} of ${missed.length} ` +
|
|
1013
|
+
`block(s) ${why} — written as ` +
|
|
1014
|
+
`'${fallbackPrefix}'-prefixed source. ` +
|
|
1015
|
+
describeFallenBack({
|
|
1016
|
+
file: relPath, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: batchOutcome.refusedBy,
|
|
1017
|
+
})
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
496
1020
|
// Fallen-back segments are never TM-cached — an error cached
|
|
497
1021
|
// is an error forever (they re-bill on the next sync).
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
1022
|
+
missed.forEach((r, i) => { r.out = batchOutcome.outs[i]; });
|
|
1023
|
+
pendingBlockStores.push(...batchOutcome.stores);
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
// Reassemble in order with the source's exact separators.
|
|
1027
|
+
translatedBody = rendered.map(r => r.out).join('');
|
|
502
1028
|
}
|
|
503
1029
|
|
|
504
|
-
//
|
|
505
|
-
|
|
506
|
-
|
|
1030
|
+
// A page translated whole that fails the checks below is
|
|
1031
|
+
// remembered as refused: the next plain sync does not send it to
|
|
1032
|
+
// the same method again (lib/content-refusals.js "page").
|
|
1033
|
+
const refusePage = (err) => {
|
|
1034
|
+
if (!pageRefusedBy) return err;
|
|
1035
|
+
if (!refusal.refused.some(r => r.unit === PAGE_UNIT)) {
|
|
1036
|
+
refusal.refused.push({ unit: PAGE_UNIT, source: body, methods: pageRefusedBy });
|
|
1037
|
+
}
|
|
1038
|
+
err.message += `\n ${describeNewHold({ file: relPath, pairKey, pairConfig, count: 1 })}`;
|
|
1039
|
+
err.heldNext = true;
|
|
1040
|
+
return err;
|
|
1041
|
+
};
|
|
507
1042
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
1043
|
+
// Orphaned-placeholder check on the REASSEMBLED body — the same
|
|
1044
|
+
// gate for both modes.
|
|
1045
|
+
if (hasOrphanedPlaceholders(translatedBody)) {
|
|
1046
|
+
// Placeholder corruption — loud error, skip this file
|
|
1047
|
+
output.error('PLACEHOLDER CORRUPTION');
|
|
1048
|
+
throw refusePage(new Error(
|
|
1049
|
+
`Content sync body for ${code}: placeholder corruption detected.\n` +
|
|
1050
|
+
' Code blocks were corrupted during translation. Retry or report this issue.'
|
|
1051
|
+
));
|
|
1052
|
+
}
|
|
518
1053
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
1054
|
+
// Content-preservation check on the REASSEMBLED body. This lane
|
|
1055
|
+
// never ran the quality gate at all — translateBatch/
|
|
1056
|
+
// translateRawContent output went straight to disk AND into the TM
|
|
1057
|
+
// — so a body hollowed of its letters was written silently and then
|
|
1058
|
+
// re-served from cache forever. Throwing here matches the
|
|
1059
|
+
// placeholder gate above: the file is skipped, its manifest entry is
|
|
1060
|
+
// not advanced, and nothing is cached, so the next sync retries.
|
|
1061
|
+
const bodyHollowed = !bodyUsedFallback && checkContentPreservation(body, translatedBody);
|
|
1062
|
+
if (bodyHollowed) {
|
|
1063
|
+
output.error('CONTENT LOSS');
|
|
1064
|
+
throw refusePage(new Error(
|
|
1065
|
+
`Content sync body for ${code}: ${bodyHollowed.reason}.\n` +
|
|
1066
|
+
' Nothing was written or cached. If this is a low-coverage target language,\n' +
|
|
1067
|
+
' the model has no vocabulary for this text — fix the prompt or the pair, not the gate.'
|
|
1068
|
+
));
|
|
1069
|
+
}
|
|
1070
|
+
if (refusal.pageSource) refusal.filled.add(PAGE_UNIT);
|
|
535
1071
|
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
1072
|
+
// Store per-block AND whole-body entries only AFTER the corruption
|
|
1073
|
+
// check — the whole-body entry makes reverts/duplicates free. A
|
|
1074
|
+
// fallback body is NEVER stored whole: it contains untranslated
|
|
1075
|
+
// '[EN] ' text, and an error cached is an error forever.
|
|
1076
|
+
for (const s of pendingBlockStores) {
|
|
1077
|
+
storeTM(tm, s.source, code, s.tmKey, s.translation);
|
|
1078
|
+
}
|
|
1079
|
+
// Never a whole-body entry holding a person's paragraphs: the TM
|
|
1080
|
+
// holds machine output only, under the method that produced it.
|
|
1081
|
+
if (!bodyUsedFallback && !mixedProducers && keptBlocks.size === 0) {
|
|
1082
|
+
storeTM(tm, body, code, wholeBodyKey, translatedBody);
|
|
1083
|
+
}
|
|
1084
|
+
if (apiCalled && !bodyUsedFallback) output.raw(' [OK]');
|
|
545
1085
|
}
|
|
546
|
-
if (apiCalled && !bodyUsedFallback) output.raw(' [OK]');
|
|
547
1086
|
}
|
|
548
|
-
}
|
|
549
1087
|
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
1088
|
+
// Reassemble and write
|
|
1089
|
+
const assembled = reassembleContentFile({
|
|
1090
|
+
rawFrontMatter, translatedFields, translatedBody,
|
|
1091
|
+
hasFrontMatter, frontMatterFormat,
|
|
1092
|
+
});
|
|
1093
|
+
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
|
|
1094
|
+
fs.writeFileSync(targetPath, assembled, 'utf-8');
|
|
1095
|
+
|
|
1096
|
+
// Record what is now on disk, block by block, with the paragraphs
|
|
1097
|
+
// and fields that are a person's marked — so the next sync can tell
|
|
1098
|
+
// a later edit from this write, and keeps these ones again.
|
|
1099
|
+
updatedManifest[recordKey] = buildWrittenRecord({
|
|
1100
|
+
sourceBody: body, sourceFields, written: assembled,
|
|
1101
|
+
ownedBlocks: keptBlocks, ownedFields: keptFieldNames,
|
|
1102
|
+
});
|
|
1103
|
+
|
|
1104
|
+
result.translated = true;
|
|
1105
|
+
const keptCount = keptBlocks.size + keptFieldNames.size;
|
|
1106
|
+
output.event('file', {
|
|
1107
|
+
lane: 'content', file: relPath, locale: code,
|
|
1108
|
+
status: bodyUsedFallback ? 'fallback' : 'translated',
|
|
1109
|
+
...(keptCount > 0 && { keptEdits: keptCount }),
|
|
1110
|
+
});
|
|
1111
|
+
if (keptCount > 0) {
|
|
1112
|
+
result.keptEdits = true;
|
|
1113
|
+
output.info(
|
|
1114
|
+
`${code} — kept the edits made by hand to ${describeEdits({ paragraphs: keptBlocks.size, fields: [...keptFieldNames] })} ` +
|
|
1115
|
+
`of ${targetRel}. To re-translate them: champollion sync --redo files:${relPath}`
|
|
1116
|
+
);
|
|
1117
|
+
}
|
|
1118
|
+
for (const note of supersededNotes) output.warn(note);
|
|
1119
|
+
// A fallback body keeps its OLD manifest entry so the file re-fires
|
|
1120
|
+
// next sync — every good block is a TM hit, only the fallen-back
|
|
1121
|
+
// segment re-bills. Self-healing at bounded cost.
|
|
1122
|
+
if (!bodyUsedFallback) {
|
|
1123
|
+
updatedManifest[manifestKey] = currentSourceHash;
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
return result;
|
|
1127
|
+
};
|
|
1128
|
+
|
|
1129
|
+
// One (file × locale) failing must not stop the others — or discard
|
|
1130
|
+
// their work: the run carries on, and the lock + TM are saved below for
|
|
1131
|
+
// everything that succeeded (dogfood 2026-08-28, finding 5). Only a
|
|
1132
|
+
// FATAL error (a method that cannot run at all) stops the run.
|
|
1133
|
+
const perPairResults = await pMap(pairEntries, async (entry) => {
|
|
1134
|
+
try {
|
|
1135
|
+
return await translateOne(entry);
|
|
1136
|
+
} catch (err) {
|
|
1137
|
+
if (err.fatal) throw err;
|
|
1138
|
+
const code = entry[1].target;
|
|
1139
|
+
output.error(`${relPath} → ${code} — ${err.message}`);
|
|
1140
|
+
// heldNext: refused by the quality gate — held back from now on,
|
|
1141
|
+
// not retried by the next plain sync (lib/content-refusals.js).
|
|
1142
|
+
failedItems.push({ file: relPath, locale: code, error: err.message, ...(err.heldNext && { heldNext: true }) });
|
|
1143
|
+
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'failed', error: err.message });
|
|
1144
|
+
return { translated: false, fallback: false, skipped: false, retranslated: false };
|
|
1145
|
+
}
|
|
1146
|
+
}, { concurrency });
|
|
565
1147
|
|
|
566
|
-
|
|
567
|
-
|
|
1148
|
+
// Aggregate per-pair results into totals
|
|
1149
|
+
for (const r of perPairResults) {
|
|
1150
|
+
if (r.translated) translated++;
|
|
1151
|
+
if (r.skipped) skipped++;
|
|
1152
|
+
if (r.retranslated) retranslated++;
|
|
1153
|
+
if (r.keptEdits) keptEditFiles++;
|
|
1154
|
+
if (r.heldBack) heldBackSeen += r.heldBack;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
} catch (err) {
|
|
1158
|
+
// A fatal error still lets the saves below run: files finished before
|
|
1159
|
+
// it paid for their translations, and their lock entries + TM entries
|
|
1160
|
+
// are what make the retry cheap.
|
|
1161
|
+
fatalError = err;
|
|
1162
|
+
}
|
|
568
1163
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
1164
|
+
// What each translation's refusals are now (lib/content-refusals.js):
|
|
1165
|
+
// earlier ones that still apply, plus this run's — held back from the
|
|
1166
|
+
// method that refused them until a redo names the page, the source text
|
|
1167
|
+
// changes, or the model/method does.
|
|
1168
|
+
const heldBackItems = [];
|
|
1169
|
+
let heldBack = 0;
|
|
1170
|
+
let refusedNow = 0;
|
|
1171
|
+
for (const [manifestKey, st] of refusalStates) {
|
|
1172
|
+
storeRefusals(updatedManifest, manifestKey, nextRefusals(st.prior, st));
|
|
1173
|
+
if (st.held.length > 0) {
|
|
1174
|
+
heldBack += st.held.length;
|
|
1175
|
+
heldBackItems.push({ file: st.file, locale: st.locale, units: st.held, ...(st.pageHeld && { pageNotWritten: true }) });
|
|
574
1176
|
}
|
|
1177
|
+
refusedNow += new Set(st.refused.map(r => r.unit)).size;
|
|
575
1178
|
}
|
|
576
1179
|
|
|
577
1180
|
// Write updated content manifest (skip in dry-run)
|
|
@@ -585,23 +1188,85 @@ async function runContentSync(options) {
|
|
|
585
1188
|
|
|
586
1189
|
// Persist TM if it was mutated during this content sync (same rationale as
|
|
587
1190
|
// the key-value path in sync.js: dirty tracking, not size comparison).
|
|
588
|
-
//
|
|
589
|
-
if (!dryRun &&
|
|
590
|
-
const tmFinalSize = tmSize(tm);
|
|
591
|
-
const delta = tmFinalSize - tmInitialSize;
|
|
1191
|
+
// Under --fresh/--no-tm too: what was paid for is cached.
|
|
1192
|
+
if (!dryRun && isTMDirty(tm)) {
|
|
592
1193
|
saveTM(cwd, tm);
|
|
593
|
-
output.info(`[TM] Saved ${
|
|
1194
|
+
output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
|
|
594
1195
|
}
|
|
595
1196
|
|
|
596
1197
|
const totalCreated = translated;
|
|
597
|
-
if (totalCreated > 0 || skipped > 0 || retranslated > 0) {
|
|
1198
|
+
if (totalCreated > 0 || skipped > 0 || retranslated > 0 || heldItems.length > 0 || (dryRun && heldBackSeen > 0)) {
|
|
598
1199
|
const retranslateNote = retranslated > 0 ? ` (${retranslated} re-translated)` : '';
|
|
1200
|
+
const keptNote = keptEditFiles > 0 ? `, ${keptEditFiles} keeping edits made by hand` : '';
|
|
1201
|
+
const heldNote = heldItems.length > 0 ? `, ${heldItems.length} left as is (edited by hand — listed below)` : '';
|
|
599
1202
|
if (dryRun) {
|
|
600
|
-
|
|
1203
|
+
const heldBackNote = heldBackSeen > 0 ? `; ${heldBackSeen} block(s)/field(s) refused before would be held back (not sent)` : '';
|
|
1204
|
+
output.info(`Would have created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}${heldBackNote}.`);
|
|
601
1205
|
} else {
|
|
602
|
-
output.ok(`Created ${totalCreated} content file(s)${retranslateNote}, ${skipped} unchanged.`);
|
|
1206
|
+
output.ok(`Created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}.`);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
if (heldBack > 0) {
|
|
1210
|
+
// Not sent, not billed — but not translated either (exit 2, as for a
|
|
1211
|
+
// held-back key). Each page was named above with its own repair.
|
|
1212
|
+
output.warn(
|
|
1213
|
+
`${heldBack} content block(s)/field(s) held back in ${heldBackItems.length} translation(s) — the quality gate refused the method's `
|
|
1214
|
+
+ 'translation of their current text before; not sent, not billed (listed above). Ask again for all of them: '
|
|
1215
|
+
+ '`champollion sync --redo content`; for one page: `champollion sync --redo files:<page>`.'
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
if (heldItems.length > 0) {
|
|
1219
|
+
// Not a failure (nothing was lost, nothing billed) — but out of date
|
|
1220
|
+
// with its source, so it is listed on every run until resolved.
|
|
1221
|
+
output.raw('');
|
|
1222
|
+
output.raw(` Translations left as is because they were edited by hand and their source changed (${heldItems.length}):`);
|
|
1223
|
+
for (const h of heldItems) output.raw(` ${h.target} (source: ${h.file})`);
|
|
1224
|
+
output.raw(' Update each by hand (the next sync takes it as current), or replace it: champollion sync --redo files:<source>');
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
if (!dryRun) {
|
|
1228
|
+
for (const [pairKey, tally] of fallbackTallies) {
|
|
1229
|
+
printFallbackReport(pairKey, pairs.get(pairKey).method, tally, { unit: 'content segment(s)', producedLabel: 'pages' });
|
|
1230
|
+
warnFallbackMajority(pairKey, pairs.get(pairKey).method, tally, { unit: 'content segment(s)' });
|
|
603
1231
|
}
|
|
604
1232
|
}
|
|
1233
|
+
|
|
1234
|
+
if (fatalError) throw fatalError;
|
|
1235
|
+
if (failedItems.length > 0) {
|
|
1236
|
+
output.raw('');
|
|
1237
|
+
output.raw(` Failed content translations (${failedItems.length}) — not recorded as done; the next sync retries them`
|
|
1238
|
+
+ `${failedItems.some(f => f.heldNext) ? ', except what the quality gate refused (held back — `--redo files:<page>` asks again)' : ''}:`);
|
|
1239
|
+
for (const f of failedItems) output.raw(` ${f.file} → ${f.locale}${f.heldNext ? ' (refused by the quality gate: held back)' : ''}`);
|
|
1240
|
+
output.error(
|
|
1241
|
+
`${failedItems.length} content translation(s) failed (listed above). ` +
|
|
1242
|
+
'Completed files and their translations are saved; re-run sync to retry.'
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
// Returned, not thrown: a partly failed run still did real work, and the
|
|
1247
|
+
// caller folds this into the run summary and the exit code (2 = partial).
|
|
1248
|
+
return {
|
|
1249
|
+
translated,
|
|
1250
|
+
skipped,
|
|
1251
|
+
retranslated,
|
|
1252
|
+
failed: failedItems.length,
|
|
1253
|
+
failedItems: failedItems.map(({ heldNext, ...f }) => ({ ...f, state: heldNext ? 'held-back' : 'will-retry' })),
|
|
1254
|
+
// Blocks/fields the quality gate refused before, not sent this run (the
|
|
1255
|
+
// '[EN] ' text stays; a page with a held front-matter field is not
|
|
1256
|
+
// written), and blocks/fields it refused this run (held from the next).
|
|
1257
|
+
heldBack,
|
|
1258
|
+
heldBackItems,
|
|
1259
|
+
refused: refusedNow,
|
|
1260
|
+
// Translations written with a person's edits kept, and translations
|
|
1261
|
+
// left as is because their edits could not be merged with a source change.
|
|
1262
|
+
keptEdits: keptEditFiles,
|
|
1263
|
+
held: heldItems.length,
|
|
1264
|
+
heldItems,
|
|
1265
|
+
// What each pair's fallback method did (pairs with a fallback, real runs).
|
|
1266
|
+
...(!dryRun && fallbackTallies.size > 0 && {
|
|
1267
|
+
fallback: [...fallbackTallies].map(([pair, tally]) => ({ pair, ...fallbackSummary(tally) })),
|
|
1268
|
+
}),
|
|
1269
|
+
};
|
|
605
1270
|
}
|
|
606
1271
|
|
|
607
1272
|
// -----------------------------------------------------------------
|
|
@@ -642,6 +1307,9 @@ function readContentManifest(cwd) {
|
|
|
642
1307
|
*/
|
|
643
1308
|
function writeContentManifest(cwd, manifest) {
|
|
644
1309
|
const lockPath = path.join(cwd, CONTENT_LOCK_FILENAME);
|
|
1310
|
+
// Nothing to record and no lock yet (e.g. the method could not run at
|
|
1311
|
+
// all): don't leave an empty lock file in a folder the user never synced.
|
|
1312
|
+
if (Object.keys(manifest).length === 0 && !fs.existsSync(lockPath)) return;
|
|
645
1313
|
const sorted = {};
|
|
646
1314
|
for (const key of Object.keys(manifest).sort()) {
|
|
647
1315
|
sorted[key] = manifest[key];
|
|
@@ -673,18 +1341,28 @@ function hashFileContent(filePath) {
|
|
|
673
1341
|
*
|
|
674
1342
|
* Local file I/O only — cheap relative to the API dollars it estimates.
|
|
675
1343
|
*
|
|
676
|
-
* @param {string} contentDir - Path to
|
|
1344
|
+
* @param {string} contentDir - Path to the content directory
|
|
677
1345
|
* @param {string} sourceLocale - Source language code
|
|
678
1346
|
* @param {Array<[string, object]>} pairEntries - Pair graph entries [pairKey, pairConfig]
|
|
679
1347
|
* @param {string} cwd - Project root (for the content lock file)
|
|
1348
|
+
* @param {object} [options]
|
|
1349
|
+
* @param {object} [options.tm] - Loaded TM. When given, each pending item also
|
|
1350
|
+
* reports billedChars: only the front-matter fields and body blocks the TM
|
|
1351
|
+
* does NOT hold — what the run will actually pay for (lib/content-estimate.js).
|
|
1352
|
+
* @param {string[]|null} [options.translatableFields] - Front-matter fields
|
|
1353
|
+
* (null → DEFAULT_TRANSLATABLE_FIELDS, as runContentSync)
|
|
1354
|
+
* @param {boolean} [options.fresh] - --fresh: nothing refused before is held back
|
|
680
1355
|
* @returns {{ sourceFileCount: number, pendingTranslations: number, pendingSourceChars: number,
|
|
681
|
-
* byTarget: Object<string, { pendingTranslations: number, pendingSourceChars: number
|
|
1356
|
+
* byTarget: Object<string, { pendingTranslations: number, pendingSourceChars: number,
|
|
1357
|
+
* billedChars: number|null }> }}
|
|
682
1358
|
* pendingSourceChars sums the source file characters of every pending
|
|
683
1359
|
* (file × pair) translation — the rough basis for token estimation.
|
|
684
1360
|
* byTarget breaks the same counts down per target code so the cost table
|
|
685
1361
|
* can price each pair with its own method.
|
|
686
1362
|
*/
|
|
687
|
-
function countPendingContentTranslations(contentDir, sourceLocale, pairEntries, cwd
|
|
1363
|
+
function countPendingContentTranslations(contentDir, sourceLocale, pairEntries, cwd, {
|
|
1364
|
+
tm = null, translatableFields = null, fileScope = null, forceContent = false, fresh = false,
|
|
1365
|
+
} = {}) {
|
|
688
1366
|
const result = { sourceFileCount: 0, pendingTranslations: 0, pendingSourceChars: 0, byTarget: {} };
|
|
689
1367
|
if (!contentDir || !fs.existsSync(contentDir)) return result;
|
|
690
1368
|
|
|
@@ -693,9 +1371,13 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
|
|
|
693
1371
|
if (sourceFiles.length === 0) return result;
|
|
694
1372
|
|
|
695
1373
|
const contentManifest = readContentManifest(cwd);
|
|
1374
|
+
const fieldsList = translatableFields || DEFAULT_TRANSLATABLE_FIELDS;
|
|
1375
|
+
const parsedCache = new Map(); // sourcePath → { parsed, blocks } (locale-independent)
|
|
696
1376
|
|
|
697
1377
|
for (const sourcePath of sourceFiles) {
|
|
698
1378
|
const relPath = path.relative(contentDir, sourcePath);
|
|
1379
|
+
if (fileScope && !fileScope.includes(relPath)) continue;
|
|
1380
|
+
const retranslate = fileScope ? fileScope.retranslates(relPath) : false;
|
|
699
1381
|
const raw = fs.readFileSync(sourcePath, 'utf-8');
|
|
700
1382
|
const currentSourceHash = crypto.createHash('sha256').update(raw, 'utf-8').digest('hex');
|
|
701
1383
|
|
|
@@ -705,27 +1387,128 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
|
|
|
705
1387
|
if (!isPathContained(targetPath, contentDir)) continue;
|
|
706
1388
|
|
|
707
1389
|
const manifestKey = `${relPath}:${code}`;
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
1390
|
+
const targetExists = fs.existsSync(targetPath);
|
|
1391
|
+
const storedHash = contentManifest[manifestKey];
|
|
1392
|
+
if (!retranslate && targetExists) {
|
|
1393
|
+
if (storedHash && storedHash === currentSourceHash && !forceContent) continue; // unchanged — skipped by sync
|
|
711
1394
|
if (!storedHash) {
|
|
712
1395
|
// Hashless target: sync preserves it unless it is a legacy [EN] fallback
|
|
713
1396
|
const existingContent = fs.readFileSync(targetPath, 'utf-8');
|
|
714
1397
|
if (!existingContent.includes('[EN] ')) continue;
|
|
715
1398
|
}
|
|
716
1399
|
}
|
|
1400
|
+
if (targetExists) {
|
|
1401
|
+
// The same decision runContentSync makes (lib/content-review.js): a
|
|
1402
|
+
// translation edited by hand that the sync would leave as is costs
|
|
1403
|
+
// nothing. An unreadable record counts as pending (sync proceeds).
|
|
1404
|
+
let record = null;
|
|
1405
|
+
try { record = parseWrittenRecord(contentManifest[writtenRecordKey(manifestKey)]); } catch { record = null; }
|
|
1406
|
+
if (record) {
|
|
1407
|
+
const verdict = assessExistingTarget({
|
|
1408
|
+
targetRaw: fs.readFileSync(targetPath, 'utf-8'),
|
|
1409
|
+
record,
|
|
1410
|
+
segMode: pairConfig.contentSegmentation || 'block',
|
|
1411
|
+
replaceEdits: retranslate || (forceContent && Boolean(fileScope && fileScope.limitsFiles)),
|
|
1412
|
+
sourceCurrent: Boolean(storedHash) && storedHash === currentSourceHash,
|
|
1413
|
+
});
|
|
1414
|
+
if (verdict.action !== 'proceed') continue;
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
717
1417
|
|
|
718
1418
|
result.pendingTranslations += 1;
|
|
719
1419
|
result.pendingSourceChars += raw.length;
|
|
720
1420
|
if (!result.byTarget[code]) {
|
|
721
|
-
result.byTarget[code] = { pendingTranslations: 0, pendingSourceChars: 0 };
|
|
1421
|
+
result.byTarget[code] = { pendingTranslations: 0, pendingSourceChars: 0, billedChars: tm ? 0 : null };
|
|
722
1422
|
}
|
|
723
1423
|
result.byTarget[code].pendingTranslations += 1;
|
|
724
1424
|
result.byTarget[code].pendingSourceChars += raw.length;
|
|
1425
|
+
|
|
1426
|
+
if (tm) {
|
|
1427
|
+
// The same TM ladder runContentSync runs: per field, whole body,
|
|
1428
|
+
// then per block — only misses are billed.
|
|
1429
|
+
if (!parsedCache.has(sourcePath)) parsedCache.set(sourcePath, { parsed: parseContentFile(raw), blocks: null });
|
|
1430
|
+
const cached = parsedCache.get(sourcePath);
|
|
1431
|
+
const fields = {};
|
|
1432
|
+
if (cached.parsed.hasFrontMatter) {
|
|
1433
|
+
for (const field of fieldsList) {
|
|
1434
|
+
const v = cached.parsed.frontMatter[field];
|
|
1435
|
+
if (v && typeof v === 'string') fields[field] = v;
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
const { billedChars, totalChars } = billableContentChars({
|
|
1439
|
+
tm,
|
|
1440
|
+
code,
|
|
1441
|
+
tmKey: tmMethodKey(pairConfig),
|
|
1442
|
+
fallbackTmKey: pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null,
|
|
1443
|
+
fields,
|
|
1444
|
+
body: cached.parsed.body,
|
|
1445
|
+
segMode: pairConfig.contentSegmentation || 'block',
|
|
1446
|
+
blockSources: () => (cached.blocks ??= translatableBlockSources(cached.parsed.body)),
|
|
1447
|
+
// What the gate refused before is not sent to the method again
|
|
1448
|
+
// (lib/content-refusals.js) — the same holds the sync applies.
|
|
1449
|
+
holds: contentHolds(readRefusals(contentManifest, manifestKey), pairConfig, {
|
|
1450
|
+
redo: retranslate || forceContent || fresh,
|
|
1451
|
+
}),
|
|
1452
|
+
});
|
|
1453
|
+
// --retranslate bypasses the TM for this file: all of it is billed.
|
|
1454
|
+
result.byTarget[code].billedChars += retranslate ? totalChars : billedChars;
|
|
1455
|
+
}
|
|
725
1456
|
}
|
|
726
1457
|
}
|
|
727
1458
|
|
|
728
1459
|
return result;
|
|
729
1460
|
}
|
|
730
1461
|
|
|
731
|
-
|
|
1462
|
+
/**
|
|
1463
|
+
* The state of the content lane (contentDir) per target locale, for
|
|
1464
|
+
* `champollion status` — which had shown only the key-value lane, so a
|
|
1465
|
+
* newsletter folder and its files' state were invisible (Round 7, school
|
|
1466
|
+
* persona). Local file reads only; never fails (a diagnostic).
|
|
1467
|
+
*
|
|
1468
|
+
* translated — the target exists and was made from the current source
|
|
1469
|
+
* (the content lock's hash matches), with no '[EN] ' block
|
|
1470
|
+
* outOfDate — the target was made from an older source text
|
|
1471
|
+
* pending — no target yet, or blocks written as the '[EN] ' last
|
|
1472
|
+
* resort (the next sync asks for them again)
|
|
1473
|
+
* unrecorded — a target the lock has no record of (made by hand or by
|
|
1474
|
+
* another tool): sync keeps it as it is
|
|
1475
|
+
*
|
|
1476
|
+
* @param {string} contentDir
|
|
1477
|
+
* @param {string} sourceLocale
|
|
1478
|
+
* @param {Array<[string, object]>} pairEntries
|
|
1479
|
+
* @param {string} cwd
|
|
1480
|
+
* @param {{ fallbackPrefix?: string }} [opts]
|
|
1481
|
+
* @returns {{ dir: string, files: number, locales: Object<string, { translated: number,
|
|
1482
|
+
* outOfDate: string[], pending: string[], unrecorded: string[] }> }|null}
|
|
1483
|
+
*/
|
|
1484
|
+
function contentStatus(contentDir, sourceLocale, pairEntries, cwd, { fallbackPrefix = '[EN] ' } = {}) {
|
|
1485
|
+
if (!contentDir || !fs.existsSync(contentDir)) return null;
|
|
1486
|
+
let sourceFiles = [];
|
|
1487
|
+
try { sourceFiles = discoverContentFiles(contentDir, sourceLocale); } catch { return null; }
|
|
1488
|
+
const manifest = readContentManifest(cwd);
|
|
1489
|
+
const out = { dir: path.relative(cwd, contentDir) || '.', files: sourceFiles.length, locales: {} };
|
|
1490
|
+
for (const [, pairConfig] of pairEntries) {
|
|
1491
|
+
out.locales[pairConfig.target] ??= { translated: 0, outOfDate: [], pending: [], unrecorded: [] };
|
|
1492
|
+
}
|
|
1493
|
+
for (const sourcePath of sourceFiles) {
|
|
1494
|
+
const relPath = path.relative(contentDir, sourcePath).split(path.sep).join('/');
|
|
1495
|
+
let hash;
|
|
1496
|
+
try { hash = hashFileContent(sourcePath); } catch { continue; }
|
|
1497
|
+
for (const [, pairConfig] of pairEntries) {
|
|
1498
|
+
const code = pairConfig.target;
|
|
1499
|
+
const state = out.locales[code];
|
|
1500
|
+
const targetPath = getTargetContentPath(sourcePath, code, sourceLocale);
|
|
1501
|
+
if (!fs.existsSync(targetPath)) { state.pending.push(relPath); continue; }
|
|
1502
|
+
let text = '';
|
|
1503
|
+
try { text = fs.readFileSync(targetPath, 'utf-8'); } catch { /* unreadable: by its record only */ }
|
|
1504
|
+
const stored = manifest[`${path.relative(contentDir, sourcePath)}:${code}`];
|
|
1505
|
+
if (text.includes(fallbackPrefix)) state.pending.push(relPath);
|
|
1506
|
+
else if (!stored) state.unrecorded.push(relPath);
|
|
1507
|
+
else if (stored !== hash) state.outOfDate.push(relPath);
|
|
1508
|
+
else state.translated += 1;
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
return out;
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
export { runContentSync, countPendingContentTranslations, readContentManifest, contentStatus, CONTENT_LOCK_FILENAME };
|