champollion 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/autofix.js +7 -9
- package/lib/commands/init.js +20 -14
- package/lib/commands/wrap.js +2 -2
- package/lib/config.js +34 -3
- package/lib/content-estimate.js +5 -5
- package/lib/content-refusals.js +27 -13
- package/lib/content-sync.js +59 -51
- package/lib/cost-report.js +5 -5
- package/lib/docusaurus-sync.js +42 -41
- package/lib/fallback.js +157 -19
- package/lib/locale-state.js +4 -0
- package/lib/methods/direct-llm.js +13 -8
- package/lib/methods/llm-coached.js +3 -1
- package/lib/methods/llm.js +7 -2
- package/lib/models.js +91 -55
- package/lib/pairs.js +8 -8
- package/lib/segment.js +22 -7
- package/lib/validate.js +118 -6
- package/lib/verify.js +38 -2
- package/package.json +1 -1
- package/shared/licenses.json +99 -3
- package/shared/retired-model-aliases.json +9 -0
- package/shared/schemas/human-services.schema.json +1 -1
- package/shared/schemas/retired-model-aliases.schema.json +32 -0
- package/shared/model-aliases.json +0 -7
- package/shared/schemas/model-aliases.schema.json +0 -27
package/lib/autofix.js
CHANGED
|
@@ -385,14 +385,16 @@ function wrapUnsupportedReason(file) {
|
|
|
385
385
|
* (this used to look only for `<locale>.json`, so TOML/YAML projects lost
|
|
386
386
|
* every extracted key without a word).
|
|
387
387
|
*
|
|
388
|
-
* Source gets the extracted text
|
|
389
|
-
* keys
|
|
390
|
-
*
|
|
388
|
+
* Source gets the extracted text. Target files are left alone: the next sync
|
|
389
|
+
* sees the new keys missing there and translates them (an "[EN] " placeholder
|
|
390
|
+
* used to be written into each target, and shipped if no sync followed —
|
|
391
|
+
* nothing marked untranslated is written any more). Keys already present are
|
|
392
|
+
* never overwritten.
|
|
391
393
|
*
|
|
392
394
|
* @param {object[]} fixes - Array of { key, text } from processFile
|
|
393
395
|
* @param {{ path: string, format: string }} sourceFile - Source locale file
|
|
394
|
-
* @param {Array<{ path: string, format: string }>} targetFiles - Target locale files
|
|
395
|
-
* @returns {{ source: boolean, targets: number }} What was written
|
|
396
|
+
* @param {Array<{ path: string, format: string }>} targetFiles - Target locale files (checked, not written)
|
|
397
|
+
* @returns {{ source: boolean, targets: number }} What was written (targets: always 0)
|
|
396
398
|
*/
|
|
397
399
|
function addKeysToLocales(fixes, sourceFile, targetFiles = []) {
|
|
398
400
|
const written = { source: false, targets: 0 };
|
|
@@ -435,10 +437,6 @@ function addKeysToLocales(fixes, sourceFile, targetFiles = []) {
|
|
|
435
437
|
};
|
|
436
438
|
|
|
437
439
|
written.source = addTo(sourceFile, '');
|
|
438
|
-
// Use [EN] prefix for target locales as untranslated markers
|
|
439
|
-
for (const target of targetFiles) {
|
|
440
|
-
if (addTo(target, '[EN] ')) written.targets++;
|
|
441
|
-
}
|
|
442
440
|
return written;
|
|
443
441
|
}
|
|
444
442
|
|
package/lib/commands/init.js
CHANGED
|
@@ -32,7 +32,7 @@ import readline from 'node:readline';
|
|
|
32
32
|
import { CONFIG_FILENAMES, DEFAULT_OPENROUTER_MODEL, DEFAULT_BATCH_SIZE, DEFAULT_TEMPERATURE, DEFAULT_COACHED_TEMPERATURE, detectDocusaurus } from '../config.js';
|
|
33
33
|
import { DEFAULT_REGISTERS, getLanguageCard, getRegisterPresets, isMethodSupported, resolveCode, summarizeGenderGuidance, isPrivateUseCode } from '../registers.js';
|
|
34
34
|
import { getConverterInfo, resolveTargetScript, converterKeyForLocale } from '../scripts.js';
|
|
35
|
-
import { fetchAvailableModels, resolveProviderApiKey, isListableProvider, getProviderLabel } from '../models.js';
|
|
35
|
+
import { fetchAvailableModels, resolveProviderApiKey, isListableProvider, getProviderLabel, requireExactModelId } from '../models.js';
|
|
36
36
|
import { showCommandHelp } from '../command-help.js';
|
|
37
37
|
import { output } from '../output.js';
|
|
38
38
|
import { flutterLocaleLines } from '../flutter-locales.js';
|
|
@@ -953,20 +953,21 @@ async function pickModelForProvider(rl, method, presetModel = null) {
|
|
|
953
953
|
}
|
|
954
954
|
console.log('');
|
|
955
955
|
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
956
|
+
// A typed id is checked here, as --model is: an alias or a floating id is
|
|
957
|
+
// refused with the exact slug to write, and asked again — not written into
|
|
958
|
+
// the config to fail at the first sync.
|
|
959
|
+
for (;;) {
|
|
960
|
+
const modelChoice = await ask(rl, 'Choose (number or model ID)', '1');
|
|
961
|
+
const modelNum = parseInt(modelChoice, 10);
|
|
962
|
+
if (modelNum >= 1 && modelNum <= models.length) return models[modelNum - 1];
|
|
963
|
+
if (!modelChoice.trim()) return models[0];
|
|
964
|
+
try {
|
|
965
|
+
requireExactModelId(modelChoice.trim(), { from: 'typed' });
|
|
966
|
+
return modelChoice.trim();
|
|
967
|
+
} catch (err) {
|
|
968
|
+
console.log(` ${err.message}`);
|
|
969
|
+
}
|
|
966
970
|
}
|
|
967
|
-
|
|
968
|
-
// Empty input — use the first model from the list
|
|
969
|
-
return models[0];
|
|
970
971
|
}
|
|
971
972
|
|
|
972
973
|
/**
|
|
@@ -1777,6 +1778,11 @@ function validateInitFlags(args) {
|
|
|
1777
1778
|
return `--endpoint applies to --method api (a champollion API endpoint)${args.method ? `, not --method ${args.method}` : ''}. `
|
|
1778
1779
|
+ 'For an OpenAI-compatible server use --method local and LOCAL_API_BASE.';
|
|
1779
1780
|
}
|
|
1781
|
+
// Exact model slugs only (founder ruling 2026-10-05): a retired alias or a
|
|
1782
|
+
// floating id is refused before init writes it into the config.
|
|
1783
|
+
if (typeof args.model === 'string' && args.model) {
|
|
1784
|
+
try { requireExactModelId(args.model, { from: 'from --model' }); } catch (err) { return err.message; }
|
|
1785
|
+
}
|
|
1780
1786
|
if (args['accepts-instructions'] != null) {
|
|
1781
1787
|
if (args.method !== API_METHOD) return '--accepts-instructions applies to --method api.';
|
|
1782
1788
|
if (!['true', 'false'].includes(String(args['accepts-instructions']))) {
|
package/lib/commands/wrap.js
CHANGED
|
@@ -127,9 +127,9 @@ async function run(args, cwd) {
|
|
|
127
127
|
|
|
128
128
|
// Add extracted keys to locale files (only if not dry-run)
|
|
129
129
|
if (!isDry && allFixes.length > 0) {
|
|
130
|
-
|
|
130
|
+
addKeysToLocales(allFixes, destination.source, destination.targets);
|
|
131
131
|
output.info(`Added ${allFixes.length} key(s) to ${destination.source.rel}`
|
|
132
|
-
+ (
|
|
132
|
+
+ (destination.targets.length > 0 ? ' — run `champollion sync` to translate them into each target' : ''));
|
|
133
133
|
} else if (isDry && allFixes.length > 0) {
|
|
134
134
|
output.info(`Would add ${allFixes.length} key(s) to ${destination.source.rel}`);
|
|
135
135
|
}
|
package/lib/config.js
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
import fs from 'node:fs';
|
|
15
15
|
import path from 'node:path';
|
|
16
16
|
import { DEFAULT_REGISTERS, getLanguageCard, getRegister, resolveCode } from './registers.js';
|
|
17
|
-
import {
|
|
17
|
+
import { requireExactModelId } from './models.js';
|
|
18
18
|
import { validateNoTranslateConfig } from './no-translate.js';
|
|
19
19
|
import { output } from './output.js';
|
|
20
20
|
import { splitKeyList } from './redo.js';
|
|
@@ -65,6 +65,31 @@ const DEFAULT_METHOD_CONCURRENCY = 4;
|
|
|
65
65
|
const EST_INPUT_TOKENS_PER_KEY = 200;
|
|
66
66
|
const EST_OUTPUT_TOKENS_PER_KEY = 30;
|
|
67
67
|
const EST_CHARS_PER_KEY = 25;
|
|
68
|
+
// Markdown content priced by a token-billed method (an LLM): one key-unit per
|
|
69
|
+
// 60 source characters, not 25. A UI key carries its share of the prompt on
|
|
70
|
+
// every 25-char slice; a content batch carries the prompt once per page and
|
|
71
|
+
// the text itself is ~4 characters a token. Measured on champollion.dev with
|
|
72
|
+
// google/gemini-3.8-flash (2026-10-05): at 25 the estimate ran 2.8x over the
|
|
73
|
+
// bill for 95 Spanish pages and 8.2x over for one Arabic page; at 60 it is
|
|
74
|
+
// ~1.2x and ~3.4x — still above the bill, as the --max-cost cap needs.
|
|
75
|
+
// Character-billed engines (DeepL, Google, Microsoft) keep 25: for them the
|
|
76
|
+
// key-unit converts back to the exact characters they bill.
|
|
77
|
+
const EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED = 60;
|
|
78
|
+
const TOKEN_PRICED_METHODS = new Set(['llm', 'llm-coached', 'openai', 'anthropic', 'gemini', 'local']);
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Key-units for a run of Markdown source characters, priced by `pairConfig`'s
|
|
82
|
+
* method (see EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED).
|
|
83
|
+
*
|
|
84
|
+
* @param {number} chars
|
|
85
|
+
* @param {object} [pairConfig]
|
|
86
|
+
* @returns {number}
|
|
87
|
+
*/
|
|
88
|
+
function contentKeyUnits(chars, pairConfig = {}) {
|
|
89
|
+
const perKey = TOKEN_PRICED_METHODS.has(pairConfig?.method || 'llm')
|
|
90
|
+
? EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED : EST_CHARS_PER_KEY;
|
|
91
|
+
return Math.ceil(chars / perKey);
|
|
92
|
+
}
|
|
68
93
|
|
|
69
94
|
const DEFAULTS = {
|
|
70
95
|
version: 3,
|
|
@@ -446,8 +471,12 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
|
|
|
446
471
|
config.localesDir = compiled.base;
|
|
447
472
|
}
|
|
448
473
|
|
|
449
|
-
//
|
|
450
|
-
|
|
474
|
+
// Exact model slugs only (founder ruling 2026-10-05): a retired alias
|
|
475
|
+
// ("gemini-flash") or a floating id ("~google/gemini-flash-latest") is
|
|
476
|
+
// refused here, naming the slug to write — it never resolves to a model.
|
|
477
|
+
// The file's own model is checked too: a pair's fallback runs on it.
|
|
478
|
+
requireExactModelId(config.model, { from: config._modelOverride ? 'from --model' : 'from the top-level "model"' });
|
|
479
|
+
if (config._modelOverride) requireExactModelId(config._fileModel, { from: 'from the top-level "model"' });
|
|
451
480
|
|
|
452
481
|
// Coaching file: read the file contents into coachingPrompt if coachingFile is set.
|
|
453
482
|
// This allows users to maintain coaching prompts as separate text files rather
|
|
@@ -700,6 +729,8 @@ export {
|
|
|
700
729
|
EST_INPUT_TOKENS_PER_KEY,
|
|
701
730
|
EST_OUTPUT_TOKENS_PER_KEY,
|
|
702
731
|
EST_CHARS_PER_KEY,
|
|
732
|
+
EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED,
|
|
733
|
+
contentKeyUnits,
|
|
703
734
|
DEFAULT_MAX_RETRIES,
|
|
704
735
|
DEFAULT_METHOD_CONCURRENCY,
|
|
705
736
|
};
|
package/lib/content-estimate.js
CHANGED
|
@@ -55,9 +55,10 @@ export function translatableBlockSources(body) {
|
|
|
55
55
|
* @param {ReturnType<import('./content-refusals.js').contentHolds>|null} [args.holds] -
|
|
56
56
|
* What the quality gate refused before (lib/content-refusals.js): a block
|
|
57
57
|
* or field the pair's method refused is not sent to it (held, or the
|
|
58
|
-
* fallback's — which this estimate does not price); a held
|
|
59
|
-
*
|
|
60
|
-
*
|
|
58
|
+
* fallback's — which this estimate does not price); a held page body in
|
|
59
|
+
* 'page' segmentation holds its whole page, so nothing of it is billed. A
|
|
60
|
+
* held front-matter field keeps its source text and the rest of the page
|
|
61
|
+
* is still sent (since 2026-10-05)
|
|
61
62
|
* @returns {{ billedChars: number, totalChars: number }}
|
|
62
63
|
*/
|
|
63
64
|
export function billableContentChars({ tm, code, tmKey, fallbackTmKey = null, fields, body, segMode, blockSources, holds = null }) {
|
|
@@ -71,8 +72,7 @@ export function billableContentChars({ tm, code, tmKey, fallbackTmKey = null, fi
|
|
|
71
72
|
totalChars += text.length;
|
|
72
73
|
if (cached(text)) continue;
|
|
73
74
|
const hold = holds ? holds.field(field, text) : 'send';
|
|
74
|
-
if (hold === '
|
|
75
|
-
else if (hold === 'send') billedChars += text.length;
|
|
75
|
+
if (hold === 'send') billedChars += text.length;
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
if (body.trim()) {
|
package/lib/content-refusals.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* own entries), one entry per (page × locale) that has refusals:
|
|
17
17
|
*
|
|
18
18
|
* "refused:<manifestKey>": {
|
|
19
|
-
* "block:<hash12>": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" },
|
|
19
|
+
* "block:<hash12>": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04", "gate": 2 },
|
|
20
20
|
* "field:title": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" },
|
|
21
21
|
* "page": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" }
|
|
22
22
|
* }
|
|
@@ -52,10 +52,24 @@
|
|
|
52
52
|
*/
|
|
53
53
|
|
|
54
54
|
import { tmMethodKey, lookupTM } from './tm.js';
|
|
55
|
+
import { GATE_VERSION } from './validate.js';
|
|
55
56
|
import { holdState, shortSourceHash } from './locale-state.js';
|
|
56
57
|
import { contentRedoCommand } from './verify.js';
|
|
57
58
|
import { translatableBlockSources } from './content-estimate.js';
|
|
58
59
|
|
|
60
|
+
/**
|
|
61
|
+
* The lock value of a page written with something left in the source language
|
|
62
|
+
* (a block or front-matter field the gate refused): `pending:<source hash>`.
|
|
63
|
+
* It differs from the source hash, so the next sync processes the page again
|
|
64
|
+
* (the cache makes that free; a held unit is not re-sent), and it says the
|
|
65
|
+
* page is the tool's — a page with no lock entry is taken for a person's
|
|
66
|
+
* work and preserved. Before 2026-10-05 a visible '[EN] ' marker in the page
|
|
67
|
+
* did that job; no marker is written any more.
|
|
68
|
+
*/
|
|
69
|
+
export const PENDING_LOCK_PREFIX = 'pending:';
|
|
70
|
+
export const pendingLockValue = (sourceHash) => `${PENDING_LOCK_PREFIX}${sourceHash}`;
|
|
71
|
+
export const isPendingLock = (value) => typeof value === 'string' && value.startsWith(PENDING_LOCK_PREFIX);
|
|
72
|
+
|
|
59
73
|
/** The content-lock key of one page × locale's refusals. */
|
|
60
74
|
export function refusalRecordKey(manifestKey) {
|
|
61
75
|
return `refused:${manifestKey}`;
|
|
@@ -85,7 +99,10 @@ export function readRefusals(manifest, manifestKey) {
|
|
|
85
99
|
if (!rec || typeof rec !== 'object' || Array.isArray(rec)) return {};
|
|
86
100
|
const out = {};
|
|
87
101
|
for (const [id, r] of Object.entries(rec)) {
|
|
88
|
-
|
|
102
|
+
// A refusal by an earlier gate lifts by itself (validate.js GATE_VERSION):
|
|
103
|
+
// what an over-strict check refused is asked again once it is fixed.
|
|
104
|
+
if (r && typeof r === 'object' && typeof r.source === 'string' && Array.isArray(r.methods)
|
|
105
|
+
&& r.gate === GATE_VERSION) out[id] = r;
|
|
89
106
|
}
|
|
90
107
|
return out;
|
|
91
108
|
}
|
|
@@ -140,7 +157,7 @@ export function nextRefusals(prior, { blockSources = [], pageSource = null, fiel
|
|
|
140
157
|
if (!methods || methods.length === 0 || filled.has(unit)) continue;
|
|
141
158
|
const src = shortSourceHash(source);
|
|
142
159
|
const known = out[unit] && out[unit].source === src ? out[unit].methods : [];
|
|
143
|
-
out[unit] = { source: src, methods: [...new Set([...known, ...methods])], on };
|
|
160
|
+
out[unit] = { source: src, methods: [...new Set([...known, ...methods])], on, gate: GATE_VERSION };
|
|
144
161
|
}
|
|
145
162
|
return Object.keys(out).length > 0 ? out : null;
|
|
146
163
|
}
|
|
@@ -161,21 +178,18 @@ export function storeRefusals(manifest, manifestKey, record) {
|
|
|
161
178
|
* @param {string} p.pairKey
|
|
162
179
|
* @param {object} p.pairConfig
|
|
163
180
|
* @param {string[]} p.names - "paragraph 2", 'front matter "title"'
|
|
164
|
-
* @param {boolean} [p.pageHeld] -
|
|
165
|
-
*
|
|
181
|
+
* @param {boolean} [p.pageHeld] - The page body translated whole (page mode)
|
|
182
|
+
* is held: the page is not written
|
|
166
183
|
* @param {boolean} [p.handWritten] - The lane keeps a paragraph written by hand (contentDir)
|
|
167
|
-
* @param {string} [p.fallbackPrefix]
|
|
168
184
|
* @returns {string}
|
|
169
185
|
*/
|
|
170
|
-
export function describeHeld({ file, code, pairKey, pairConfig, names, pageHeld = false, handWritten = false
|
|
186
|
+
export function describeHeld({ file, code, pairKey, pairConfig, names, pageHeld = false, handWritten = false }) {
|
|
171
187
|
const shown = names.slice(0, 5).join(', ') + (names.length > 5 ? `, +${names.length - 5} more` : '');
|
|
172
188
|
const fb = pairConfig.fallback;
|
|
173
189
|
const who = fb ? `${pairConfig.method}'s and its fallback's (${fb.method})` : `${pairConfig.method}'s`;
|
|
174
190
|
const left = !pageHeld
|
|
175
|
-
?
|
|
176
|
-
:
|
|
177
|
-
? 'A page translated whole has no \'[EN]\' text to keep, so it is not written (and nothing of it is sent) until it is filled.'
|
|
178
|
-
: 'A front-matter field the gate refuses fails its page, so the page is not written (and nothing of it is sent) until it is filled.';
|
|
191
|
+
? 'They stay in the source language (unmarked) until they are filled; the rest of the page is written.'
|
|
192
|
+
: 'A page translated whole is not written (and nothing of it is sent) until it is filled.';
|
|
179
193
|
const other = fb
|
|
180
194
|
? 'another "fallback" method on the pair'
|
|
181
195
|
: `a "fallback" method on the pair (it is asked for what ${pairConfig.method} refused)`;
|
|
@@ -222,12 +236,12 @@ export function describeNewHold({ file, pairKey, pairConfig, count, fallbackAske
|
|
|
222
236
|
* @returns {string}
|
|
223
237
|
*/
|
|
224
238
|
export function describeFallenBack({ file, pairKey, pairConfig, fellBack, newlyHeld, refusedBy = [] }) {
|
|
225
|
-
if (newlyHeld.length === 0) return 'Not cached
|
|
239
|
+
if (newlyHeld.length === 0) return 'Not cached; the page is recorded as pending: the next sync retries just those block(s).';
|
|
226
240
|
const retry = fellBack - newlyHeld.length;
|
|
227
241
|
const fbKey = pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null;
|
|
228
242
|
const fallbackAsked = !!fbKey && newlyHeld.some(i => !(refusedBy[i] || []).includes(fbKey));
|
|
229
243
|
const n = newlyHeld.length;
|
|
230
|
-
return 'Not cached
|
|
244
|
+
return 'Not cached; the page is recorded as pending. '
|
|
231
245
|
+ (retry > 0 ? `The next sync asks again for the ${retry} that got no usable answer. ` : '')
|
|
232
246
|
+ describeNewHold({
|
|
233
247
|
file, pairKey, pairConfig, count: n, fallbackAsked,
|
package/lib/content-sync.js
CHANGED
|
@@ -89,7 +89,7 @@ import {
|
|
|
89
89
|
} from './fallback.js';
|
|
90
90
|
import {
|
|
91
91
|
readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
|
|
92
|
-
describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME,
|
|
92
|
+
describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock,
|
|
93
93
|
} from './content-refusals.js';
|
|
94
94
|
import { contentRedoCommand } from './verify.js';
|
|
95
95
|
|
|
@@ -364,7 +364,8 @@ async function runContentSync(options) {
|
|
|
364
364
|
|
|
365
365
|
if (!retranslate && targetExists && !storedHash) {
|
|
366
366
|
// No stored hash — check if this file was generated by a prior
|
|
367
|
-
// champollion run (
|
|
367
|
+
// champollion run (a legacy '[EN] ' fallback marker, written before
|
|
368
|
+
// 2026-10-05 — such pages are now recorded as pending:<hash>) or is a genuine
|
|
368
369
|
// hand-translated file that should be preserved.
|
|
369
370
|
//
|
|
370
371
|
// BUG FIX: Previously, this always skipped hashless files.
|
|
@@ -455,11 +456,14 @@ async function runContentSync(options) {
|
|
|
455
456
|
if (sourceCurrent) {
|
|
456
457
|
output.info(`${code} — re-processing (--force-content; cached text is reused)`);
|
|
457
458
|
result.retranslated = true;
|
|
459
|
+
} else if (storedHash === pendingLockValue(currentSourceHash)) {
|
|
460
|
+
output.info(`${code} — re-checking what was left in the source language last time (cached text is free)`);
|
|
461
|
+
result.retranslated = true;
|
|
458
462
|
} else if (storedHash) {
|
|
459
463
|
output.info(`${code} — source updated, re-translating`);
|
|
460
464
|
result.retranslated = true;
|
|
461
465
|
} else {
|
|
462
|
-
output.info(`${code} — replacing [EN] fallback`);
|
|
466
|
+
output.info(`${code} — replacing an older '[EN] ' fallback`);
|
|
463
467
|
}
|
|
464
468
|
}
|
|
465
469
|
|
|
@@ -532,7 +536,7 @@ async function runContentSync(options) {
|
|
|
532
536
|
const holdPage = (names) => {
|
|
533
537
|
refusal.held.push(...names);
|
|
534
538
|
refusal.pageHeld = true;
|
|
535
|
-
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, pageHeld: true, handWritten: true
|
|
539
|
+
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, pageHeld: true, handWritten: true }));
|
|
536
540
|
output.event('file', { lane: 'content', file: relPath, locale: code, status: 'held-refused', held: names.length });
|
|
537
541
|
result.heldBack = names.length;
|
|
538
542
|
return result;
|
|
@@ -596,6 +600,9 @@ async function runContentSync(options) {
|
|
|
596
600
|
// Each field is cached on its own source text, exactly like a
|
|
597
601
|
// key-value sync key: a title edit re-pays only the title.
|
|
598
602
|
const translatedFields = {};
|
|
603
|
+
// A field left in the source language (refused, or held from an earlier
|
|
604
|
+
// refusal): the page is written, its lock marked pending.
|
|
605
|
+
let fieldsLeftInSource = false;
|
|
599
606
|
// Edits made by hand that this write keeps, and the ones whose
|
|
600
607
|
// source text changed under them (re-translated; wording printed).
|
|
601
608
|
const keptFieldNames = new Set();
|
|
@@ -648,7 +655,15 @@ async function runContentSync(options) {
|
|
|
648
655
|
for (const f of Object.keys(translatedFields)) refusal.filled.add(fieldUnit(f));
|
|
649
656
|
// Refused before: held back (the page with it), or the fallback's only.
|
|
650
657
|
const heldFields = fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'held');
|
|
651
|
-
if (heldFields.length > 0)
|
|
658
|
+
if (heldFields.length > 0) {
|
|
659
|
+
// Not sent again: the field keeps its source text and the page is
|
|
660
|
+
// written (its lock marked pending).
|
|
661
|
+
fieldsLeftInSource = true;
|
|
662
|
+
const names = heldFields.map(f => `front matter "${f}"`);
|
|
663
|
+
refusal.held.push(...names);
|
|
664
|
+
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true }));
|
|
665
|
+
for (const f of heldFields) fieldsToSend.splice(fieldsToSend.indexOf(f), 1);
|
|
666
|
+
}
|
|
652
667
|
const fallbackOnlyFields = new Set(fieldsToSend.filter(f => holds.field(f, fieldsToTranslate[f]) === 'fallback-only'));
|
|
653
668
|
|
|
654
669
|
if (fieldsToSend.length > 0) {
|
|
@@ -668,9 +683,9 @@ async function runContentSync(options) {
|
|
|
668
683
|
budget: fallbackBudget,
|
|
669
684
|
sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
|
|
670
685
|
label: `content:${relPath} front matter`,
|
|
671
|
-
translate: (keys, cfg) => translateBatch(
|
|
686
|
+
translate: (keys, cfg, extra = {}) => translateBatch(
|
|
672
687
|
keys, fieldsToTranslate, cfg,
|
|
673
|
-
{ apiKey, cwd, model: cfg.model, batchSize: cfg.batchSize || 30 },
|
|
688
|
+
{ apiKey, cwd, model: cfg.model, batchSize: cfg.batchSize || 30, ...extra },
|
|
674
689
|
),
|
|
675
690
|
fallbackOnly: fallbackOnlyFields,
|
|
676
691
|
});
|
|
@@ -681,29 +696,20 @@ async function runContentSync(options) {
|
|
|
681
696
|
refusal.refused.push({ unit: fieldUnit(field), source: fieldsToTranslate[field], methods });
|
|
682
697
|
}
|
|
683
698
|
if (fm.hollowed.length > 0) {
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
`Content sync for ${code}: front matter "${h.field}" — ${h.reason}.\n` +
|
|
699
|
-
` source: ${JSON.stringify(fieldsToTranslate[h.field])}\n` +
|
|
700
|
-
` got: ${JSON.stringify(h.value)}\n` +
|
|
701
|
-
(h.fallbackReason ? ` the fallback (${pairConfig.fallback.method}) failed it too: ${h.fallbackReason}\n` : '') +
|
|
702
|
-
' Nothing was written or cached.\n' + advice + '\n' +
|
|
703
|
-
` ${describeNewHold({ file: relPath, pairKey, pairConfig, count: fm.hollowed.length })}`
|
|
704
|
-
);
|
|
705
|
-
err.heldNext = true;
|
|
706
|
-
throw err;
|
|
699
|
+
// Refused, and refused again when asked with the reason: the
|
|
700
|
+
// field keeps its source text; the page is still written.
|
|
701
|
+
fieldsLeftInSource = true;
|
|
702
|
+
for (const h of fm.hollowed) {
|
|
703
|
+
const advice = h.sharedOutput
|
|
704
|
+
? ` ${pairConfig.method} answers it only with a sentence it gave for other strings — `
|
|
705
|
+
+ `${pairConfig.fallback ? 'and the fallback did not translate it either' : 'add a "fallback" method to the pair'}, or write it in ${targetRel} by hand (kept).`
|
|
706
|
+
: '';
|
|
707
|
+
output.warn(
|
|
708
|
+
`${relPath} → ${code}: front matter "${h.field}" kept in the source language — ${h.reason}`
|
|
709
|
+
+ `${h.fallbackReason ? `; the fallback (${pairConfig.fallback.method}): ${h.fallbackReason}` : ''}.${advice}`
|
|
710
|
+
);
|
|
711
|
+
}
|
|
712
|
+
output.warn(describeNewHold({ file: relPath, pairKey, pairConfig, count: fm.hollowed.length }));
|
|
707
713
|
}
|
|
708
714
|
if (fm.noResults) {
|
|
709
715
|
// Front matter translation failed — loud error, skip this file
|
|
@@ -722,8 +728,10 @@ async function runContentSync(options) {
|
|
|
722
728
|
// the pair's method, so the page is held, as with no fallback.
|
|
723
729
|
const unfilled = [...fallbackOnlyFields].filter(f => !(f in fm.translated));
|
|
724
730
|
if (unfilled.length > 0) {
|
|
725
|
-
|
|
726
|
-
|
|
731
|
+
fieldsLeftInSource = true;
|
|
732
|
+
const names = unfilled.map(f => `front matter "${f}"`);
|
|
733
|
+
refusal.held.push(...names);
|
|
734
|
+
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true }));
|
|
727
735
|
}
|
|
728
736
|
Object.assign(translatedFields, fm.translated);
|
|
729
737
|
for (const f of Object.keys(fm.translated)) refusal.filled.add(fieldUnit(f));
|
|
@@ -870,7 +878,7 @@ async function runContentSync(options) {
|
|
|
870
878
|
}));
|
|
871
879
|
|
|
872
880
|
const missed = [];
|
|
873
|
-
const heldBlocks = []; // refused before: not sent,
|
|
881
|
+
const heldBlocks = []; // refused before: not sent, the source text kept
|
|
874
882
|
let position = 0; // index among translatable blocks (= translatableBlockSources order)
|
|
875
883
|
for (const r of rendered) {
|
|
876
884
|
if (r.seg.type !== 'translatable') {
|
|
@@ -900,7 +908,7 @@ async function runContentSync(options) {
|
|
|
900
908
|
if (hold === 'held') {
|
|
901
909
|
// Refused before by this method (and its fallback): the
|
|
902
910
|
// honest last resort stays, nothing is sent or billed.
|
|
903
|
-
r.out =
|
|
911
|
+
r.out = r.source;
|
|
904
912
|
heldBlocks.push(r);
|
|
905
913
|
continue;
|
|
906
914
|
}
|
|
@@ -912,7 +920,7 @@ async function runContentSync(options) {
|
|
|
912
920
|
const names = heldBlocks.map(r => `paragraph ${r.pos + 1}`);
|
|
913
921
|
refusal.held.push(...names);
|
|
914
922
|
result.heldBack = (result.heldBack || 0) + names.length;
|
|
915
|
-
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true
|
|
923
|
+
output.warn(describeHeld({ file: relPath, code, pairKey, pairConfig, names, handWritten: true }));
|
|
916
924
|
}
|
|
917
925
|
|
|
918
926
|
if (missed.length > 0) {
|
|
@@ -926,8 +934,8 @@ async function runContentSync(options) {
|
|
|
926
934
|
// Self-repair ladder (translateBlockBatchResilient): full
|
|
927
935
|
// batch → one missing-segments-only retry → (with a fallback
|
|
928
936
|
// method: one batch through it for every block the primary
|
|
929
|
-
// dropped or damaged — lib/fallback.js) →
|
|
930
|
-
//
|
|
937
|
+
// dropped or damaged — lib/fallback.js) → the source text,
|
|
938
|
+
// unmarked, for anything still missing. A
|
|
931
939
|
// duplicate/unknown marker (untrustworthy mapping) or an
|
|
932
940
|
// empty first response still fails the file whole when no
|
|
933
941
|
// fallback rescues it.
|
|
@@ -937,7 +945,6 @@ async function runContentSync(options) {
|
|
|
937
945
|
missed,
|
|
938
946
|
blocks,
|
|
939
947
|
pairConfig,
|
|
940
|
-
fallbackPrefix,
|
|
941
948
|
budget: fallbackBudget,
|
|
942
949
|
sharedOutputs: sharedOutputsFor ? sharedOutputsFor(code) : null,
|
|
943
950
|
label: `content:${relPath}`,
|
|
@@ -949,7 +956,6 @@ async function runContentSync(options) {
|
|
|
949
956
|
cwd,
|
|
950
957
|
pairConfig: cfg,
|
|
951
958
|
}),
|
|
952
|
-
fallbackPrefix,
|
|
953
959
|
}),
|
|
954
960
|
fallbackOnly: new Set(missed.flatMap((r, i) => (r.fallbackOnly ? [i] : []))),
|
|
955
961
|
});
|
|
@@ -986,7 +992,7 @@ async function runContentSync(options) {
|
|
|
986
992
|
const refusedByGate = batchOutcome.refused || [];
|
|
987
993
|
if (refusedByGate.length > 0) {
|
|
988
994
|
output.warn(
|
|
989
|
-
`Content sync body for ${code}: ${refusedByGate.length} block(s) of ${relPath} refused by the quality gate — `
|
|
995
|
+
`Content sync body for ${code}: ${refusedByGate.length} block(s) of ${relPath} refused by the quality gate, also when asked again with the reason — `
|
|
990
996
|
+ refusedByGate.slice(0, 3).map(r => `paragraph ${r.block}: ${r.reason}`).join('; ')
|
|
991
997
|
+ `${refusedByGate.length > 3 ? '; …' : ''}`
|
|
992
998
|
+ (pairConfig.fallback
|
|
@@ -997,7 +1003,7 @@ async function runContentSync(options) {
|
|
|
997
1003
|
}
|
|
998
1004
|
if (fellBack.length > 0) {
|
|
999
1005
|
bodyUsedFallback = true;
|
|
1000
|
-
output.raw(' [
|
|
1006
|
+
output.raw(' [PARTIAL]');
|
|
1001
1007
|
// Refused as a memorized sentence (said just above) is not
|
|
1002
1008
|
// "missing from the response".
|
|
1003
1009
|
const refusedShared = (batchOutcome.sharedOutput || []).length + refusedByGate.length;
|
|
@@ -1010,8 +1016,7 @@ async function runContentSync(options) {
|
|
|
1010
1016
|
: 'missing from the model response after a retry';
|
|
1011
1017
|
output.warn(
|
|
1012
1018
|
`Content sync body for ${code}: ${fellBack.length} of ${missed.length} ` +
|
|
1013
|
-
`block(s) ${why} —
|
|
1014
|
-
`'${fallbackPrefix}'-prefixed source. ` +
|
|
1019
|
+
`block(s) ${why} — left in the source language, unmarked (\`champollion status\` lists them). ` +
|
|
1015
1020
|
describeFallenBack({
|
|
1016
1021
|
file: relPath, pairKey, pairConfig, fellBack: fellBack.length, newlyHeld, refusedBy: batchOutcome.refusedBy,
|
|
1017
1022
|
})
|
|
@@ -1119,9 +1124,11 @@ async function runContentSync(options) {
|
|
|
1119
1124
|
// A fallback body keeps its OLD manifest entry so the file re-fires
|
|
1120
1125
|
// next sync — every good block is a TM hit, only the fallen-back
|
|
1121
1126
|
// segment re-bills. Self-healing at bounded cost.
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1127
|
+
// Something left in the source language: `pending:<hash>` — the page
|
|
1128
|
+
// is processed again next sync (cache-free; held units not re-sent)
|
|
1129
|
+
// and stays the tool's, not taken for a person's file.
|
|
1130
|
+
updatedManifest[manifestKey] = !bodyUsedFallback && !fieldsLeftInSource
|
|
1131
|
+
? currentSourceHash : pendingLockValue(currentSourceHash);
|
|
1125
1132
|
|
|
1126
1133
|
return result;
|
|
1127
1134
|
};
|
|
@@ -1466,10 +1473,11 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
|
|
|
1466
1473
|
* persona). Local file reads only; never fails (a diagnostic).
|
|
1467
1474
|
*
|
|
1468
1475
|
* translated — the target exists and was made from the current source
|
|
1469
|
-
* (the content lock's hash matches)
|
|
1476
|
+
* (the content lock's hash matches) and nothing left untranslated
|
|
1470
1477
|
* outOfDate — the target was made from an older source text
|
|
1471
|
-
* pending — no target yet, or
|
|
1472
|
-
*
|
|
1478
|
+
* pending — no target yet, or parts the quality gate refused left in
|
|
1479
|
+
* the source language (the lock says `pending:<hash>`, or a
|
|
1480
|
+
* legacy '[EN] ' marker from before 2026-10-05 is in the file)
|
|
1473
1481
|
* unrecorded — a target the lock has no record of (made by hand or by
|
|
1474
1482
|
* another tool): sync keeps it as it is
|
|
1475
1483
|
*
|
|
@@ -1477,7 +1485,7 @@ function countPendingContentTranslations(contentDir, sourceLocale, pairEntries,
|
|
|
1477
1485
|
* @param {string} sourceLocale
|
|
1478
1486
|
* @param {Array<[string, object]>} pairEntries
|
|
1479
1487
|
* @param {string} cwd
|
|
1480
|
-
* @param {{ fallbackPrefix?: string }} [opts]
|
|
1488
|
+
* @param {{ fallbackPrefix?: string }} [opts] - the legacy marker to look for
|
|
1481
1489
|
* @returns {{ dir: string, files: number, locales: Object<string, { translated: number,
|
|
1482
1490
|
* outOfDate: string[], pending: string[], unrecorded: string[] }> }|null}
|
|
1483
1491
|
*/
|
|
@@ -1502,7 +1510,7 @@ function contentStatus(contentDir, sourceLocale, pairEntries, cwd, { fallbackPre
|
|
|
1502
1510
|
let text = '';
|
|
1503
1511
|
try { text = fs.readFileSync(targetPath, 'utf-8'); } catch { /* unreadable: by its record only */ }
|
|
1504
1512
|
const stored = manifest[`${path.relative(contentDir, sourcePath)}:${code}`];
|
|
1505
|
-
if (text.includes(fallbackPrefix)) state.pending.push(relPath);
|
|
1513
|
+
if (isPendingLock(stored) || text.includes(fallbackPrefix)) state.pending.push(relPath);
|
|
1506
1514
|
else if (!stored) state.unrecorded.push(relPath);
|
|
1507
1515
|
else if (stored !== hash) state.outOfDate.push(relPath);
|
|
1508
1516
|
else state.translated += 1;
|
package/lib/cost-report.js
CHANGED
|
@@ -40,7 +40,7 @@ import { diffLocale } from './diff.js';
|
|
|
40
40
|
import { readLocaleFile } from './format.js';
|
|
41
41
|
import { expectedForTarget, keysForNamespace, readLocaleFlat } from './locale-layout.js';
|
|
42
42
|
import { mapSourceKeysToTarget } from './plurals.js';
|
|
43
|
-
import {
|
|
43
|
+
import { contentKeyUnits } from './config.js';
|
|
44
44
|
import { formatPerMillion } from './methods/openrouter-pricing.js';
|
|
45
45
|
import { countPendingContentTranslations } from './content-sync.js';
|
|
46
46
|
import { loadTM, lookupTM, partitionByTM, tmMethodKey, findModelSwitchStrandedEntries, reusableFromEarlierModels, carriedFromModel } from './tm.js';
|
|
@@ -382,9 +382,9 @@ export function abortForMaxCost(maxCost, estimatedCost, reason) {
|
|
|
382
382
|
|
|
383
383
|
/**
|
|
384
384
|
* Price per-pair content characters through each pair's own estimateCost().
|
|
385
|
-
* Characters become
|
|
386
|
-
* char-priced providers; conservative for token-priced
|
|
387
|
-
*
|
|
385
|
+
* Characters become key-equivalents (lib/config.js contentKeyUnits): exact
|
|
386
|
+
* for char-priced providers; measured, still conservative, for token-priced
|
|
387
|
+
* LLMs, which pay the prompt once per page rather than per 25-char slice. A pair with 0 chars is a
|
|
388
388
|
* KNOWN $0 and never consults estimateCost — an unknown-pricing null there
|
|
389
389
|
* would wrongly abort a free run under --max-cost.
|
|
390
390
|
*
|
|
@@ -405,7 +405,7 @@ export async function priceContentChars(charsByPair, pairEntries, estimateCost)
|
|
|
405
405
|
const chars = charsByPair.get(pairConfig);
|
|
406
406
|
if (!chars) continue;
|
|
407
407
|
// eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
|
|
408
|
-
const estimate = await estimateCost(
|
|
408
|
+
const estimate = await estimateCost(contentKeyUnits(chars, pairConfig), pairConfig);
|
|
409
409
|
if (estimate.estimatedCost !== null) {
|
|
410
410
|
cost += estimate.estimatedCost;
|
|
411
411
|
if (estimate.rate) rates.push(estimate.rate);
|