champollion 0.3.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +82 -37
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1709 -101
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +230 -17
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +284 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1136 -345
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +727 -79
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +975 -262
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +1102 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +575 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +257 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +71 -129
- package/lib/methods/llm.js +87 -33
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +102 -55
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +24 -8
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +996 -21
- package/lib/verify.js +1332 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/licenses.json +99 -3
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/retired-model-aliases.json +9 -0
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/human-services.schema.json +1 -1
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/schemas/retired-model-aliases.schema.json +32 -0
- package/shared/docent/corpus.json +0 -11739
- package/shared/model-aliases.json +0 -7
- package/shared/schemas/model-aliases.schema.json +0 -27
|
@@ -189,7 +189,7 @@ export class CommercialRouteBlockedError extends Error {
|
|
|
189
189
|
+ `${verdict.reason}`
|
|
190
190
|
+ (verdict.license ? ` (license: ${verdict.license})` : '')
|
|
191
191
|
+ '. Route it in the non-commercial lane, or use a method whose license '
|
|
192
|
-
+ 'permits commercial use (`champollion recommend <src> <tgt> '
|
|
192
|
+
+ 'permits commercial use (`champollion network recommend <src> <tgt> '
|
|
193
193
|
+ '--use commercial` lists them).'
|
|
194
194
|
);
|
|
195
195
|
this.name = 'CommercialRouteBlockedError';
|
package/lib/config.js
CHANGED
|
@@ -14,8 +14,12 @@
|
|
|
14
14
|
import fs from 'node:fs';
|
|
15
15
|
import path from 'node:path';
|
|
16
16
|
import { DEFAULT_REGISTERS, getLanguageCard, getRegister, resolveCode } from './registers.js';
|
|
17
|
-
import {
|
|
17
|
+
import { requireExactModelId } from './models.js';
|
|
18
18
|
import { validateNoTranslateConfig } from './no-translate.js';
|
|
19
|
+
import { output } from './output.js';
|
|
20
|
+
import { splitKeyList } from './redo.js';
|
|
21
|
+
import { LOCALE_FILE_FORMATS } from './format.js';
|
|
22
|
+
import { discoverLocaleLayout, compileLocalesPattern, LAYOUT_KINDS, FORMAT_BY_EXT } from './locale-layout.js';
|
|
19
23
|
|
|
20
24
|
const CONFIG_FILENAMES = ['champollion.config.json'];
|
|
21
25
|
|
|
@@ -61,19 +65,63 @@ const DEFAULT_METHOD_CONCURRENCY = 4;
|
|
|
61
65
|
const EST_INPUT_TOKENS_PER_KEY = 200;
|
|
62
66
|
const EST_OUTPUT_TOKENS_PER_KEY = 30;
|
|
63
67
|
const EST_CHARS_PER_KEY = 25;
|
|
68
|
+
// Markdown content priced by a token-billed method (an LLM): one key-unit per
|
|
69
|
+
// 60 source characters, not 25. A UI key carries its share of the prompt on
|
|
70
|
+
// every 25-char slice; a content batch carries the prompt once per page and
|
|
71
|
+
// the text itself is ~4 characters a token. Measured on champollion.dev with
|
|
72
|
+
// google/gemini-3.8-flash (2026-10-05): at 25 the estimate ran 2.8x over the
|
|
73
|
+
// bill for 95 Spanish pages and 8.2x over for one Arabic page; at 60 it is
|
|
74
|
+
// ~1.2x and ~3.4x — still above the bill, as the --max-cost cap needs.
|
|
75
|
+
// Character-billed engines (DeepL, Google, Microsoft) keep 25: for them the
|
|
76
|
+
// key-unit converts back to the exact characters they bill.
|
|
77
|
+
const EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED = 60;
|
|
78
|
+
const TOKEN_PRICED_METHODS = new Set(['llm', 'llm-coached', 'openai', 'anthropic', 'gemini', 'local']);
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Key-units for a run of Markdown source characters, priced by `pairConfig`'s
|
|
82
|
+
* method (see EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED).
|
|
83
|
+
*
|
|
84
|
+
* @param {number} chars
|
|
85
|
+
* @param {object} [pairConfig]
|
|
86
|
+
* @returns {number}
|
|
87
|
+
*/
|
|
88
|
+
function contentKeyUnits(chars, pairConfig = {}) {
|
|
89
|
+
const perKey = TOKEN_PRICED_METHODS.has(pairConfig?.method || 'llm')
|
|
90
|
+
? EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED : EST_CHARS_PER_KEY;
|
|
91
|
+
return Math.ceil(chars / perKey);
|
|
92
|
+
}
|
|
64
93
|
|
|
65
94
|
const DEFAULTS = {
|
|
66
95
|
version: 3,
|
|
67
96
|
inputLocale: 'en',
|
|
68
97
|
baseUrl: '',
|
|
69
98
|
localesDir: './locales',
|
|
70
|
-
|
|
99
|
+
// Where every locale's key-value files live when the default shapes do
|
|
100
|
+
// not fit (lib/locale-layout.js): a path with {lang} and optional {ns},
|
|
101
|
+
// e.g. "public/locales/{lang}/{ns}.json". Relative to the project root.
|
|
102
|
+
// When set, it fixes localesDir (the part before the first placeholder).
|
|
103
|
+
localesPattern: null,
|
|
104
|
+
// Force the layout auto-detection's answer: 'flat' (<dir>/<lang>.json)
|
|
105
|
+
// or 'dir' (<dir>/<lang>/<ns>.json). null = detect from the source files.
|
|
106
|
+
localesLayout: null,
|
|
107
|
+
// Namespace `wrap` adds extracted keys to in a folder-per-locale project
|
|
108
|
+
// with several namespace files (e.g. "common"). null = the only file.
|
|
109
|
+
defaultNamespace: null,
|
|
110
|
+
contentDir: null, // Folder of Markdown/MDX to translate (e.g. './content', './newsletters'). null = disabled.
|
|
71
111
|
// Markdown body translation granularity: 'block' (default — segment the
|
|
72
112
|
// body, TM-cache per block, one batched API call for the misses) or
|
|
73
113
|
// 'page' (single whole-body prompt, still TM-threaded). Overridable per
|
|
74
114
|
// pair. Validated in docusaurus-sync.js — anything else fails loud.
|
|
75
115
|
contentSegmentation: 'block',
|
|
76
116
|
promptContext: null, // Global context injected into all translation prompts (e.g. "This is a developer tool README")
|
|
117
|
+
// Gender guidance in LLM prompts: null = each language's guidance from the
|
|
118
|
+
// catalogue (French: écriture inclusive, "Connecté·e"); false = none; a
|
|
119
|
+
// string = your own instruction. Overridable per language and per pair.
|
|
120
|
+
genderGuidance: null,
|
|
121
|
+
// Names kept exactly as written in every language (people, companies,
|
|
122
|
+
// products). Told to the model, and exempt from the wrong-script /
|
|
123
|
+
// untranslated checks. Value-level, unlike noTranslate (which skips KEYS).
|
|
124
|
+
protectedTerms: [],
|
|
77
125
|
translatableFields: null, // Override DEFAULT_TRANSLATABLE_FIELDS from content.js
|
|
78
126
|
languages: [],
|
|
79
127
|
// Keys whose correct translation is the source value, verbatim: dot-paths
|
|
@@ -89,6 +137,11 @@ const DEFAULTS = {
|
|
|
89
137
|
pairs: null, // Advanced per-pair overrides (see pairs.js)
|
|
90
138
|
model: DEFAULT_OPENROUTER_MODEL,
|
|
91
139
|
defaultMethod: 'llm', // Global default: llm, llm-coached, google-translate, api, deepl, microsoft-translator, libretranslate, openai, anthropic, gemini
|
|
140
|
+
// Transport for the LLM lane (llm / llm-coached pairs): openrouter (the
|
|
141
|
+
// default when null), openai, anthropic, gemini, local. Harness
|
|
142
|
+
// `export-config` emits it for runs validated on a direct provider.
|
|
143
|
+
// Overridable per language and per pair; validated in pairs.js.
|
|
144
|
+
provider: null,
|
|
92
145
|
batchSize: DEFAULT_BATCH_SIZE,
|
|
93
146
|
temperature: null, // null = use method default (0.3 standard, 0.2 coached)
|
|
94
147
|
coachingFile: null, // Path to free-text coaching prompt file (relative to cwd)
|
|
@@ -190,11 +243,12 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
|
|
|
190
243
|
batch: 'batchSize',
|
|
191
244
|
key: 'apiKeyEnvVar',
|
|
192
245
|
apiKey: 'apiKeyEnvVar',
|
|
193
|
-
provider: 'defaultMethod',
|
|
194
246
|
concurrency: 'concurrency', // valid — not in DEFAULTS but consumed by sync
|
|
195
247
|
skipKeys: 'noTranslate', // canonicalized above; alias kept for the hint
|
|
196
248
|
noTranslateURLs: 'noTranslateUrls',
|
|
197
249
|
neverTranslate: 'noTranslate',
|
|
250
|
+
doNotTranslate: 'protectedTerms',
|
|
251
|
+
keepVerbatim: 'protectedTerms',
|
|
198
252
|
};
|
|
199
253
|
|
|
200
254
|
// Warn on unknown config fields — prevents silent acceptance of
|
|
@@ -208,20 +262,36 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
|
|
|
208
262
|
if (!knownFields.has(key)) {
|
|
209
263
|
const suggestion = FIELD_ALIASES[key];
|
|
210
264
|
if (suggestion) {
|
|
211
|
-
|
|
265
|
+
output.warn(`Unknown config field "${key}" — did you mean "${suggestion}"?`);
|
|
212
266
|
} else {
|
|
213
|
-
|
|
267
|
+
output.warn(`Unknown config field "${key}" in ${path.basename(configPath)} — this field has no effect. Check spelling or see docs for supported fields.`);
|
|
214
268
|
}
|
|
215
269
|
}
|
|
216
270
|
}
|
|
217
271
|
|
|
218
272
|
Object.assign(config, fileConfig);
|
|
273
|
+
// localesPattern fixes localesDir; a separately WRITTEN localesDir
|
|
274
|
+
// that disagrees with it is checked below, once paths are resolved.
|
|
275
|
+
config._localesDirExplicit = fileConfig.localesDir != null;
|
|
276
|
+
// Whether the top-level model was WRITTEN (vs. the built-in OpenRouter
|
|
277
|
+
// default). pairs.js needs the difference: a top-level `model` is sent
|
|
278
|
+
// to a top-level direct `provider` only when someone named it — the
|
|
279
|
+
// OpenRouter default slug must never reach OpenAI or Ollama.
|
|
280
|
+
config._modelExplicit = fileConfig.model != null;
|
|
219
281
|
|
|
220
282
|
// Validate the no-translate fields against the file that set them, so
|
|
221
283
|
// the error names the right place. A bad value must never fall through
|
|
222
284
|
// to "translate everything" — that is the corruption path the feature
|
|
223
285
|
// exists to close.
|
|
224
286
|
validateNoTranslateConfig(config.noTranslate, config.noTranslateUrls);
|
|
287
|
+
if (!Array.isArray(config.protectedTerms)
|
|
288
|
+
|| config.protectedTerms.some(t => typeof t !== 'string' || t.trim() === '')) {
|
|
289
|
+
const e = new Error(
|
|
290
|
+
'"protectedTerms" must be an array of non-empty strings — the names to keep as written, '
|
|
291
|
+
+ 'e.g. "protectedTerms": ["Curtis Forbes", "Game Day Suits"].');
|
|
292
|
+
e.code = 'CHAMPOLLION_CONFIG_INVALID';
|
|
293
|
+
throw e;
|
|
294
|
+
}
|
|
225
295
|
} catch (err) {
|
|
226
296
|
// A field-level validation error is already specific and actionable —
|
|
227
297
|
// wrapping it in "Could not parse" would misreport valid JSON as a
|
|
@@ -253,9 +323,49 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
|
|
|
253
323
|
|
|
254
324
|
// Layer 3: CLI overrides
|
|
255
325
|
if (cliArgs.source) config.inputLocale = cliArgs.source;
|
|
256
|
-
if (cliArgs.dir)
|
|
257
|
-
|
|
258
|
-
|
|
326
|
+
if (cliArgs.dir) {
|
|
327
|
+
config.localesDir = cliArgs.dir;
|
|
328
|
+
config._localesDirExplicit = true;
|
|
329
|
+
}
|
|
330
|
+
// A pair's `fallback` is NOT touched by --model / --method (they override
|
|
331
|
+
// the pair's own method only), so pairs.js resolves fallbacks against the
|
|
332
|
+
// model and default method the FILE set. Recorded before the flags apply.
|
|
333
|
+
config._fileModel = config.model;
|
|
334
|
+
config._fileModelExplicit = !!config._modelExplicit;
|
|
335
|
+
config._fileDefaultMethod = config.defaultMethod;
|
|
336
|
+
if (cliArgs.model) {
|
|
337
|
+
config.model = cliArgs.model;
|
|
338
|
+
config._modelExplicit = true;
|
|
339
|
+
// Like --method: the flag beats a pair's own model for this run (how you
|
|
340
|
+
// try or price a switch before committing to it), and says so.
|
|
341
|
+
config._modelOverride = cliArgs.model;
|
|
342
|
+
const pinned = Object.entries(config.pairs || {})
|
|
343
|
+
.filter(([, p]) => p && p.model && p.model !== cliArgs.model)
|
|
344
|
+
.map(([key, p]) => `${key} (${p.model})`);
|
|
345
|
+
if (pinned.length > 0) {
|
|
346
|
+
output.info(`--model ${cliArgs.model} overrides the configured model for this run: ${pinned.join(', ')}. Use --pair to limit it.`);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
if (cliArgs.method) {
|
|
350
|
+
config.defaultMethod = cliArgs.method;
|
|
351
|
+
// A flag beats the file for this run — including a pair's own `method`.
|
|
352
|
+
// It used to set only the default, so `sync --method local` (what the
|
|
353
|
+
// forge export's DEPLOY.md says to run) silently kept a pair configured
|
|
354
|
+
// for llm. Say which pairs it overrides; --pair scopes it.
|
|
355
|
+
config._methodOverride = cliArgs.method;
|
|
356
|
+
const overridden = [];
|
|
357
|
+
for (const [key, p] of Object.entries(config.pairs || {})) {
|
|
358
|
+
if (p && p.method && p.method !== cliArgs.method) overridden.push(`${key} (${p.method})`);
|
|
359
|
+
}
|
|
360
|
+
if (config.languages && !Array.isArray(config.languages)) {
|
|
361
|
+
for (const [code, l] of Object.entries(config.languages)) {
|
|
362
|
+
if (l && l.method && l.method !== cliArgs.method) overridden.push(`${code} (${l.method})`);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
if (overridden.length > 0) {
|
|
366
|
+
output.info(`--method ${cliArgs.method} overrides the configured method for this run: ${overridden.join(', ')}. Use --pair to limit it.`);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
259
369
|
// --batch-size: keys per translation API call. Accept both the CLI flag
|
|
260
370
|
// spelling ('batch-size' from bin/cli.js parseArgs) and the programmatic
|
|
261
371
|
// camelCase form (watch mode / tests pass cliArgs objects directly).
|
|
@@ -290,10 +400,9 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
|
|
|
290
400
|
config.contentConcurrency = parseConcurrency(cliArgs['content-concurrency'], '--content-concurrency');
|
|
291
401
|
}
|
|
292
402
|
|
|
293
|
-
// Parse --force-keys: comma-separated
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
: [];
|
|
403
|
+
// Parse --force-keys: comma-separated keys to force re-translate (`\,` is a
|
|
404
|
+
// comma inside a key — gettext msgids are sentences)
|
|
405
|
+
config.forceKeys = cliArgs['force-keys'] ? splitKeyList(cliArgs['force-keys']) : [];
|
|
297
406
|
|
|
298
407
|
// Docusaurus auto-detection: if format is still 'auto' and docusaurus.config.js
|
|
299
408
|
// exists in the project root, switch to 'docusaurus' mode and use the standard
|
|
@@ -308,14 +417,66 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
|
|
|
308
417
|
}
|
|
309
418
|
}
|
|
310
419
|
|
|
420
|
+
// `format` must name something a reader exists for. An unknown value used
|
|
421
|
+
// to fall through every `format === ...` branch and parse the files as
|
|
422
|
+
// JSON — "jsn" or "yml" produced confusing parse errors (or, for TOML,
|
|
423
|
+
// garbage) instead of a clear answer.
|
|
424
|
+
const validFormats = ['auto', 'docusaurus', ...LOCALE_FILE_FORMATS];
|
|
425
|
+
if (!validFormats.includes(config.format)) {
|
|
426
|
+
const hint = config.format === 'yml' ? ' (use "yaml" — .yml files are read as YAML)' : '';
|
|
427
|
+
const e = new Error(
|
|
428
|
+
`Unknown "format": ${JSON.stringify(config.format)}${hint}. `
|
|
429
|
+
+ `Supported: ${validFormats.join(', ')}.`);
|
|
430
|
+
e.code = 'CHAMPOLLION_CONFIG_INVALID';
|
|
431
|
+
throw e;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
if (config.localesLayout != null && !LAYOUT_KINDS.includes(config.localesLayout)) {
|
|
435
|
+
const e = new Error(
|
|
436
|
+
`"localesLayout" must be one of ${LAYOUT_KINDS.join(', ')} (got ${JSON.stringify(config.localesLayout)}). `
|
|
437
|
+
+ 'For any other shape, set "localesPattern", e.g. "public/locales/{lang}/{ns}.json".');
|
|
438
|
+
e.code = 'CHAMPOLLION_CONFIG_INVALID';
|
|
439
|
+
throw e;
|
|
440
|
+
}
|
|
441
|
+
|
|
311
442
|
// Resolve localesDir and contentDir to absolute paths
|
|
312
443
|
config.localesDir = path.resolve(cwd, config.localesDir);
|
|
313
444
|
if (config.contentDir) {
|
|
314
445
|
config.contentDir = path.resolve(cwd, config.contentDir);
|
|
315
446
|
}
|
|
316
447
|
|
|
317
|
-
//
|
|
318
|
-
|
|
448
|
+
// localesPattern: absolute, validated now (a malformed pattern must stop
|
|
449
|
+
// every command, not surface as "0 files" mid-sync), and the source of
|
|
450
|
+
// truth for localesDir — every path-containment check uses localesDir.
|
|
451
|
+
if (config.localesPattern != null) {
|
|
452
|
+
if (config.format === 'docusaurus') {
|
|
453
|
+
const e = new Error('"localesPattern" does not apply to Docusaurus projects — their locale files live in i18n/<locale>/.');
|
|
454
|
+
e.code = 'CHAMPOLLION_CONFIG_INVALID';
|
|
455
|
+
throw e;
|
|
456
|
+
}
|
|
457
|
+
if (config.localesLayout != null) {
|
|
458
|
+
const e = new Error('Set either "localesPattern" or "localesLayout", not both — a pattern already fixes the layout.');
|
|
459
|
+
e.code = 'CHAMPOLLION_CONFIG_INVALID';
|
|
460
|
+
throw e;
|
|
461
|
+
}
|
|
462
|
+
const compiled = compileLocalesPattern(path.resolve(cwd, String(config.localesPattern)));
|
|
463
|
+
if (config._localesDirExplicit && path.resolve(config.localesDir) !== compiled.base) {
|
|
464
|
+
const e = new Error(
|
|
465
|
+
`"localesDir" (${config.localesDir}) disagrees with "localesPattern" (files under ${compiled.base}). `
|
|
466
|
+
+ 'The pattern already says where locale files live — remove "localesDir".');
|
|
467
|
+
e.code = 'CHAMPOLLION_CONFIG_INVALID';
|
|
468
|
+
throw e;
|
|
469
|
+
}
|
|
470
|
+
config.localesPattern = path.resolve(cwd, String(config.localesPattern));
|
|
471
|
+
config.localesDir = compiled.base;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// Exact model slugs only (founder ruling 2026-10-05): a retired alias
|
|
475
|
+
// ("gemini-flash") or a floating id ("~google/gemini-flash-latest") is
|
|
476
|
+
// refused here, naming the slug to write — it never resolves to a model.
|
|
477
|
+
// The file's own model is checked too: a pair's fallback runs on it.
|
|
478
|
+
requireExactModelId(config.model, { from: config._modelOverride ? 'from --model' : 'from the top-level "model"' });
|
|
479
|
+
if (config._modelOverride) requireExactModelId(config._fileModel, { from: 'from the top-level "model"' });
|
|
319
480
|
|
|
320
481
|
// Coaching file: read the file contents into coachingPrompt if coachingFile is set.
|
|
321
482
|
// This allows users to maintain coaching prompts as separate text files rather
|
|
@@ -407,10 +568,30 @@ function resolveLanguages(config) {
|
|
|
407
568
|
formalitySystem: card?.formality?.system || null,
|
|
408
569
|
...(value.method && { method: value.method }),
|
|
409
570
|
...(value.model && { model: value.model }),
|
|
571
|
+
...(value.provider != null && { provider: value.provider }),
|
|
410
572
|
...(value.batchSize && { batchSize: value.batchSize }),
|
|
411
573
|
...(value.maxRetries != null && { maxRetries: value.maxRetries }),
|
|
412
574
|
...(value.script && { script: value.script }),
|
|
413
575
|
...(value.scriptFallback && { scriptFallback: value.scriptFallback }),
|
|
576
|
+
// The remaining fields pairs.js reads from a language. They used
|
|
577
|
+
// to be dropped here, so `{ "method": "api", "endpoint": … }` in
|
|
578
|
+
// the object form ran with no endpoint, and the documented
|
|
579
|
+
// temperature / coachingFile / promptContext had no effect.
|
|
580
|
+
...(value.endpoint && { endpoint: value.endpoint }),
|
|
581
|
+
...(value.apiKey && { apiKey: value.apiKey }),
|
|
582
|
+
// An `api` endpoint's declared instruction capability (lib/methods/api.js).
|
|
583
|
+
...(typeof value.acceptsInstructions === 'boolean' && { acceptsInstructions: value.acceptsInstructions }),
|
|
584
|
+
...(value.temperature != null && { temperature: value.temperature }),
|
|
585
|
+
...(value.coachingFile != null && { coachingFile: value.coachingFile }),
|
|
586
|
+
...(value.coachingPrompt != null && { coachingPrompt: value.coachingPrompt }),
|
|
587
|
+
...(value.promptContext != null && { promptContext: value.promptContext }),
|
|
588
|
+
...(value.contentSegmentation != null && { contentSegmentation: value.contentSegmentation }),
|
|
589
|
+
// Gender guidance in the prompt: false = none, a string = your own
|
|
590
|
+
// (absent: the catalogue's for the language — lib/pairs.js).
|
|
591
|
+
...((value.genderGuidance === false || typeof value.genderGuidance === 'string') && { genderGuidance: value.genderGuidance }),
|
|
592
|
+
// A second method for what this language's own method cannot
|
|
593
|
+
// translate safely — resolved and validated in pairs.js.
|
|
594
|
+
...(value.fallback !== undefined && { fallback: value.fallback }),
|
|
414
595
|
};
|
|
415
596
|
}
|
|
416
597
|
}
|
|
@@ -422,7 +603,8 @@ function resolveLanguages(config) {
|
|
|
422
603
|
|
|
423
604
|
/**
|
|
424
605
|
* Auto-detect target languages by scanning the locales directory
|
|
425
|
-
* for locale files (
|
|
606
|
+
* for locale files (every extension the layout module reads — JSON, TOML,
|
|
607
|
+
* YAML, gettext .po, Flutter .arb) that aren't the source file.
|
|
426
608
|
*
|
|
427
609
|
* @param {import('./types.js').ChampollionConfig} config - Resolved config
|
|
428
610
|
* @returns {Object<string, import('./types.js').LanguageConfig & { filename: string }>} Map of locale code → language config with filename
|
|
@@ -431,10 +613,39 @@ function autoDetectLanguages(config) {
|
|
|
431
613
|
const detected = {};
|
|
432
614
|
const inputLocale = config.inputLocale || 'en';
|
|
433
615
|
|
|
616
|
+
// Folder-per-locale and pattern layouts: the locales are what the layout
|
|
617
|
+
// finds on disk — sub-folders of localesDir (stray files at its root are
|
|
618
|
+
// not locales) or every {lang} the pattern matches. Exact code match, so
|
|
619
|
+
// en-GB is a target when the source is en.
|
|
620
|
+
const sourceDirExists = (() => {
|
|
621
|
+
try { return fs.statSync(path.join(config.localesDir, inputLocale)).isDirectory(); } catch { return false; }
|
|
622
|
+
})();
|
|
623
|
+
if (config.format !== 'docusaurus'
|
|
624
|
+
&& (config.localesPattern || config.localesLayout === 'dir' || sourceDirExists)) {
|
|
625
|
+
const layout = discoverLocaleLayout(config);
|
|
626
|
+
if (layout.kind !== 'flat') {
|
|
627
|
+
for (const code of layout.listLocales()) {
|
|
628
|
+
const canonical = resolveCode(code);
|
|
629
|
+
const card = getLanguageCard(canonical);
|
|
630
|
+
detected[code] = {
|
|
631
|
+
name: card?.name || code,
|
|
632
|
+
register: getRegister(canonical),
|
|
633
|
+
registerPreset: card?.formality?.default || null,
|
|
634
|
+
dir: card?.dir || 'ltr',
|
|
635
|
+
formalitySystem: card?.formality?.system || null,
|
|
636
|
+
filename: layout.filesFor(code)[0]?.rel ?? code,
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
return detected;
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
434
643
|
if (!fs.existsSync(config.localesDir)) return detected;
|
|
435
644
|
|
|
436
|
-
//
|
|
437
|
-
|
|
645
|
+
// Locale file extensions — the layout module's own list (lib/locale-layout.js
|
|
646
|
+
// FORMAT_BY_EXT), so a format the reader learns is a format detection
|
|
647
|
+
// finds. A gettext template (.pot) is not a locale and is not listed there.
|
|
648
|
+
const LOCALE_EXTS = Object.keys(FORMAT_BY_EXT);
|
|
438
649
|
|
|
439
650
|
const files = fs.readdirSync(config.localesDir)
|
|
440
651
|
.filter(f => {
|
|
@@ -518,6 +729,8 @@ export {
|
|
|
518
729
|
EST_INPUT_TOKENS_PER_KEY,
|
|
519
730
|
EST_OUTPUT_TOKENS_PER_KEY,
|
|
520
731
|
EST_CHARS_PER_KEY,
|
|
732
|
+
EST_CONTENT_CHARS_PER_KEY_TOKEN_PRICED,
|
|
733
|
+
contentKeyUnits,
|
|
521
734
|
DEFAULT_MAX_RETRIES,
|
|
522
735
|
DEFAULT_METHOD_CONCURRENCY,
|
|
523
736
|
};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* content-estimate.js — what a content file will actually BILL.
|
|
3
|
+
*
|
|
4
|
+
* The cost gate (--max-cost) must price what the run will pay for, not the
|
|
5
|
+
* whole file: front-matter fields and body blocks the Translation Memory
|
|
6
|
+
* already holds are served at $0 by the sync. Pricing whole files made a
|
|
7
|
+
* $100 cap refuse runs whose real cost was ~$30, and operators raised the
|
|
8
|
+
* cap past the honest number to get work done (dogfood 2026-08-28, finding
|
|
9
|
+
* 1) — the guardrail stopped guarding.
|
|
10
|
+
*
|
|
11
|
+
* This walks the SAME ladder both content lanes run at sync time:
|
|
12
|
+
* 1. each front-matter field, cached on its own source text;
|
|
13
|
+
* 2. the whole body (a revert or lock-loss re-run is free);
|
|
14
|
+
* 3. in 'block' segmentation, each translatable block.
|
|
15
|
+
* One implementation for the Hugo and Docusaurus estimators, so the two can
|
|
16
|
+
* no longer disagree about what is billable.
|
|
17
|
+
*
|
|
18
|
+
* Pure lookups (lookupTM, never the evicting lookupTMValidated): an
|
|
19
|
+
* estimate must not change the cache it is estimating against.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { lookupTM } from './tm.js';
|
|
23
|
+
import { splitBlocks } from './segment.js';
|
|
24
|
+
import { protectBlocks, restoreBlocks } from './content.js';
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Restored source texts of a body's translatable blocks — the exact unit
|
|
28
|
+
* block-mode sync caches and bills.
|
|
29
|
+
*
|
|
30
|
+
* @param {string} body - Markdown body (front matter stripped)
|
|
31
|
+
* @returns {string[]}
|
|
32
|
+
*/
|
|
33
|
+
export function translatableBlockSources(body) {
|
|
34
|
+
const { protectedBody, blocks } = protectBlocks(body);
|
|
35
|
+
return splitBlocks(protectedBody)
|
|
36
|
+
.filter(seg => seg.type === 'translatable')
|
|
37
|
+
.map(seg => restoreBlocks(seg.text, blocks));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Billable vs total source characters for one (file × locale) work item.
|
|
42
|
+
*
|
|
43
|
+
* @param {object} args
|
|
44
|
+
* @param {object} args.tm - Loaded TM (carry-over settings apply)
|
|
45
|
+
* @param {string} args.code - Target locale
|
|
46
|
+
* @param {string} args.tmKey - tmMethodKey(pairConfig)
|
|
47
|
+
* @param {string|null} [args.fallbackTmKey] - tmMethodKey(pairConfig.fallback)
|
|
48
|
+
* when the pair has a fallback: the sync serves the fallback's cached text
|
|
49
|
+
* before asking the primary (lib/fallback.js), so it is free here too
|
|
50
|
+
* @param {Record<string, string>} args.fields - Front-matter fields to translate
|
|
51
|
+
* @param {string} args.body - Markdown body
|
|
52
|
+
* @param {'block'|'page'} args.segMode - Content segmentation mode
|
|
53
|
+
* @param {() => string[]} args.blockSources - Lazily computed block sources
|
|
54
|
+
* (callers cache per source file: segmentation is locale-independent)
|
|
55
|
+
* @param {ReturnType<import('./content-refusals.js').contentHolds>|null} [args.holds] -
|
|
56
|
+
* What the quality gate refused before (lib/content-refusals.js): a block
|
|
57
|
+
* or field the pair's method refused is not sent to it (held, or the
|
|
58
|
+
* fallback's — which this estimate does not price); a held page body in
|
|
59
|
+
* 'page' segmentation holds its whole page, so nothing of it is billed. A
|
|
60
|
+
* held front-matter field keeps its source text and the rest of the page
|
|
61
|
+
* is still sent (since 2026-10-05)
|
|
62
|
+
* @returns {{ billedChars: number, totalChars: number }}
|
|
63
|
+
*/
|
|
64
|
+
export function billableContentChars({ tm, code, tmKey, fallbackTmKey = null, fields, body, segMode, blockSources, holds = null }) {
|
|
65
|
+
let billedChars = 0;
|
|
66
|
+
let totalChars = 0;
|
|
67
|
+
let pageHeld = false;
|
|
68
|
+
const cached = (text) => lookupTM(tm, text, code, tmKey) !== null
|
|
69
|
+
|| (fallbackTmKey !== null && lookupTM(tm, text, code, fallbackTmKey) !== null);
|
|
70
|
+
|
|
71
|
+
for (const [field, text] of Object.entries(fields)) {
|
|
72
|
+
totalChars += text.length;
|
|
73
|
+
if (cached(text)) continue;
|
|
74
|
+
const hold = holds ? holds.field(field, text) : 'send';
|
|
75
|
+
if (hold === 'send') billedChars += text.length;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (body.trim()) {
|
|
79
|
+
totalChars += body.length;
|
|
80
|
+
if (!cached(body)) {
|
|
81
|
+
if (segMode === 'page') {
|
|
82
|
+
// A page translated whole and refused before: held (its page is
|
|
83
|
+
// not processed — nothing of it is billed), or the fallback's.
|
|
84
|
+
const hold = holds ? holds.page(body) : 'send';
|
|
85
|
+
if (hold === 'held') pageHeld = true;
|
|
86
|
+
else if (hold === 'send') billedChars += body.length;
|
|
87
|
+
} else {
|
|
88
|
+
for (const source of blockSources()) {
|
|
89
|
+
if (!cached(source) && (!holds || holds.block(source) === 'send')) billedChars += source.length;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return { billedChars: pageHeld ? 0 : billedChars, totalChars };
|
|
96
|
+
}
|