champollion 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -37
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +51 -3
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +289 -88
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +649 -130
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +16 -10
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +197 -38
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +193 -106
- package/lib/seal.mjs +6 -5
- package/lib/sealed-qualifier.mjs +2 -2
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +3 -2
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/DATA-SOVEREIGNTY.md +19 -20
- package/shared/LANGUAGE-CARD-FIELDS.md +1 -1
- package/shared/cards-fallback.json +1 -1
- package/shared/catalogue/card-config.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/docent/faq.en.json +14 -16
- package/shared/docent/system-prompt.md +17 -19
- package/shared/explainers/tc-features.json +15 -15
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/human-services.json +1 -1
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +20 -10
- package/shared/schemas/human-services.schema.json +2 -2
- package/shared/schemas/language-card.schema.json +1 -1
- package/shared/schemas/method-card.schema.json +1 -1
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11333
package/README.md
CHANGED
|
@@ -16,9 +16,11 @@ npx champollion sync
|
|
|
16
16
|
|
|
17
17
|
Champollion auto-detects your locale files, their format, and the target languages. It translates missing keys, skips what's already done, and writes the results. That's it.
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
>
|
|
19
|
+
**License:** source-available under the [PolyForm Noncommercial License 1.0.0](LICENSE) — free to use, change and share for noncommercial purposes (personal projects, research, teaching, and use by charities, schools and public bodies). Using it for a commercial purpose is not covered by this license. The evaluation harness is open source (AGPL-3.0-or-later). [Who may use this](https://champollion.dev/docs/getting-started/who-may-use-this) · [Details](#license).
|
|
20
|
+
|
|
21
|
+
> **Part of Champollion** — infrastructure for trustworthy machine-translation
|
|
22
|
+
> evaluation across every language, source-available and free for noncommercial
|
|
23
|
+
> use (the evaluation harness and shared registries are open source). This CLI is the
|
|
22
24
|
> deployment end of a larger
|
|
23
25
|
> project that builds the test sets and the map showing who can translate what,
|
|
24
26
|
> how good each method is on each kind of text, and where the gaps still are. It
|
|
@@ -26,8 +28,8 @@ Champollion auto-detects your locale files, their format, and the target languag
|
|
|
26
28
|
> every method welcome) and sovereign benchmarks — secret test sets that
|
|
27
29
|
> communities create, own, and control, and that we never see. The infrastructure
|
|
28
30
|
> is source-available and singly stewarded; the test sets and the methods for a
|
|
29
|
-
> community's language belong to that community.
|
|
30
|
-
>
|
|
31
|
+
> community's language belong to that community. Designed to work with communities,
|
|
32
|
+
> never hosting their corpora — the design puts the keys in their hands. Every method is welcome, human and
|
|
31
33
|
> machine. Explore the network at
|
|
32
34
|
> [champollion.dev/docs/network](https://champollion.dev/docs/network/).
|
|
33
35
|
|
|
@@ -37,7 +39,7 @@ You could write a quick script that loops through your English keys and calls Go
|
|
|
37
39
|
|
|
38
40
|
- **No change detection.** When you update an English string, the translation stays stale forever. Champollion tracks every source value with SHA-256 hashes and re-translates only what changed.
|
|
39
41
|
- **No batching.** One API call per key means 200 keys = 200 round trips. Champollion batches intelligently (configurable, default 80 keys/batch for LLM, 128 for Google).
|
|
40
|
-
- **No quality gate.** Machine translation hallucinates, echoes the source back, or outputs in the wrong script. Champollion
|
|
42
|
+
- **No quality gate.** Machine translation hallucinates, echoes the source back, or outputs in the wrong script. Champollion checks every translation before writing it — empty output, source echoes, repetition loops, length inflation, deleted content and wrong script are caught and rejected. The gate catches broken output, not wrong meaning.
|
|
41
43
|
- **No format awareness.** Hardcoded to JSON? Champollion handles JSON, TOML, YAML, and Hugo Markdown (frontmatter + body) with auto-detection.
|
|
42
44
|
- **No safety.** Champollion guards against prototype pollution, path traversal via crafted locale codes, and code block corruption during Markdown translation.
|
|
43
45
|
|
|
@@ -118,17 +120,17 @@ You handle the i18n framework (next-intl, i18next, Hugo). Champollion handles th
|
|
|
118
120
|
- **Multi-format** — JSON, TOML, YAML, Hugo Markdown (front matter + body), and XLIFF 1.2
|
|
119
121
|
- **Incremental** — Only translates what changed (SHA-256 hash tracking)
|
|
120
122
|
- **Cached** — Translation Memory stores previous results; re-running sync costs nothing for unchanged keys
|
|
121
|
-
- **Quality-gated** —
|
|
123
|
+
- **Quality-gated** — Checks every translation for broken output: repetition loops, wrong-script output, source echoes, length inflation and deleted content (it does not check meaning)
|
|
122
124
|
- **Content-aware** — LLM methods shield code blocks, shortcodes, links, and interpolation variables during Markdown translation
|
|
123
125
|
- **Pipeline tools** — `lint`, `audit`, `integrity`, `seo` for CI gates
|
|
124
126
|
- **XLIFF interop** — Export translations for professional review in CAT tools (memoQ, SDL Trados, Phrase), import them back
|
|
125
|
-
- **Minimal dependencies** — two runtime dependencies (
|
|
127
|
+
- **Minimal dependencies** — two runtime dependencies: `@translated/lara` (the Lara provider SDK) and `cree-sro-syllabics` (ALTLab/NRC's Plains Cree SRO ⇄ syllabics converter). Requires Node 20+
|
|
126
128
|
|
|
127
129
|
## Beyond Google Translate
|
|
128
130
|
|
|
129
131
|
The quick start gets you running with an LLM or Google Translate. But Google Translate supports ~130 languages. There are over 7,000.
|
|
130
132
|
|
|
131
|
-
**Champollion's core idea: the translation method is configurable per language pair.** Use Google Translate for French, an LLM with
|
|
133
|
+
**Champollion's core idea: the translation method is configurable per language pair.** Use Google Translate for French, an LLM coached with your own grammar notes and dictionary for Plains Cree, and an API you host for a language no service covers — all in the same project, all with the same CLI.
|
|
132
134
|
|
|
133
135
|
```json
|
|
134
136
|
{
|
|
@@ -136,7 +138,7 @@ The quick start gets you running with an LLM or Google Translate. But Google Tra
|
|
|
136
138
|
"pairs": {
|
|
137
139
|
"en:fr": { "method": "google-translate" },
|
|
138
140
|
"en:ja": { "method": "llm" },
|
|
139
|
-
"en:crk": { "
|
|
141
|
+
"en:crk": { "method": "llm-coached" }
|
|
140
142
|
}
|
|
141
143
|
}
|
|
142
144
|
```
|
|
@@ -160,6 +162,7 @@ Champollion supports 10 translation methods. Each language pair can use a differ
|
|
|
160
162
|
| `openai` | `OPENAI_API_KEY` | Direct OpenAI API (gpt-4o, gpt-4o-mini) |
|
|
161
163
|
| `anthropic` | `ANTHROPIC_API_KEY` | Direct Anthropic API (Claude Sonnet, Haiku, Opus) |
|
|
162
164
|
| `gemini` | `GEMINI_API_KEY` | Direct Google Gemini API (Flash, Pro) — free tier available |
|
|
165
|
+
| `local` | *(none)* | Your own model behind an OpenAI-compatible endpoint (Ollama, vLLM, LM Studio, a model trained with `nmt-forge`). Set `LOCAL_API_BASE` if it is not at Ollama's default. Text never leaves your infrastructure |
|
|
163
166
|
|
|
164
167
|
**Traditional MT** — best for speed, cost, and high-volume key-value pairs:
|
|
165
168
|
|
|
@@ -188,7 +191,7 @@ champollion sync --method deepl
|
|
|
188
191
|
"pairs": {
|
|
189
192
|
"en:fr": { "method": "deepl" },
|
|
190
193
|
"en:ja": { "method": "openai", "model": "gpt-4o" },
|
|
191
|
-
"en:crk": { "
|
|
194
|
+
"en:crk": { "method": "llm-coached" }
|
|
192
195
|
}
|
|
193
196
|
}
|
|
194
197
|
```
|
|
@@ -203,7 +206,7 @@ Plugins are pre-packaged translation recipes for specific language pairs. They'r
|
|
|
203
206
|
champollion plugin install ./french-formal-v1/ # install from directory
|
|
204
207
|
champollion plugin list # see installed plugins
|
|
205
208
|
champollion plugin remove french-formal-v1 # uninstall
|
|
206
|
-
champollion status # shows
|
|
209
|
+
champollion status # shows each pair's method, model + plugin benchmarks
|
|
207
210
|
```
|
|
208
211
|
|
|
209
212
|
See [the reference docs](https://champollion.dev/docs/reference/plugin-spec) for the manifest format.
|
|
@@ -217,12 +220,8 @@ See [the reference docs](https://champollion.dev/docs/reference/plugin-spec) for
|
|
|
217
220
|
| `serve` | Serve this project's translation stack over HTTP (api-method contract) |
|
|
218
221
|
| `watch` | Auto-sync on file changes |
|
|
219
222
|
| `audit` | Flag incomplete locales (CI gate) |
|
|
220
|
-
| `card` | Pretty-print a language card (`card <code>`, `--json` for raw) |
|
|
221
|
-
| `register-corpus` | Register an evaluation corpus: pick a license + exposure tier (local-only/private/public/sealed) |
|
|
222
|
-
| `seal-corpus` | Sealed-tier crypto verbs: `keygen` / `seal` / `open` (organizer-node bridge) |
|
|
223
|
-
| `submit` | Propose an index entry (review-gated) — prints a pre-filled GitHub issue |
|
|
224
223
|
| `lint` | Find hardcoded strings in source code |
|
|
225
|
-
| `status` | Show pair configuration, methods, registers, and
|
|
224
|
+
| `status` | Show pair configuration, methods, registers, and plugin benchmarks |
|
|
226
225
|
| `provenance` | Audit translation resource licensing |
|
|
227
226
|
| `wrap` | Auto-wrap hardcoded strings in `t()` calls (with undo) |
|
|
228
227
|
| `seo` | Generate hreflang, sitemap.xml, or JSON-LD schema |
|
|
@@ -233,10 +232,21 @@ See [the reference docs](https://champollion.dev/docs/reference/plugin-spec) for
|
|
|
233
232
|
| `xliff` | Export/import XLIFF 1.2 for professional translator review |
|
|
234
233
|
| `models` | List available models for a provider (`--method gemini`) |
|
|
235
234
|
| `verify` | Re-read written locale files and confirm translations are present and correct (CI gate) |
|
|
236
|
-
| `leaderboard` | Show the MT leaderboard (`--pair`, `--sort`, `--install N`) |
|
|
237
|
-
| `recommend` | Method guidance for a pair — availability + cited evidence (`--use`, `--json`) |
|
|
238
235
|
| `doctor` | System health check: cards, config, methods, and converters |
|
|
239
236
|
|
|
237
|
+
The commands that work with the shared index and leaderboard, rather than your
|
|
238
|
+
project, are grouped under `champollion network` (each also works without the
|
|
239
|
+
prefix):
|
|
240
|
+
|
|
241
|
+
| Command | Purpose |
|
|
242
|
+
|---------|---------|
|
|
243
|
+
| `network card` | Pretty-print a language card (`network card <code>`, `--json` for raw) |
|
|
244
|
+
| `network recommend` | Method guidance for a pair — availability + cited evidence (`network recommend eng fra`, `--use`, `--json`) |
|
|
245
|
+
| `network leaderboard` | Show the MT leaderboard (`--pair`, `--sort`, `--install N`) |
|
|
246
|
+
| `network register-corpus` | Register an evaluation corpus: pick a license + exposure tier (local-only/private/public/sealed) |
|
|
247
|
+
| `network seal-corpus` | Sealed-tier crypto verbs: `keygen` / `seal` / `open` (organizer-node bridge) |
|
|
248
|
+
| `network submit` | Propose an index entry (review-gated) — prints a pre-filled GitHub issue |
|
|
249
|
+
|
|
240
250
|
Run `champollion <command> --help` for detailed help on any command.
|
|
241
251
|
|
|
242
252
|
Full reference: [the reference docs](https://champollion.dev/docs/reference/cli)
|
|
@@ -247,7 +257,7 @@ Full reference: [the reference docs](https://champollion.dev/docs/reference/cli)
|
|
|
247
257
|
|
|
248
258
|
```bash
|
|
249
259
|
mkdir -p .githooks
|
|
250
|
-
printf '#!/bin/sh\nnpx champollion lint\n' > .githooks/pre-commit
|
|
260
|
+
printf '#!/bin/sh\nnpx --yes champollion@0.4 lint\n' > .githooks/pre-commit # pinned: a new release never changes what the hook runs
|
|
251
261
|
chmod +x .githooks/pre-commit
|
|
252
262
|
git config core.hooksPath .githooks # once per clone
|
|
253
263
|
```
|
|
@@ -284,8 +294,9 @@ Create `champollion.config.json` or run `champollion init`:
|
|
|
284
294
|
| Option | Default | Description |
|
|
285
295
|
|--------|---------|-------------|
|
|
286
296
|
| `inputLocale` | `"en"` | Source language code |
|
|
287
|
-
| `localesDir` | `"./locales"` | Path to locale files |
|
|
288
|
-
| `
|
|
297
|
+
| `localesDir` | `"./locales"` | Path to locale files: one file per language (`fr.json`) or one folder per language (`fr/common.json`, i18next) |
|
|
298
|
+
| `localesPattern` | `null` | Any other layout, e.g. `"src/i18n/{ns}/{lang}.json"` (replaces `localesDir`) |
|
|
299
|
+
| `contentDir` | `null` | Folder of Markdown/MDX to translate (a Hugo `content/` or any folder); each translation is written beside its source as `<name>.<locale>.md` |
|
|
289
300
|
| `format` | `"auto"` | File format: `json`, `toml`, `yaml`, or `auto` |
|
|
290
301
|
| `model` | `"google/gemini-3.5-flash"` | Default model (OpenRouter slug). Direct providers resolve their own default at runtime. Run `champollion models --method gemini` to discover available models. |
|
|
291
302
|
| `defaultMethod` | `"llm"` | Default translation method (overridden by `--method` flag) |
|
|
@@ -302,7 +313,7 @@ Create `champollion.config.json` or run `champollion init`:
|
|
|
302
313
|
"crk": {
|
|
303
314
|
"name": "Plains Cree",
|
|
304
315
|
"register": "SRO syllabics with grammatical precision.",
|
|
305
|
-
"model": "google/gemini-
|
|
316
|
+
"model": "google/gemini-3.1-pro-preview",
|
|
306
317
|
"batchSize": 5,
|
|
307
318
|
"maxRetries": 5,
|
|
308
319
|
"script": "Cans"
|
|
@@ -322,30 +333,34 @@ See the [CLI Reference](https://champollion.dev/docs/reference/cli) for framewor
|
|
|
322
333
|
When you run `sync`, champollion shows exactly what's happening:
|
|
323
334
|
|
|
324
335
|
```
|
|
325
|
-
champollion v0.
|
|
336
|
+
champollion v0.4.0
|
|
326
337
|
|
|
327
338
|
[INFO] Detected format: json (auto)
|
|
328
339
|
[INFO] Detected framework: Hugo
|
|
329
|
-
[INFO] Source: en.json (
|
|
340
|
+
[INFO] Source: en.json (2847 keys)
|
|
330
341
|
[INFO] Pairs: es-MX:llm, fr:deepl, it:llm
|
|
331
|
-
|
|
342
|
+
Estimated translation cost:
|
|
332
343
|
|
|
333
|
-
Pair
|
|
334
|
-
────────
|
|
335
|
-
en:es-MX
|
|
336
|
-
en:fr
|
|
337
|
-
en:it
|
|
344
|
+
Pair Method Keys TM hits Est. Cost
|
|
345
|
+
──────── ────── ────── ─────── ──────────
|
|
346
|
+
en:es-MX llm 2847 0 ~$0.3843
|
|
347
|
+
en:fr deepl 2847 0 ~$1.7794
|
|
348
|
+
en:it llm 2847 0 ~$0.3843
|
|
338
349
|
|
|
339
|
-
Total: ~$2.
|
|
350
|
+
Total: ~$2.5480
|
|
351
|
+
Rates: google/gemini-3.5-flash $0.30 input / $2.50 output per 1M tokens — OpenRouter's price list, read 2026-10-04 14:02 UTC; deepl $25.00 per 1M characters — the provider's published price, as checked 2026-06-08. An estimate: ~200 input + ~30 output tokens or ~25 characters per key assumed — the bill depends on the real lengths (--json has the detail).
|
|
340
352
|
|
|
341
|
-
[INFO]
|
|
353
|
+
[INFO] Translating 3 locale(s) with concurrency 50
|
|
354
|
+
[INFO] es-MX.json — 2847 missing
|
|
355
|
+
████████████████████████████████ 2,847/2,847 keys
|
|
356
|
+
[INFO] fr.json — 2847 missing
|
|
342
357
|
████████████████████████████████ 2,847/2,847 keys
|
|
343
|
-
[INFO]
|
|
358
|
+
[INFO] it.json — 2847 missing
|
|
344
359
|
████████████████████████████████ 2,847/2,847 keys
|
|
345
|
-
[OK] Synced
|
|
360
|
+
[OK] Synced 8541 keys total — 8541 key(s) sent to the model, 0 served from the cache (free).
|
|
346
361
|
```
|
|
347
362
|
|
|
348
|
-
The progress bar updates in-place as each batch completes (~80 keys per update).
|
|
363
|
+
The estimate's last line names the rate each figure was priced at, where it came from and when (`--json` carries the detail). "Translating …" is said only when something will be sent: a dry run, or a run with nothing to send, says it is checking the locales instead. The progress bar updates in-place as each batch completes (~80 keys per update). When `contentDir` is set, sync names the folder and where its translations go; it says `Detected framework: Hugo` only when it finds a real Hugo site (`hugo.toml`, a `config.toml` with Hugo settings, `archetypes/`, …). Format detection distinguishes `(auto)` from `(config)` to clarify how the format was resolved.
|
|
349
364
|
|
|
350
365
|
**Output modes**: `--quiet` suppresses informational output (errors and warnings only). `--json` emits machine-readable NDJSON for CI/CD pipelines.
|
|
351
366
|
|
|
@@ -384,4 +399,4 @@ npm run test:methods # translation method suite
|
|
|
384
399
|
|
|
385
400
|
## License
|
|
386
401
|
|
|
387
|
-
PolyForm Noncommercial 1.0.0. The Champollion CLI is source-available — free to install, use, and modify for noncommercial purposes (research, education, community work) under the [PolyForm Noncommercial License 1.0.0](LICENSE); commercial
|
|
402
|
+
PolyForm Noncommercial 1.0.0. The Champollion CLI is source-available — free to install, use, and modify for noncommercial purposes (research, education, community work) under the [PolyForm Noncommercial License 1.0.0](LICENSE); using it for a commercial purpose is not covered by this license. In plain words, with examples — a school, a public hospital or clinic, a charity, a personal or research project is covered; a for-profit business's product, such as a shop's storefront, is not: [Who may use this](https://champollion.dev/docs/getting-started/who-may-use-this) (a summary, not legal advice; the license text governs). The published `champollion` npm package is PolyForm-Noncommercial-1.0.0; `cli/LICENSE` is the authoritative license for the distributed package. The companion MT Eval Harness and specs are open source, licensed AGPL-3.0-or-later — with a §7 eval-standard-plugin exception — at the public [harness repository](https://github.com/gamedaysuits/Champollion).
|
package/bin/cli.js
CHANGED
|
@@ -34,10 +34,17 @@ const CLI_OPTIONS = {
|
|
|
34
34
|
version: { type: 'boolean', short: 'v' },
|
|
35
35
|
yes: { type: 'boolean', short: 'y' },
|
|
36
36
|
'warn-only': { type: 'boolean' },
|
|
37
|
+
strict: { type: 'boolean' }, // verify --strict: warnings fail the check (exit 1)
|
|
37
38
|
'no-verify': { type: 'boolean' }, // skip post-sync verification
|
|
38
39
|
undo: { type: 'boolean' },
|
|
39
40
|
'force-content': { type: 'boolean' }, // ignore the content lock (TM still serves unchanged bodies/blocks)
|
|
41
|
+
files: { type: 'string', multiple: true }, // sync: only these content files (glob, repeatable)
|
|
42
|
+
retranslate: { type: 'string', multiple: true }, // sync: translate these content files fresh (glob, repeatable)
|
|
43
|
+
redo: { type: 'string', multiple: true }, // sync: queue again — all | keys:<k> | content | files:<glob> (lib/redo.js)
|
|
44
|
+
fresh: { type: 'boolean' }, // sync: do not use the cache for what is queued (billed again)
|
|
45
|
+
prune: { type: 'string', multiple: true }, // sync: remove what the locale cannot use — plural-extras (i18next keys for forms the language does not have)
|
|
40
46
|
'no-tm': { type: 'boolean' }, // skip Translation Memory for this sync run
|
|
47
|
+
'fresh-on-model-change': { type: 'boolean' }, // don't reuse other models' TM entries
|
|
41
48
|
css: { type: 'boolean' }, // fonts install --css: generate CSS snippet
|
|
42
49
|
json: { type: 'boolean' }, // machine-readable NDJSON output
|
|
43
50
|
quiet: { type: 'boolean', short: 'q' }, // suppress info/ok messages, show only warnings/errors
|
|
@@ -49,12 +56,14 @@ const CLI_OPTIONS = {
|
|
|
49
56
|
'content-dir': { type: 'string' },
|
|
50
57
|
source: { type: 'string' },
|
|
51
58
|
langs: { type: 'string' },
|
|
59
|
+
script: { type: 'string' }, // init: writing system per target, e.g. crk=Cans (a language with two real orthographies)
|
|
52
60
|
model: { type: 'string' },
|
|
53
61
|
method: { type: 'string' },
|
|
54
62
|
format: { type: 'string' },
|
|
55
63
|
'base-url': { type: 'string' },
|
|
56
64
|
out: { type: 'string' },
|
|
57
65
|
src: { type: 'string' },
|
|
66
|
+
ignore: { type: 'string' }, // lint: comma-separated directory/file names to skip
|
|
58
67
|
'min-length': { type: 'string' },
|
|
59
68
|
'force-keys': { type: 'string' },
|
|
60
69
|
'list-keys': { type: 'boolean' }, // sync --dry: name every queued key per reason
|
|
@@ -63,7 +72,8 @@ const CLI_OPTIONS = {
|
|
|
63
72
|
'json-concurrency': { type: 'string' }, // max parallel API calls for JSON key-value translation (default: 50)
|
|
64
73
|
'content-concurrency': { type: 'string' }, // max parallel API calls for markdown content (default: 12)
|
|
65
74
|
'batch-size': { type: 'string' }, // keys per translation API call (positive integer; overrides config batchSize)
|
|
66
|
-
'max-cost': { type: 'string' },
|
|
75
|
+
'max-cost': { type: 'string' },
|
|
76
|
+
'show-prompt': { type: 'string' }, // sync --dry: print the exact request the method would get (optionally for one key) // sync: USD cap — abort (exit 2) before any API call if the estimate exceeds it or is unknown
|
|
67
77
|
temperature: { type: 'string' }, // sampling temperature for LLM methods
|
|
68
78
|
'coaching-file': { type: 'string' }, // path to coaching prompt text file
|
|
69
79
|
locale: { type: 'string' }, // target locale for xliff export, tm clear/seed
|
|
@@ -80,7 +90,7 @@ const CLI_OPTIONS = {
|
|
|
80
90
|
license: { type: 'string' }, // register-corpus: license key / SPDX id / list number
|
|
81
91
|
tier: { type: 'string' }, // register-corpus: exposure tier (local-only|private|public)
|
|
82
92
|
exposure: { type: 'string' }, // register-corpus: alias for --tier
|
|
83
|
-
name: { type: 'string' }, // register-corpus: corpus name
|
|
93
|
+
name: { type: 'string' }, // register-corpus: corpus name; init: display name of a language with no card (qaa="…")
|
|
84
94
|
publisher: { type: 'string' }, // register-corpus: publisher / author
|
|
85
95
|
description: { type: 'string' }, // register-corpus: one-line description
|
|
86
96
|
'source-lang': { type: 'string' }, // register-corpus: source language code (alt to --pair)
|
|
@@ -90,10 +100,12 @@ const CLI_OPTIONS = {
|
|
|
90
100
|
'license-url': { type: 'string' }, // register-corpus: upstream license URL
|
|
91
101
|
builder: { type: 'string' }, // register-corpus: builder adapter id
|
|
92
102
|
sha256: { type: 'string' }, // register-corpus: built-corpus hash (optional)
|
|
103
|
+
data: { type: 'string' }, // register-corpus: the local file the card describes (hashed, never uploaded)
|
|
93
104
|
contamination: { type: 'string' }, // register-corpus: NONE|LOW|MEDIUM|HIGH
|
|
94
105
|
domain: { type: 'string' }, // register-corpus: corpus domain
|
|
95
106
|
size: { type: 'string' }, // register-corpus: number of sentence pairs
|
|
96
107
|
id: { type: 'string' }, // register-corpus: override card id
|
|
108
|
+
role: { type: 'string' }, // register-corpus: test|dev|train — in the id only when stated
|
|
97
109
|
version: { type: 'string' }, // register-corpus: corpus version (major)
|
|
98
110
|
'do-not-train': { type: 'boolean' }, // register-corpus: benchmark-integrity flag (default true)
|
|
99
111
|
'cards-dir': { type: 'string' }, // register-corpus: corpora-cards dir override (advanced/testing)
|
|
@@ -133,7 +145,8 @@ const CLI_OPTIONS = {
|
|
|
133
145
|
'max-cost-per-request': { type: 'string' }, // serve: USD cap per request (unknown estimates refuse)
|
|
134
146
|
'max-session-cost': { type: 'string' }, // serve: USD spend ceiling for the server process
|
|
135
147
|
'emit-manifest': { type: 'boolean' }, // serve: write the consumer method.json and exit
|
|
136
|
-
endpoint: { type: 'string' }, // serve --emit-manifest: public endpoint URL
|
|
148
|
+
endpoint: { type: 'string' }, // serve --emit-manifest: public endpoint URL; init --method api: the champollion API endpoint
|
|
149
|
+
'accepts-instructions': { type: 'string' }, // init --method api: does the endpoint follow per-key instructions (true|false)
|
|
137
150
|
};
|
|
138
151
|
|
|
139
152
|
const { values, positionals } = parseArgs({
|
|
@@ -169,6 +182,17 @@ const args = { _: positionals, ...values };
|
|
|
169
182
|
// --dry-run is an alias for --dry — merge so command modules only check args.dry
|
|
170
183
|
if (args['dry-run']) args.dry = true;
|
|
171
184
|
|
|
185
|
+
// --redo / --fresh are the one way to say "translate this again"; they are
|
|
186
|
+
// rewritten into the older flags here so sync keeps one set of semantics.
|
|
187
|
+
if (args.redo !== undefined || args.fresh) {
|
|
188
|
+
const { applyRedo } = await import('../lib/redo.js');
|
|
189
|
+
const redoError = applyRedo(args);
|
|
190
|
+
if (redoError) {
|
|
191
|
+
console.error(`[ERR] ${String(redoError).replace(/\u0004/g, '\u2404')}`);
|
|
192
|
+
process.exit(1);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
172
196
|
/**
|
|
173
197
|
* Exit only once stdout has actually been written.
|
|
174
198
|
*
|
|
@@ -187,7 +211,24 @@ function exitWhenFlushed(code) {
|
|
|
187
211
|
process.stdout.write('', () => process.exit(code));
|
|
188
212
|
}
|
|
189
213
|
|
|
190
|
-
|
|
214
|
+
// `champollion network <command>` — the commands that work with the shared
|
|
215
|
+
// index and leaderboard rather than this project. Each also works without
|
|
216
|
+
// the prefix (the names people already use).
|
|
217
|
+
// (scripts/check_doc_commands.py reads COMMAND_GROUPS to check the docs.)
|
|
218
|
+
const COMMAND_GROUPS = {
|
|
219
|
+
network: ['card', 'recommend', 'leaderboard', 'register-corpus', 'seal-corpus', 'submit'],
|
|
220
|
+
};
|
|
221
|
+
const NETWORK_COMMANDS = COMMAND_GROUPS.network;
|
|
222
|
+
let command = args._[0] || 'help';
|
|
223
|
+
if (command === 'network' && args._[1]) {
|
|
224
|
+
if (!NETWORK_COMMANDS.includes(args._[1])) {
|
|
225
|
+
console.error(`[ERR] Unknown network command: "${args._[1]}"`);
|
|
226
|
+
console.error(` Network commands: ${NETWORK_COMMANDS.join(', ')}`);
|
|
227
|
+
process.exit(1);
|
|
228
|
+
}
|
|
229
|
+
args._ = args._.slice(1);
|
|
230
|
+
command = args._[0];
|
|
231
|
+
}
|
|
191
232
|
const cwd = process.cwd();
|
|
192
233
|
|
|
193
234
|
// -----------------------------------------------------------------
|
|
@@ -264,11 +305,18 @@ if (args.help && command !== 'help') {
|
|
|
264
305
|
.then(mod => mod.run(args, cwd))
|
|
265
306
|
.then(code => exitWhenFlushed(code || 0))
|
|
266
307
|
.catch(err => {
|
|
267
|
-
|
|
308
|
+
// A gettext context key in the message shows its U+0004 as "␄",
|
|
309
|
+
// the form --redo keys: accepts back (lib/output.js showKey).
|
|
310
|
+
console.error(`[ERR] ${command} failed:`, String(err.message).replace(/\u0004/g, '\u2404'));
|
|
268
311
|
exitWhenFlushed(1);
|
|
269
312
|
});
|
|
270
313
|
} else if (command === 'help') {
|
|
271
314
|
import('../lib/commands/help.js').then(mod => mod.run());
|
|
315
|
+
} else if (command === 'network') {
|
|
316
|
+
import('../lib/command-help.js').then(({ showCommandHelp }) => {
|
|
317
|
+
showCommandHelp('network');
|
|
318
|
+
process.exit(0);
|
|
319
|
+
});
|
|
272
320
|
} else {
|
|
273
321
|
// Unknown command — error loudly so CI typos don't silently pass
|
|
274
322
|
console.error(`[ERR] Unknown command: "${command}"`);
|
package/index.js
CHANGED
|
@@ -15,8 +15,11 @@
|
|
|
15
15
|
* getRegisterPresets, getFormality, getGenderGuidance, getAllLanguageCodes,
|
|
16
16
|
* resolveCode
|
|
17
17
|
* - Sync: runSync, runContentSync
|
|
18
|
+
* - One pair's pipeline: translateWithFallback, translateAndValidate,
|
|
19
|
+
* previewRequests, createFallbackBudget
|
|
20
|
+
* - Locale layout: discoverLocaleLayout, resolveLocaleFiles
|
|
18
21
|
* - Quality: validateTranslations
|
|
19
|
-
* - Utilities: loadApiKey, getEnvOrFileVar
|
|
22
|
+
* - Utilities: loadApiKey, getEnvOrFileVar, costLabel
|
|
20
23
|
*/
|
|
21
24
|
|
|
22
25
|
// ── Translation method classes ─────────────────────────────────────
|
|
@@ -49,6 +52,19 @@ export {
|
|
|
49
52
|
} from './lib/config.js';
|
|
50
53
|
export { resolvePairs } from './lib/pairs.js';
|
|
51
54
|
|
|
55
|
+
// ── Writing systems (a pair's `script`) ────────────────────────────
|
|
56
|
+
// The same decision sync makes: a locale with two real orthographies (crk,
|
|
57
|
+
// sr) refuses until one is chosen; a chosen display script is produced by
|
|
58
|
+
// converting the working-script translation. The MCP translate tool uses
|
|
59
|
+
// these so it never picks an orthography sync would refuse to pick.
|
|
60
|
+
export {
|
|
61
|
+
resolveTargetScript,
|
|
62
|
+
formatScriptChoiceError,
|
|
63
|
+
convertScript,
|
|
64
|
+
applyScriptFallback,
|
|
65
|
+
getConverterInfo,
|
|
66
|
+
} from './lib/scripts.js';
|
|
67
|
+
|
|
52
68
|
// ── Language cards & registers ─────────────────────────────────────
|
|
53
69
|
export {
|
|
54
70
|
getLanguageCard,
|
|
@@ -60,6 +76,10 @@ export {
|
|
|
60
76
|
getAllLanguageCodes,
|
|
61
77
|
getMethodSupport,
|
|
62
78
|
resolveCode,
|
|
79
|
+
// A code or a language NAME ("French") → the code to work in; with a
|
|
80
|
+
// project's locales, the project's own spelling ("fr"). Never guesses.
|
|
81
|
+
findLanguagesByName,
|
|
82
|
+
resolveLanguageInput,
|
|
63
83
|
// Dynamic card tier (packaged installs): warm the per-user cache up
|
|
64
84
|
// front instead of paying a per-miss synchronous fetch.
|
|
65
85
|
prefetchLanguageCards,
|
|
@@ -83,12 +103,39 @@ export {
|
|
|
83
103
|
// ── Sync pipeline ──────────────────────────────────────────────────
|
|
84
104
|
export { runSync, runContentSync } from './lib/sync.js';
|
|
85
105
|
|
|
106
|
+
// ── One pair's translation pipeline ────────────────────────────────
|
|
107
|
+
// What `champollion sync` and `serve` run for a batch of keys: cache →
|
|
108
|
+
// method → quality gate → cache, then the pair's fallback for what that
|
|
109
|
+
// left (a resolved pair from resolvePairs; `options.tm` from loadTM;
|
|
110
|
+
// `options.cwd` = the project directory, where methods read their key,
|
|
111
|
+
// endpoint, coaching and glossary). The MCP translate tool runs a project's
|
|
112
|
+
// fallback through it. createFallbackBudget caps a fallback under a cost.
|
|
113
|
+
export { translateWithFallback, translateAndValidate, previewRequests } from './lib/translate-pair.js';
|
|
114
|
+
export { createFallbackBudget } from './lib/fallback.js';
|
|
115
|
+
|
|
116
|
+
// ── Locale layout ───────────────────────────────────────────────────
|
|
117
|
+
// Which files make up each locale (flat, folder per locale, localesPattern)
|
|
118
|
+
// — the same answer sync, verify and xliff use, for build scripts and
|
|
119
|
+
// agents that need to find a project's locale files.
|
|
120
|
+
export { discoverLocaleLayout, resolveLocaleFiles } from './lib/locale-layout.js';
|
|
121
|
+
|
|
86
122
|
// ── Quality gate ───────────────────────────────────────────────────
|
|
87
|
-
|
|
123
|
+
// The shared-output rule (one text for several different sources) — the MCP
|
|
124
|
+
// translate tool holds its answers to it, as sync does.
|
|
125
|
+
export { validateTranslations, SharedOutputIndex, sharedOutputItems, sharedOutputReason } from './lib/validate.js';
|
|
126
|
+
// The index a sync of one locale starts with (the memorized sentences its
|
|
127
|
+
// cache remembers + what the project's files and pages already hold): the
|
|
128
|
+
// MCP translate tool, given a project_dir, refuses what sync would refuse.
|
|
129
|
+
export { projectSharedOutputIndex } from './lib/shared-output-seed.js';
|
|
88
130
|
|
|
89
131
|
// ── API key resolution ─────────────────────────────────────────────
|
|
90
132
|
export { loadApiKey, getEnvOrFileVar } from './lib/api-key.js';
|
|
91
133
|
|
|
134
|
+
// ── Cost wording ───────────────────────────────────────────────────
|
|
135
|
+
// One label for an estimate ("$0 API cost (runs on this machine)",
|
|
136
|
+
// "est. ~$0.0123"): the MCP translate tool names a price in the CLI's words.
|
|
137
|
+
export { costLabel, LOCAL_COST_LABEL } from './lib/cost-label.js';
|
|
138
|
+
|
|
92
139
|
// ── Coaching utilities (for custom method implementations) ─────────
|
|
93
140
|
export {
|
|
94
141
|
loadCoachingData,
|
|
@@ -107,7 +154,21 @@ export {
|
|
|
107
154
|
partitionByTM,
|
|
108
155
|
tmMethodKey,
|
|
109
156
|
tmSize,
|
|
157
|
+
// A project's cache from before coaching was keyed reads as sync reads it.
|
|
158
|
+
adoptLegacyCoachingKeys,
|
|
159
|
+
// Which setup wrote the entry a lookup serves (an earlier model's, under
|
|
160
|
+
// model carry-over), and that key in words — so a cached answer can say
|
|
161
|
+
// who wrote it when that is not the engine named (MCP translate).
|
|
162
|
+
servingMethodKey,
|
|
163
|
+
describeMethodKey,
|
|
110
164
|
} from './lib/tm.js';
|
|
165
|
+
// The text a key is cached under — a gettext msgctxt folded in
|
|
166
|
+
// ("msgctxt\u0004msgid" → "msgctxt\u0004<text>") — and which contexts a
|
|
167
|
+
// project's source gives each text: the MCP translate tool keys a context as
|
|
168
|
+
// sync does, and never writes a context-free entry for a text the project
|
|
169
|
+
// has only with a context.
|
|
170
|
+
export { tmSourceText, CONTEXT_SEPARATOR } from './lib/tm-evict.js';
|
|
171
|
+
export { sourceTextContexts } from './lib/source-contexts.js';
|
|
111
172
|
|
|
112
173
|
// ── XLIFF interchange ──────────────────────────────────────────────
|
|
113
174
|
export {
|
package/lib/api-key.js
CHANGED
|
@@ -83,22 +83,35 @@ function loadApiKey(config, cwd) {
|
|
|
83
83
|
* @returns {string|null} The variable value, or null if not found
|
|
84
84
|
*/
|
|
85
85
|
function getEnvOrFileVar(keyName, cwd = process.cwd()) {
|
|
86
|
+
return findEnvOrFileVar(keyName, cwd)?.value ?? null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* getEnvOrFileVar, saying WHERE the value was found — for messages that
|
|
91
|
+
* must name the setting a run used (e.g. the endpoint a local method could
|
|
92
|
+
* not reach, and whether it came from the shell or a .env file).
|
|
93
|
+
*
|
|
94
|
+
* @param {string} keyName - Name of the environment variable
|
|
95
|
+
* @param {string} [cwd=process.cwd()] - Project root directory
|
|
96
|
+
* @returns {{ value: string, origin: 'environment'|'.env.local'|'.env', file: string|null }|null}
|
|
97
|
+
*/
|
|
98
|
+
function findEnvOrFileVar(keyName, cwd = process.cwd()) {
|
|
86
99
|
// Priority 1: Check environment directly. Normalize identically to file
|
|
87
100
|
// values (trim + strip surrounding quotes) so a quoted/padded exported secret
|
|
88
101
|
// never reaches an Authorization header verbatim.
|
|
89
102
|
if (process.env[keyName]) {
|
|
90
|
-
return normalizeKeyValue(process.env[keyName]);
|
|
103
|
+
return { value: normalizeKeyValue(process.env[keyName]), origin: 'environment', file: null };
|
|
91
104
|
}
|
|
92
105
|
|
|
93
106
|
// Priority 2: .env.local (Vercel/Next.js convention)
|
|
94
107
|
const envLocalPath = path.join(cwd, '.env.local');
|
|
95
108
|
const fromLocal = readKeyFromFile(envLocalPath, keyName);
|
|
96
|
-
if (fromLocal) return fromLocal;
|
|
109
|
+
if (fromLocal) return { value: fromLocal, origin: '.env.local', file: envLocalPath };
|
|
97
110
|
|
|
98
111
|
// Priority 3: .env
|
|
99
112
|
const envPath = path.join(cwd, '.env');
|
|
100
113
|
const fromEnv = readKeyFromFile(envPath, keyName);
|
|
101
|
-
if (fromEnv) return fromEnv;
|
|
114
|
+
if (fromEnv) return { value: fromEnv, origin: '.env', file: envPath };
|
|
102
115
|
|
|
103
116
|
return null;
|
|
104
117
|
}
|
|
@@ -124,4 +137,4 @@ function readKeyFromFile(filePath, targetKey) {
|
|
|
124
137
|
return null;
|
|
125
138
|
}
|
|
126
139
|
|
|
127
|
-
export { loadApiKey, parseEnvLine, getEnvOrFileVar };
|
|
140
|
+
export { loadApiKey, parseEnvLine, getEnvOrFileVar, findEnvOrFileVar };
|