champollion 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/README.md +52 -37
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +83 -36
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +51 -3
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +289 -88
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1689 -87
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +649 -130
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +16 -10
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +196 -14
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +270 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1127 -344
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +197 -38
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +726 -78
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +965 -253
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +964 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +571 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +252 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +68 -128
  70. package/lib/methods/llm.js +80 -31
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +11 -0
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +193 -106
  100. package/lib/seal.mjs +6 -5
  101. package/lib/sealed-qualifier.mjs +2 -2
  102. package/lib/segment.js +2 -1
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +880 -17
  117. package/lib/verify.js +1296 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +3 -2
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/DATA-SOVEREIGNTY.md +19 -20
  123. package/shared/LANGUAGE-CARD-FIELDS.md +1 -1
  124. package/shared/cards-fallback.json +1 -1
  125. package/shared/catalogue/card-config.json +1 -1
  126. package/shared/curated-orthography-conventions.json +26 -8
  127. package/shared/docent/faq.en.json +14 -16
  128. package/shared/docent/system-prompt.md +17 -19
  129. package/shared/explainers/tc-features.json +15 -15
  130. package/shared/gettext-plural-forms.json +45 -0
  131. package/shared/human-services.json +1 -1
  132. package/shared/method-registry.json +2 -0
  133. package/shared/metric-registry.json +96 -18
  134. package/shared/schemas/champollion-plugin.schema.json +4 -0
  135. package/shared/schemas/corpora-card.schema.json +20 -10
  136. package/shared/schemas/human-services.schema.json +2 -2
  137. package/shared/schemas/language-card.schema.json +1 -1
  138. package/shared/schemas/method-card.schema.json +1 -1
  139. package/shared/schemas/method-index-record.schema.json +67 -0
  140. package/shared/schemas/method-registry.schema.json +4 -0
  141. package/shared/schemas/metric-registry.schema.json +55 -1
  142. package/shared/docent/corpus.json +0 -11333
package/README.md CHANGED
@@ -16,9 +16,11 @@ npx champollion sync
16
16
 
17
17
  Champollion auto-detects your locale files, their format, and the target languages. It translates missing keys, skips what's already done, and writes the results. That's it.
18
18
 
19
- > **Part of Champollion** — infrastructure for trustworthy machine translation
20
- > across every language, source-available and free for noncommercial use (the
21
- > evaluation harness and shared registries are open source). This CLI is the
19
+ **License:** source-available under the [PolyForm Noncommercial License 1.0.0](LICENSE) — free to use, change and share for noncommercial purposes (personal projects, research, teaching, and use by charities, schools and public bodies). Using it for a commercial purpose is not covered by this license. The evaluation harness is open source (AGPL-3.0-or-later). [Who may use this](https://champollion.dev/docs/getting-started/who-may-use-this) · [Details](#license).
20
+
21
+ > **Part of Champollion** — infrastructure for trustworthy machine-translation
22
+ > evaluation across every language, source-available and free for noncommercial
23
+ > use (the evaluation harness and shared registries are open source). This CLI is the
22
24
  > deployment end of a larger
23
25
  > project that builds the test sets and the map showing who can translate what,
24
26
  > how good each method is on each kind of text, and where the gaps still are. It
@@ -26,8 +28,8 @@ Champollion auto-detects your locale files, their format, and the target languag
26
28
  > every method welcome) and sovereign benchmarks — secret test sets that
27
29
  > communities create, own, and control, and that we never see. The infrastructure
28
30
  > is source-available and singly stewarded; the test sets and the methods for a
29
- > community's language belong to that community. Built with communities, never
30
- > scraped from them — they hold the keys. Every method is welcome, human and
31
+ > community's language belong to that community. Designed to work with communities,
32
+ > never hosting their corpora — the design puts the keys in their hands. Every method is welcome, human and
31
33
  > machine. Explore the network at
32
34
  > [champollion.dev/docs/network](https://champollion.dev/docs/network/).
33
35
 
@@ -37,7 +39,7 @@ You could write a quick script that loops through your English keys and calls Go
37
39
 
38
40
  - **No change detection.** When you update an English string, the translation stays stale forever. Champollion tracks every source value with SHA-256 hashes and re-translates only what changed.
39
41
  - **No batching.** One API call per key means 200 keys = 200 round trips. Champollion batches intelligently (configurable, default 80 keys/batch for LLM, 128 for Google).
40
- - **No quality gate.** Machine translation hallucinates, echoes the source back, or outputs in the wrong script. Champollion validates every translation before writing it — wrong-script, length inflation, and source echoes are caught and rejected.
42
+ - **No quality gate.** Machine translation hallucinates, echoes the source back, or outputs in the wrong script. Champollion checks every translation before writing it — empty output, source echoes, repetition loops, length inflation, deleted content and wrong script are caught and rejected. The gate catches broken output, not wrong meaning.
41
43
  - **No format awareness.** Hardcoded to JSON? Champollion handles JSON, TOML, YAML, and Hugo Markdown (frontmatter + body) with auto-detection.
42
44
  - **No safety.** Champollion guards against prototype pollution, path traversal via crafted locale codes, and code block corruption during Markdown translation.
43
45
 
@@ -118,17 +120,17 @@ You handle the i18n framework (next-intl, i18next, Hugo). Champollion handles th
118
120
  - **Multi-format** — JSON, TOML, YAML, Hugo Markdown (front matter + body), and XLIFF 1.2
119
121
  - **Incremental** — Only translates what changed (SHA-256 hash tracking)
120
122
  - **Cached** — Translation Memory stores previous results; re-running sync costs nothing for unchanged keys
121
- - **Quality-gated** — Validates every translation: catches hallucinations, wrong-script output, source echoes, and length inflation
123
+ - **Quality-gated** — Checks every translation for broken output: repetition loops, wrong-script output, source echoes, length inflation and deleted content (it does not check meaning)
122
124
  - **Content-aware** — LLM methods shield code blocks, shortcodes, links, and interpolation variables during Markdown translation
123
125
  - **Pipeline tools** — `lint`, `audit`, `integrity`, `seo` for CI gates
124
126
  - **XLIFF interop** — Export translations for professional review in CAT tools (memoQ, SDL Trados, Phrase), import them back
125
- - **Minimal dependencies** — two runtime dependencies (better-sqlite3 for the bundled language database, CLDR locale names); no provider SDKs. Requires Node 20+
127
+ - **Minimal dependencies** — two runtime dependencies: `@translated/lara` (the Lara provider SDK) and `cree-sro-syllabics` (ALTLab/NRC's Plains Cree SRO ⇄ syllabics converter). Requires Node 20+
126
128
 
127
129
  ## Beyond Google Translate
128
130
 
129
131
  The quick start gets you running with an LLM or Google Translate. But Google Translate supports ~130 languages. There are over 7,000.
130
132
 
131
- **Champollion's core idea: the translation method is configurable per language pair.** Use Google Translate for French, an LLM with morphological coaching for Plains Cree, and a community-hosted API for Quechua — all in the same project, all with the same CLI.
133
+ **Champollion's core idea: the translation method is configurable per language pair.** Use Google Translate for French, an LLM coached with your own grammar notes and dictionary for Plains Cree, and an API you host for a language no service covers — all in the same project, all with the same CLI.
132
134
 
133
135
  ```json
134
136
  {
@@ -136,7 +138,7 @@ The quick start gets you running with an LLM or Google Translate. But Google Tra
136
138
  "pairs": {
137
139
  "en:fr": { "method": "google-translate" },
138
140
  "en:ja": { "method": "llm" },
139
- "en:crk": { "methodPlugin": "crk-coached-v1" }
141
+ "en:crk": { "method": "llm-coached" }
140
142
  }
141
143
  }
142
144
  ```
@@ -160,6 +162,7 @@ Champollion supports 10 translation methods. Each language pair can use a differ
160
162
  | `openai` | `OPENAI_API_KEY` | Direct OpenAI API (gpt-4o, gpt-4o-mini) |
161
163
  | `anthropic` | `ANTHROPIC_API_KEY` | Direct Anthropic API (Claude Sonnet, Haiku, Opus) |
162
164
  | `gemini` | `GEMINI_API_KEY` | Direct Google Gemini API (Flash, Pro) — free tier available |
165
+ | `local` | *(none)* | Your own model behind an OpenAI-compatible endpoint (Ollama, vLLM, LM Studio, a model trained with `nmt-forge`). Set `LOCAL_API_BASE` if it is not at Ollama's default. Text never leaves your infrastructure |
163
166
 
164
167
  **Traditional MT** — best for speed, cost, and high-volume key-value pairs:
165
168
 
@@ -188,7 +191,7 @@ champollion sync --method deepl
188
191
  "pairs": {
189
192
  "en:fr": { "method": "deepl" },
190
193
  "en:ja": { "method": "openai", "model": "gpt-4o" },
191
- "en:crk": { "methodPlugin": "crk-coached-v1" }
194
+ "en:crk": { "method": "llm-coached" }
192
195
  }
193
196
  }
194
197
  ```
@@ -203,7 +206,7 @@ Plugins are pre-packaged translation recipes for specific language pairs. They'r
203
206
  champollion plugin install ./french-formal-v1/ # install from directory
204
207
  champollion plugin list # see installed plugins
205
208
  champollion plugin remove french-formal-v1 # uninstall
206
- champollion status # shows quality tiers + benchmarks
209
+ champollion status # shows each pair's method, model + plugin benchmarks
207
210
  ```
208
211
 
209
212
  See [the reference docs](https://champollion.dev/docs/reference/plugin-spec) for the manifest format.
@@ -217,12 +220,8 @@ See [the reference docs](https://champollion.dev/docs/reference/plugin-spec) for
217
220
  | `serve` | Serve this project's translation stack over HTTP (api-method contract) |
218
221
  | `watch` | Auto-sync on file changes |
219
222
  | `audit` | Flag incomplete locales (CI gate) |
220
- | `card` | Pretty-print a language card (`card <code>`, `--json` for raw) |
221
- | `register-corpus` | Register an evaluation corpus: pick a license + exposure tier (local-only/private/public/sealed) |
222
- | `seal-corpus` | Sealed-tier crypto verbs: `keygen` / `seal` / `open` (organizer-node bridge) |
223
- | `submit` | Propose an index entry (review-gated) — prints a pre-filled GitHub issue |
224
223
  | `lint` | Find hardcoded strings in source code |
225
- | `status` | Show pair configuration, methods, registers, and quality tiers |
224
+ | `status` | Show pair configuration, methods, registers, and plugin benchmarks |
226
225
  | `provenance` | Audit translation resource licensing |
227
226
  | `wrap` | Auto-wrap hardcoded strings in `t()` calls (with undo) |
228
227
  | `seo` | Generate hreflang, sitemap.xml, or JSON-LD schema |
@@ -233,10 +232,21 @@ See [the reference docs](https://champollion.dev/docs/reference/plugin-spec) for
233
232
  | `xliff` | Export/import XLIFF 1.2 for professional translator review |
234
233
  | `models` | List available models for a provider (`--method gemini`) |
235
234
  | `verify` | Re-read written locale files and confirm translations are present and correct (CI gate) |
236
- | `leaderboard` | Show the MT leaderboard (`--pair`, `--sort`, `--install N`) |
237
- | `recommend` | Method guidance for a pair — availability + cited evidence (`--use`, `--json`) |
238
235
  | `doctor` | System health check: cards, config, methods, and converters |
239
236
 
237
+ The commands that work with the shared index and leaderboard, rather than your
238
+ project, are grouped under `champollion network` (each also works without the
239
+ prefix):
240
+
241
+ | Command | Purpose |
242
+ |---------|---------|
243
+ | `network card` | Pretty-print a language card (`network card <code>`, `--json` for raw) |
244
+ | `network recommend` | Method guidance for a pair — availability + cited evidence (`network recommend eng fra`, `--use`, `--json`) |
245
+ | `network leaderboard` | Show the MT leaderboard (`--pair`, `--sort`, `--install N`) |
246
+ | `network register-corpus` | Register an evaluation corpus: pick a license + exposure tier (local-only/private/public/sealed) |
247
+ | `network seal-corpus` | Sealed-tier crypto verbs: `keygen` / `seal` / `open` (organizer-node bridge) |
248
+ | `network submit` | Propose an index entry (review-gated) — prints a pre-filled GitHub issue |
249
+
240
250
  Run `champollion <command> --help` for detailed help on any command.
241
251
 
242
252
  Full reference: [the reference docs](https://champollion.dev/docs/reference/cli)
@@ -247,7 +257,7 @@ Full reference: [the reference docs](https://champollion.dev/docs/reference/cli)
247
257
 
248
258
  ```bash
249
259
  mkdir -p .githooks
250
- printf '#!/bin/sh\nnpx champollion lint\n' > .githooks/pre-commit
260
+ printf '#!/bin/sh\nnpx --yes champollion@0.4 lint\n' > .githooks/pre-commit # pinned: a new release never changes what the hook runs
251
261
  chmod +x .githooks/pre-commit
252
262
  git config core.hooksPath .githooks # once per clone
253
263
  ```
@@ -284,8 +294,9 @@ Create `champollion.config.json` or run `champollion init`:
284
294
  | Option | Default | Description |
285
295
  |--------|---------|-------------|
286
296
  | `inputLocale` | `"en"` | Source language code |
287
- | `localesDir` | `"./locales"` | Path to locale files |
288
- | `contentDir` | `null` | Hugo content directory (enables Markdown translation) |
297
+ | `localesDir` | `"./locales"` | Path to locale files: one file per language (`fr.json`) or one folder per language (`fr/common.json`, i18next) |
298
+ | `localesPattern` | `null` | Any other layout, e.g. `"src/i18n/{ns}/{lang}.json"` (replaces `localesDir`) |
299
+ | `contentDir` | `null` | Folder of Markdown/MDX to translate (a Hugo `content/` or any folder); each translation is written beside its source as `<name>.<locale>.md` |
289
300
  | `format` | `"auto"` | File format: `json`, `toml`, `yaml`, or `auto` |
290
301
  | `model` | `"google/gemini-3.5-flash"` | Default model (OpenRouter slug). Direct providers resolve their own default at runtime. Run `champollion models --method gemini` to discover available models. |
291
302
  | `defaultMethod` | `"llm"` | Default translation method (overridden by `--method` flag) |
@@ -302,7 +313,7 @@ Create `champollion.config.json` or run `champollion init`:
302
313
  "crk": {
303
314
  "name": "Plains Cree",
304
315
  "register": "SRO syllabics with grammatical precision.",
305
- "model": "google/gemini-2.5-pro",
316
+ "model": "google/gemini-3.1-pro-preview",
306
317
  "batchSize": 5,
307
318
  "maxRetries": 5,
308
319
  "script": "Cans"
@@ -322,30 +333,34 @@ See the [CLI Reference](https://champollion.dev/docs/reference/cli) for framewor
322
333
  When you run `sync`, champollion shows exactly what's happening:
323
334
 
324
335
  ```
325
- champollion v0.1.0
336
+ champollion v0.4.0
326
337
 
327
338
  [INFO] Detected format: json (auto)
328
339
  [INFO] Detected framework: Hugo
329
- [INFO] Source: en.json (2,847 keys)
340
+ [INFO] Source: en.json (2847 keys)
330
341
  [INFO] Pairs: es-MX:llm, fr:deepl, it:llm
331
- [INFO] Estimated translation cost:
342
+ Estimated translation cost:
332
343
 
333
- Pair Method Keys Est. Cost
334
- ──────── ────── ──── ─────────
335
- en:es-MX llm 2847 ~$0.8400
336
- en:fr deepl 2847 ~$0.5694
337
- en:it llm 2847 ~$0.8400
344
+ Pair Method Keys TM hits Est. Cost
345
+ ──────── ────── ────── ─────── ──────────
346
+ en:es-MX llm 2847 0 ~$0.3843
347
+ en:fr deepl 2847 0 ~$1.7794
348
+ en:it llm 2847 0 ~$0.3843
338
349
 
339
- Total: ~$2.2494
350
+ Total: ~$2.5480
351
+ Rates: google/gemini-3.5-flash $0.30 input / $2.50 output per 1M tokens — OpenRouter's price list, read 2026-10-04 14:02 UTC; deepl $25.00 per 1M characters — the provider's published price, as checked 2026-06-08. An estimate: ~200 input + ~30 output tokens or ~25 characters per key assumed — the bill depends on the real lengths (--json has the detail).
340
352
 
341
- [INFO] es-MX.json — 2,847 missing
353
+ [INFO] Translating 3 locale(s) with concurrency 50
354
+ [INFO] es-MX.json — 2847 missing
355
+ ████████████████████████████████ 2,847/2,847 keys
356
+ [INFO] fr.json — 2847 missing
342
357
  ████████████████████████████████ 2,847/2,847 keys
343
- [INFO] fr.json — 2,847 missing
358
+ [INFO] it.json — 2847 missing
344
359
  ████████████████████████████████ 2,847/2,847 keys
345
- [OK] Synced 5,694 keys total.
360
+ [OK] Synced 8541 keys total — 8541 key(s) sent to the model, 0 served from the cache (free).
346
361
  ```
347
362
 
348
- The progress bar updates in-place as each batch completes (~80 keys per update). Framework detection shows `Hugo` when `contentDir` is set. Format detection distinguishes `(auto)` from `(config)` to clarify how the format was resolved.
363
+ The estimate's last line names the rate each figure was priced at, where it came from and when (`--json` carries the detail). "Translating …" is said only when something will be sent: a dry run, or a run with nothing to send, says it is checking the locales instead. The progress bar updates in-place as each batch completes (~80 keys per update). When `contentDir` is set, sync names the folder and where its translations go; it says `Detected framework: Hugo` only when it finds a real Hugo site (`hugo.toml`, a `config.toml` with Hugo settings, `archetypes/`, …). Format detection distinguishes `(auto)` from `(config)` to clarify how the format was resolved.
349
364
 
350
365
  **Output modes**: `--quiet` suppresses informational output (errors and warnings only). `--json` emits machine-readable NDJSON for CI/CD pipelines.
351
366
 
@@ -384,4 +399,4 @@ npm run test:methods # translation method suite
384
399
 
385
400
  ## License
386
401
 
387
- PolyForm Noncommercial 1.0.0. The Champollion CLI is source-available — free to install, use, and modify for noncommercial purposes (research, education, community work) under the [PolyForm Noncommercial License 1.0.0](LICENSE); commercial use requires the licensor's permission. The published `champollion` npm package is PolyForm-Noncommercial-1.0.0; `cli/LICENSE` is the authoritative license for the distributed package. The companion MT Eval Harness and specs are open source, licensed AGPL-3.0-or-later — with a §7 eval-standard-plugin exception — at the public [harness repository](https://github.com/gamedaysuits/Champollion).
402
+ PolyForm Noncommercial 1.0.0. The Champollion CLI is source-available — free to install, use, and modify for noncommercial purposes (research, education, community work) under the [PolyForm Noncommercial License 1.0.0](LICENSE); using it for a commercial purpose is not covered by this license. In plain words, with examples — a school, a public hospital or clinic, a charity, a personal or research project is covered; a for-profit business's product, such as a shop's storefront, is not: [Who may use this](https://champollion.dev/docs/getting-started/who-may-use-this) (a summary, not legal advice; the license text governs). The published `champollion` npm package is PolyForm-Noncommercial-1.0.0; `cli/LICENSE` is the authoritative license for the distributed package. The companion MT Eval Harness and specs are open source, licensed AGPL-3.0-or-later — with a §7 eval-standard-plugin exception — at the public [harness repository](https://github.com/gamedaysuits/Champollion).
package/bin/cli.js CHANGED
@@ -34,10 +34,17 @@ const CLI_OPTIONS = {
34
34
  version: { type: 'boolean', short: 'v' },
35
35
  yes: { type: 'boolean', short: 'y' },
36
36
  'warn-only': { type: 'boolean' },
37
+ strict: { type: 'boolean' }, // verify --strict: warnings fail the check (exit 1)
37
38
  'no-verify': { type: 'boolean' }, // skip post-sync verification
38
39
  undo: { type: 'boolean' },
39
40
  'force-content': { type: 'boolean' }, // ignore the content lock (TM still serves unchanged bodies/blocks)
41
+ files: { type: 'string', multiple: true }, // sync: only these content files (glob, repeatable)
42
+ retranslate: { type: 'string', multiple: true }, // sync: translate these content files fresh (glob, repeatable)
43
+ redo: { type: 'string', multiple: true }, // sync: queue again — all | keys:<k> | content | files:<glob> (lib/redo.js)
44
+ fresh: { type: 'boolean' }, // sync: do not use the cache for what is queued (billed again)
45
+ prune: { type: 'string', multiple: true }, // sync: remove what the locale cannot use — plural-extras (i18next keys for forms the language does not have)
40
46
  'no-tm': { type: 'boolean' }, // skip Translation Memory for this sync run
47
+ 'fresh-on-model-change': { type: 'boolean' }, // don't reuse other models' TM entries
41
48
  css: { type: 'boolean' }, // fonts install --css: generate CSS snippet
42
49
  json: { type: 'boolean' }, // machine-readable NDJSON output
43
50
  quiet: { type: 'boolean', short: 'q' }, // suppress info/ok messages, show only warnings/errors
@@ -49,12 +56,14 @@ const CLI_OPTIONS = {
49
56
  'content-dir': { type: 'string' },
50
57
  source: { type: 'string' },
51
58
  langs: { type: 'string' },
59
+ script: { type: 'string' }, // init: writing system per target, e.g. crk=Cans (a language with two real orthographies)
52
60
  model: { type: 'string' },
53
61
  method: { type: 'string' },
54
62
  format: { type: 'string' },
55
63
  'base-url': { type: 'string' },
56
64
  out: { type: 'string' },
57
65
  src: { type: 'string' },
66
+ ignore: { type: 'string' }, // lint: comma-separated directory/file names to skip
58
67
  'min-length': { type: 'string' },
59
68
  'force-keys': { type: 'string' },
60
69
  'list-keys': { type: 'boolean' }, // sync --dry: name every queued key per reason
@@ -63,7 +72,8 @@ const CLI_OPTIONS = {
63
72
  'json-concurrency': { type: 'string' }, // max parallel API calls for JSON key-value translation (default: 50)
64
73
  'content-concurrency': { type: 'string' }, // max parallel API calls for markdown content (default: 12)
65
74
  'batch-size': { type: 'string' }, // keys per translation API call (positive integer; overrides config batchSize)
66
- 'max-cost': { type: 'string' }, // sync: USD cap — abort (exit 2) before any API call if the estimate exceeds it or is unknown
75
+ 'max-cost': { type: 'string' },
76
+ 'show-prompt': { type: 'string' }, // sync --dry: print the exact request the method would get (optionally for one key) // sync: USD cap — abort (exit 2) before any API call if the estimate exceeds it or is unknown
67
77
  temperature: { type: 'string' }, // sampling temperature for LLM methods
68
78
  'coaching-file': { type: 'string' }, // path to coaching prompt text file
69
79
  locale: { type: 'string' }, // target locale for xliff export, tm clear/seed
@@ -80,7 +90,7 @@ const CLI_OPTIONS = {
80
90
  license: { type: 'string' }, // register-corpus: license key / SPDX id / list number
81
91
  tier: { type: 'string' }, // register-corpus: exposure tier (local-only|private|public)
82
92
  exposure: { type: 'string' }, // register-corpus: alias for --tier
83
- name: { type: 'string' }, // register-corpus: corpus name
93
+ name: { type: 'string' }, // register-corpus: corpus name; init: display name of a language with no card (qaa="…")
84
94
  publisher: { type: 'string' }, // register-corpus: publisher / author
85
95
  description: { type: 'string' }, // register-corpus: one-line description
86
96
  'source-lang': { type: 'string' }, // register-corpus: source language code (alt to --pair)
@@ -90,10 +100,12 @@ const CLI_OPTIONS = {
90
100
  'license-url': { type: 'string' }, // register-corpus: upstream license URL
91
101
  builder: { type: 'string' }, // register-corpus: builder adapter id
92
102
  sha256: { type: 'string' }, // register-corpus: built-corpus hash (optional)
103
+ data: { type: 'string' }, // register-corpus: the local file the card describes (hashed, never uploaded)
93
104
  contamination: { type: 'string' }, // register-corpus: NONE|LOW|MEDIUM|HIGH
94
105
  domain: { type: 'string' }, // register-corpus: corpus domain
95
106
  size: { type: 'string' }, // register-corpus: number of sentence pairs
96
107
  id: { type: 'string' }, // register-corpus: override card id
108
+ role: { type: 'string' }, // register-corpus: test|dev|train — in the id only when stated
97
109
  version: { type: 'string' }, // register-corpus: corpus version (major)
98
110
  'do-not-train': { type: 'boolean' }, // register-corpus: benchmark-integrity flag (default true)
99
111
  'cards-dir': { type: 'string' }, // register-corpus: corpora-cards dir override (advanced/testing)
@@ -133,7 +145,8 @@ const CLI_OPTIONS = {
133
145
  'max-cost-per-request': { type: 'string' }, // serve: USD cap per request (unknown estimates refuse)
134
146
  'max-session-cost': { type: 'string' }, // serve: USD spend ceiling for the server process
135
147
  'emit-manifest': { type: 'boolean' }, // serve: write the consumer method.json and exit
136
- endpoint: { type: 'string' }, // serve --emit-manifest: public endpoint URL for the manifest
148
+ endpoint: { type: 'string' }, // serve --emit-manifest: public endpoint URL; init --method api: the champollion API endpoint
149
+ 'accepts-instructions': { type: 'string' }, // init --method api: does the endpoint follow per-key instructions (true|false)
137
150
  };
138
151
 
139
152
  const { values, positionals } = parseArgs({
@@ -169,6 +182,17 @@ const args = { _: positionals, ...values };
169
182
  // --dry-run is an alias for --dry — merge so command modules only check args.dry
170
183
  if (args['dry-run']) args.dry = true;
171
184
 
185
+ // --redo / --fresh are the one way to say "translate this again"; they are
186
+ // rewritten into the older flags here so sync keeps one set of semantics.
187
+ if (args.redo !== undefined || args.fresh) {
188
+ const { applyRedo } = await import('../lib/redo.js');
189
+ const redoError = applyRedo(args);
190
+ if (redoError) {
191
+ console.error(`[ERR] ${String(redoError).replace(/\u0004/g, '\u2404')}`);
192
+ process.exit(1);
193
+ }
194
+ }
195
+
172
196
  /**
173
197
  * Exit only once stdout has actually been written.
174
198
  *
@@ -187,7 +211,24 @@ function exitWhenFlushed(code) {
187
211
  process.stdout.write('', () => process.exit(code));
188
212
  }
189
213
 
190
- const command = args._[0] || 'help';
214
+ // `champollion network <command>` — the commands that work with the shared
215
+ // index and leaderboard rather than this project. Each also works without
216
+ // the prefix (the names people already use).
217
+ // (scripts/check_doc_commands.py reads COMMAND_GROUPS to check the docs.)
218
+ const COMMAND_GROUPS = {
219
+ network: ['card', 'recommend', 'leaderboard', 'register-corpus', 'seal-corpus', 'submit'],
220
+ };
221
+ const NETWORK_COMMANDS = COMMAND_GROUPS.network;
222
+ let command = args._[0] || 'help';
223
+ if (command === 'network' && args._[1]) {
224
+ if (!NETWORK_COMMANDS.includes(args._[1])) {
225
+ console.error(`[ERR] Unknown network command: "${args._[1]}"`);
226
+ console.error(` Network commands: ${NETWORK_COMMANDS.join(', ')}`);
227
+ process.exit(1);
228
+ }
229
+ args._ = args._.slice(1);
230
+ command = args._[0];
231
+ }
191
232
  const cwd = process.cwd();
192
233
 
193
234
  // -----------------------------------------------------------------
@@ -264,11 +305,18 @@ if (args.help && command !== 'help') {
264
305
  .then(mod => mod.run(args, cwd))
265
306
  .then(code => exitWhenFlushed(code || 0))
266
307
  .catch(err => {
267
- console.error(`[ERR] ${command} failed:`, err.message);
308
+ // A gettext context key in the message shows its U+0004 as "␄",
309
+ // the form --redo keys: accepts back (lib/output.js showKey).
310
+ console.error(`[ERR] ${command} failed:`, String(err.message).replace(/\u0004/g, '\u2404'));
268
311
  exitWhenFlushed(1);
269
312
  });
270
313
  } else if (command === 'help') {
271
314
  import('../lib/commands/help.js').then(mod => mod.run());
315
+ } else if (command === 'network') {
316
+ import('../lib/command-help.js').then(({ showCommandHelp }) => {
317
+ showCommandHelp('network');
318
+ process.exit(0);
319
+ });
272
320
  } else {
273
321
  // Unknown command — error loudly so CI typos don't silently pass
274
322
  console.error(`[ERR] Unknown command: "${command}"`);
package/index.js CHANGED
@@ -15,8 +15,11 @@
15
15
  * getRegisterPresets, getFormality, getGenderGuidance, getAllLanguageCodes,
16
16
  * resolveCode
17
17
  * - Sync: runSync, runContentSync
18
+ * - One pair's pipeline: translateWithFallback, translateAndValidate,
19
+ * previewRequests, createFallbackBudget
20
+ * - Locale layout: discoverLocaleLayout, resolveLocaleFiles
18
21
  * - Quality: validateTranslations
19
- * - Utilities: loadApiKey, getEnvOrFileVar
22
+ * - Utilities: loadApiKey, getEnvOrFileVar, costLabel
20
23
  */
21
24
 
22
25
  // ── Translation method classes ─────────────────────────────────────
@@ -49,6 +52,19 @@ export {
49
52
  } from './lib/config.js';
50
53
  export { resolvePairs } from './lib/pairs.js';
51
54
 
55
+ // ── Writing systems (a pair's `script`) ────────────────────────────
56
+ // The same decision sync makes: a locale with two real orthographies (crk,
57
+ // sr) refuses until one is chosen; a chosen display script is produced by
58
+ // converting the working-script translation. The MCP translate tool uses
59
+ // these so it never picks an orthography sync would refuse to pick.
60
+ export {
61
+ resolveTargetScript,
62
+ formatScriptChoiceError,
63
+ convertScript,
64
+ applyScriptFallback,
65
+ getConverterInfo,
66
+ } from './lib/scripts.js';
67
+
52
68
  // ── Language cards & registers ─────────────────────────────────────
53
69
  export {
54
70
  getLanguageCard,
@@ -60,6 +76,10 @@ export {
60
76
  getAllLanguageCodes,
61
77
  getMethodSupport,
62
78
  resolveCode,
79
+ // A code or a language NAME ("French") → the code to work in; with a
80
+ // project's locales, the project's own spelling ("fr"). Never guesses.
81
+ findLanguagesByName,
82
+ resolveLanguageInput,
63
83
  // Dynamic card tier (packaged installs): warm the per-user cache up
64
84
  // front instead of paying a per-miss synchronous fetch.
65
85
  prefetchLanguageCards,
@@ -83,12 +103,39 @@ export {
83
103
  // ── Sync pipeline ──────────────────────────────────────────────────
84
104
  export { runSync, runContentSync } from './lib/sync.js';
85
105
 
106
+ // ── One pair's translation pipeline ────────────────────────────────
107
+ // What `champollion sync` and `serve` run for a batch of keys: cache →
108
+ // method → quality gate → cache, then the pair's fallback for what that
109
+ // left (a resolved pair from resolvePairs; `options.tm` from loadTM;
110
+ // `options.cwd` = the project directory, where methods read their key,
111
+ // endpoint, coaching and glossary). The MCP translate tool runs a project's
112
+ // fallback through it. createFallbackBudget caps a fallback under a cost.
113
+ export { translateWithFallback, translateAndValidate, previewRequests } from './lib/translate-pair.js';
114
+ export { createFallbackBudget } from './lib/fallback.js';
115
+
116
+ // ── Locale layout ───────────────────────────────────────────────────
117
+ // Which files make up each locale (flat, folder per locale, localesPattern)
118
+ // — the same answer sync, verify and xliff use, for build scripts and
119
+ // agents that need to find a project's locale files.
120
+ export { discoverLocaleLayout, resolveLocaleFiles } from './lib/locale-layout.js';
121
+
86
122
  // ── Quality gate ───────────────────────────────────────────────────
87
- export { validateTranslations } from './lib/validate.js';
123
+ // The shared-output rule (one text for several different sources) — the MCP
124
+ // translate tool holds its answers to it, as sync does.
125
+ export { validateTranslations, SharedOutputIndex, sharedOutputItems, sharedOutputReason } from './lib/validate.js';
126
+ // The index a sync of one locale starts with (the memorized sentences its
127
+ // cache remembers + what the project's files and pages already hold): the
128
+ // MCP translate tool, given a project_dir, refuses what sync would refuse.
129
+ export { projectSharedOutputIndex } from './lib/shared-output-seed.js';
88
130
 
89
131
  // ── API key resolution ─────────────────────────────────────────────
90
132
  export { loadApiKey, getEnvOrFileVar } from './lib/api-key.js';
91
133
 
134
+ // ── Cost wording ───────────────────────────────────────────────────
135
+ // One label for an estimate ("$0 API cost (runs on this machine)",
136
+ // "est. ~$0.0123"): the MCP translate tool names a price in the CLI's words.
137
+ export { costLabel, LOCAL_COST_LABEL } from './lib/cost-label.js';
138
+
92
139
  // ── Coaching utilities (for custom method implementations) ─────────
93
140
  export {
94
141
  loadCoachingData,
@@ -107,7 +154,21 @@ export {
107
154
  partitionByTM,
108
155
  tmMethodKey,
109
156
  tmSize,
157
+ // A project's cache from before coaching was keyed reads as sync reads it.
158
+ adoptLegacyCoachingKeys,
159
+ // Which setup wrote the entry a lookup serves (an earlier model's, under
160
+ // model carry-over), and that key in words — so a cached answer can say
161
+ // who wrote it when that is not the engine named (MCP translate).
162
+ servingMethodKey,
163
+ describeMethodKey,
110
164
  } from './lib/tm.js';
165
+ // The text a key is cached under — a gettext msgctxt folded in
166
+ // ("msgctxt\u0004msgid" → "msgctxt\u0004<text>") — and which contexts a
167
+ // project's source gives each text: the MCP translate tool keys a context as
168
+ // sync does, and never writes a context-free entry for a text the project
169
+ // has only with a context.
170
+ export { tmSourceText, CONTEXT_SEPARATOR } from './lib/tm-evict.js';
171
+ export { sourceTextContexts } from './lib/source-contexts.js';
111
172
 
112
173
  // ── XLIFF interchange ──────────────────────────────────────────────
113
174
  export {
package/lib/api-key.js CHANGED
@@ -83,22 +83,35 @@ function loadApiKey(config, cwd) {
83
83
  * @returns {string|null} The variable value, or null if not found
84
84
  */
85
85
  function getEnvOrFileVar(keyName, cwd = process.cwd()) {
86
+ return findEnvOrFileVar(keyName, cwd)?.value ?? null;
87
+ }
88
+
89
+ /**
90
+ * getEnvOrFileVar, saying WHERE the value was found — for messages that
91
+ * must name the setting a run used (e.g. the endpoint a local method could
92
+ * not reach, and whether it came from the shell or a .env file).
93
+ *
94
+ * @param {string} keyName - Name of the environment variable
95
+ * @param {string} [cwd=process.cwd()] - Project root directory
96
+ * @returns {{ value: string, origin: 'environment'|'.env.local'|'.env', file: string|null }|null}
97
+ */
98
+ function findEnvOrFileVar(keyName, cwd = process.cwd()) {
86
99
  // Priority 1: Check environment directly. Normalize identically to file
87
100
  // values (trim + strip surrounding quotes) so a quoted/padded exported secret
88
101
  // never reaches an Authorization header verbatim.
89
102
  if (process.env[keyName]) {
90
- return normalizeKeyValue(process.env[keyName]);
103
+ return { value: normalizeKeyValue(process.env[keyName]), origin: 'environment', file: null };
91
104
  }
92
105
 
93
106
  // Priority 2: .env.local (Vercel/Next.js convention)
94
107
  const envLocalPath = path.join(cwd, '.env.local');
95
108
  const fromLocal = readKeyFromFile(envLocalPath, keyName);
96
- if (fromLocal) return fromLocal;
109
+ if (fromLocal) return { value: fromLocal, origin: '.env.local', file: envLocalPath };
97
110
 
98
111
  // Priority 3: .env
99
112
  const envPath = path.join(cwd, '.env');
100
113
  const fromEnv = readKeyFromFile(envPath, keyName);
101
- if (fromEnv) return fromEnv;
114
+ if (fromEnv) return { value: fromEnv, origin: '.env', file: envPath };
102
115
 
103
116
  return null;
104
117
  }
@@ -124,4 +137,4 @@ function readKeyFromFile(filePath, targetKey) {
124
137
  return null;
125
138
  }
126
139
 
127
- export { loadApiKey, parseEnvLine, getEnvOrFileVar };
140
+ export { loadApiKey, parseEnvLine, getEnvOrFileVar, findEnvOrFileVar };