champollion 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/README.md +52 -37
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +83 -36
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +51 -3
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +289 -88
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1689 -87
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +649 -130
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +16 -10
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +196 -14
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +270 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1127 -344
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +197 -38
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +726 -78
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +965 -253
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +964 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +571 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +252 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +68 -128
  70. package/lib/methods/llm.js +80 -31
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +11 -0
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +193 -106
  100. package/lib/seal.mjs +6 -5
  101. package/lib/sealed-qualifier.mjs +2 -2
  102. package/lib/segment.js +2 -1
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +880 -17
  117. package/lib/verify.js +1296 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +3 -2
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/DATA-SOVEREIGNTY.md +19 -20
  123. package/shared/LANGUAGE-CARD-FIELDS.md +1 -1
  124. package/shared/cards-fallback.json +1 -1
  125. package/shared/catalogue/card-config.json +1 -1
  126. package/shared/curated-orthography-conventions.json +26 -8
  127. package/shared/docent/faq.en.json +14 -16
  128. package/shared/docent/system-prompt.md +17 -19
  129. package/shared/explainers/tc-features.json +15 -15
  130. package/shared/gettext-plural-forms.json +45 -0
  131. package/shared/human-services.json +1 -1
  132. package/shared/method-registry.json +2 -0
  133. package/shared/metric-registry.json +96 -18
  134. package/shared/schemas/champollion-plugin.schema.json +4 -0
  135. package/shared/schemas/corpora-card.schema.json +20 -10
  136. package/shared/schemas/human-services.schema.json +2 -2
  137. package/shared/schemas/language-card.schema.json +1 -1
  138. package/shared/schemas/method-card.schema.json +1 -1
  139. package/shared/schemas/method-index-record.schema.json +67 -0
  140. package/shared/schemas/method-registry.schema.json +4 -0
  141. package/shared/schemas/metric-registry.schema.json +55 -1
  142. package/shared/docent/corpus.json +0 -11333
package/lib/scripts.js CHANGED
@@ -12,9 +12,11 @@
12
12
  * - Japanese: Romaji → Hiragana/Katakana
13
13
  * - Hindi: Romanized → Devanagari
14
14
  *
15
- * All converters here are DETERMINISTIC — no LLM needed, pure lookup tables.
16
- * They run as a post-translation hook: translate in working script, then
17
- * convert to display script.
15
+ * All converters here are DETERMINISTIC — no LLM needed: lookup tables, or a
16
+ * community-maintained converter where one exists (Plains Cree). They run as a
17
+ * post-translation hook: translate in working script, then convert to display
18
+ * script — and only over the spans of a string that are language, never over
19
+ * placeholders, ICU syntax, tags or URLs (see splitConvertible).
18
20
  *
19
21
  * ADDING A NEW CONVERTER:
20
22
  * 1. Add the conversion map below
@@ -23,115 +25,53 @@
23
25
  * 4. Add the `scripts` field to the language's register entry in registers.js
24
26
  */
25
27
 
28
+ import creeSroSyllabics from 'cree-sro-syllabics';
29
+
26
30
  // -----------------------------------------------------------------
27
31
  // Plains Cree: SRO → Syllabics
28
32
  // -----------------------------------------------------------------
29
33
 
30
34
  /**
31
- * SRO to Cree Syllabics conversion table.
32
- *
33
- * This is the standard mapping used by the University of Alberta's
34
- * ALTLab and documented in Wolvengrey's Cree: Words dictionary.
35
+ * Plains Cree SRO ⇄ syllabics is delegated to `cree-sro-syllabics` (MIT;
36
+ * Eddie Antonio Santos, NRC — the converter ALTLab's itwêwina uses). The
37
+ * hand-written table this replaced mapped 62 of the 64 consonant+w+vowel
38
+ * syllables to the wrong glyph and had no bare w-series at all, so
39
+ * "nêhiyawêwin" came out as ᓀᐦᐃᔭᐤᐁᐤᐃᐣ instead of ᓀᐦᐃᔭᐍᐏᐣ, and no test
40
+ * checked a single real spelling. Cree spelling is the community converter's
41
+ * call; this module only decides which spans of a UI string it may touch
42
+ * (see splitConvertible / convertScript).
35
43
  *
36
- * The mapping is context-sensitive: consonant+vowel combinations map
37
- * to specific syllabic characters, while standalone consonants use
38
- * finals (small superscript forms).
39
- *
40
- * ORDER MATTERS: Longer sequences must be matched before shorter ones
41
- * (e.g., "twê" before "tw" before "t").
44
+ * Library conventions are kept as they are: a hyphen between preverb and
45
+ * verb becomes U+202F NARROW NO-BREAK SPACE, and a full stop after a syllabic
46
+ * becomes ᙮ (U+166E CANADIAN SYLLABICS FULL STOP).
42
47
  */
43
- const SRO_TO_SYLLABICS_MAP = [
44
- // Long vowels with w-glide (must come before short vowel w-glide)
45
- ['pwê', 'ᐻ'], ['pwî', 'ᐽ'], ['pwô', 'ᐿ'], ['pwâ', 'ᑁ'],
46
- ['twê', 'ᑗ'], ['twî', 'ᑙ'], ['twô', 'ᑛ'], ['twâ', 'ᑝ'],
47
- ['kwê', 'ᑵ'], ['kwî', 'ᑷ'], ['kwô', 'ᑹ'], ['kwâ', 'ᑻ'],
48
- ['cwê', 'ᒑ'], ['cwî', 'ᒓ'], ['cwô', 'ᒕ'], ['cwâ', 'ᒗ'],
49
- ['mwê', 'ᒫ'], ['mwî', 'ᒭ'], ['mwô', 'ᒯ'], ['mwâ', 'ᒱ'],
50
- ['nwê', 'ᓇ'], ['nwî', 'ᓉ'], ['nwô', 'ᓋ'], ['nwâ', 'ᓍ'],
51
- ['swê', 'ᓭ'], ['swî', 'ᓯ'], ['swô', 'ᓱ'], ['swâ', 'ᓳ'],
52
- ['ywê', 'ᔋ'], ['ywî', 'ᔍ'], ['ywô', 'ᔏ'], ['ywâ', 'ᔑ'],
53
-
54
- // Short vowels with w-glide
55
- ['pwe', 'ᐺ'], ['pwi', 'ᐼ'], ['pwo', 'ᐾ'], ['pwa', 'ᑀ'],
56
- ['twe', 'ᑖ'], ['twi', 'ᑘ'], ['two', 'ᑚ'], ['twa', 'ᑜ'],
57
- ['kwe', 'ᑴ'], ['kwi', 'ᑶ'], ['kwo', 'ᑸ'], ['kwa', 'ᑺ'],
58
- ['cwe', 'ᒐ'], ['cwi', 'ᒒ'], ['cwo', 'ᒔ'], ['cwa', 'ᒖ'],
59
- ['mwe', 'ᒪ'], ['mwi', 'ᒬ'], ['mwo', 'ᒮ'], ['mwa', 'ᒰ'],
60
- ['nwe', 'ᓈ'], ['nwi', 'ᓊ'], ['nwo', 'ᓌ'], ['nwa', 'ᓎ'],
61
- ['swe', 'ᓬ'], ['swi', 'ᓮ'], ['swo', 'ᓰ'], ['swa', 'ᓲ'],
62
- ['ywe', 'ᔊ'], ['ywi', 'ᔌ'], ['ywo', 'ᔎ'], ['ywa', 'ᔐ'],
63
-
64
- // Long vowels (macron forms — these must come before short vowels)
65
- ['pê', 'ᐯ'], ['pî', 'ᐲ'], ['pô', 'ᐴ'], ['pâ', 'ᐹ'],
66
- ['tê', 'ᑌ'], ['tî', 'ᑏ'], ['tô', 'ᑑ'], ['tâ', 'ᑖ'],
67
- ['kê', 'ᑫ'], ['kî', 'ᑮ'], ['kô', 'ᑰ'], ['kâ', 'ᑳ'],
68
- ['cê', 'ᒉ'], ['cî', 'ᒌ'], ['cô', 'ᒎ'], ['câ', 'ᒑ'],
69
- ['mê', 'ᒣ'], ['mî', 'ᒦ'], ['mô', 'ᒨ'], ['mâ', 'ᒫ'],
70
- ['nê', 'ᓀ'], ['nî', 'ᓃ'], ['nô', 'ᓅ'], ['nâ', 'ᓈ'],
71
- ['sê', 'ᓭ'], ['sî', 'ᓰ'], ['sô', 'ᓲ'], ['sâ', 'ᓵ'],
72
- ['yê', 'ᔦ'], ['yî', 'ᔩ'], ['yô', 'ᔫ'], ['yâ', 'ᔮ'],
73
-
74
- // Short vowels (consonant+vowel)
75
- ['pe', 'ᐯ'], ['pi', 'ᐱ'], ['po', 'ᐳ'], ['pa', 'ᐸ'],
76
- ['te', 'ᑌ'], ['ti', 'ᑎ'], ['to', 'ᑐ'], ['ta', 'ᑕ'],
77
- ['ke', 'ᑫ'], ['ki', 'ᑭ'], ['ko', 'ᑯ'], ['ka', 'ᑲ'],
78
- ['ce', 'ᒉ'], ['ci', 'ᒋ'], ['co', 'ᒍ'], ['ca', 'ᒐ'],
79
- ['me', 'ᒣ'], ['mi', 'ᒥ'], ['mo', 'ᒧ'], ['ma', 'ᒪ'],
80
- ['ne', 'ᓀ'], ['ni', 'ᓂ'], ['no', 'ᓄ'], ['na', 'ᓇ'],
81
- ['se', 'ᓭ'], ['si', 'ᓯ'], ['so', 'ᓱ'], ['sa', 'ᓴ'],
82
- ['ye', 'ᔦ'], ['yi', 'ᔨ'], ['yo', 'ᔪ'], ['ya', 'ᔭ'],
83
-
84
- // Standalone vowels (long first)
85
- ['ê', 'ᐁ'], ['î', 'ᐄ'], ['ô', 'ᐆ'], ['â', 'ᐋ'],
86
- ['e', 'ᐁ'], ['i', 'ᐃ'], ['o', 'ᐅ'], ['a', 'ᐊ'],
87
-
88
- // Digraphs (must come before single-char finals)
89
- ['th', 'ᖧ'],
90
-
91
- // Finals (standalone consonants — no following vowel)
92
- ['p', 'ᑊ'], ['t', 'ᐟ'], ['k', 'ᐠ'], ['c', 'ᐨ'],
93
- ['m', 'ᒼ'], ['n', 'ᐣ'], ['s', 'ᐢ'], ['y', 'ᐩ'],
94
-
95
- // Special characters
96
- ['h', 'ᐦ'], ['w', 'ᐤ'], ['l', 'ᓬ'], ['r', 'ᕒ'],
97
- ];
48
+ const { sro2syllabics, syllabics2sro } = creeSroSyllabics;
98
49
 
99
50
  /**
100
- * Convert SRO text to Cree Syllabics.
101
- *
102
- * This is a greedy left-to-right scan: at each position, try the longest
103
- * possible match first. Characters that don't match any pattern (spaces,
104
- * punctuation, numbers) pass through unchanged.
51
+ * Convert SRO text to Cree Syllabics. Letters the library does not recognise
52
+ * as SRO pass through unchanged; creeLeftoverLetters() reports them.
105
53
  *
106
54
  * @param {string} sro - SRO text to convert
107
55
  * @returns {string} Syllabics text
108
56
  */
109
57
  function sroToSyllabics(sro) {
110
- const input = sro.toLowerCase();
111
- let result = '';
112
- let i = 0;
58
+ return sro2syllabics(sro);
59
+ }
113
60
 
114
- while (i < input.length) {
115
- let matched = false;
61
+ /** The Unified Canadian Aboriginal Syllabics blocks (base and extended). */
62
+ const SYLLABICS_CHAR = /[\u1400-\u167F\u18B0-\u18FF]/u;
116
63
 
117
- // Try longest matches first (up to 3 characters)
118
- for (const [from, to] of SRO_TO_SYLLABICS_MAP) {
119
- if (input.startsWith(from, i)) {
120
- result += to;
121
- i += from.length;
122
- matched = true;
123
- break;
124
- }
125
- }
126
-
127
- // No match — pass character through (space, punctuation, etc.)
128
- if (!matched) {
129
- result += input[i];
130
- i++;
131
- }
64
+ /**
65
+ * Letters the Cree converter left unconverted, derived from its behaviour:
66
+ * convert, then report every letter in the output that is not syllabics.
67
+ * Coverage therefore cannot drift from what the library actually does.
68
+ */
69
+ function creeLeftoverLetters(text) {
70
+ const out = [];
71
+ for (const ch of sroToSyllabics(text)) {
72
+ if (/\p{L}/u.test(ch) && !SYLLABICS_CHAR.test(ch) && !out.includes(ch)) out.push(ch);
132
73
  }
133
-
134
- return result;
74
+ return out;
135
75
  }
136
76
 
137
77
  // -----------------------------------------------------------------
@@ -456,9 +396,13 @@ const SCRIPT_CONVERTERS = {
456
396
  fromScript: 'Latn',
457
397
  toScript: 'Cans',
458
398
  type: 'deterministic',
459
- map: SRO_TO_SYLLABICS_MAP,
399
+ // No table of our own: conversion, coverage and reversal all come from
400
+ // the cree-sro-syllabics library (see the Plains Cree section above).
401
+ map: null,
460
402
  puaRange: null,
461
403
  converter: sroToSyllabics,
404
+ leftoverLetters: creeLeftoverLetters,
405
+ reverse: syllabics2sro,
462
406
  },
463
407
  sr: {
464
408
  from: 'Latin',
@@ -693,9 +637,14 @@ function formatScriptChoiceError(localeCode, resolution) {
693
637
  const opts = resolution.choices
694
638
  .map(c => `"script": "${c.script}" (${c.label})`)
695
639
  .join(' or ');
640
+ // Show WHERE it goes: "for crk in champollion.config.json" sent people
641
+ // hunting (Round 1, Cree school). The pair entry works with either form of
642
+ // `languages`.
643
+ const first = resolution.choices[0]?.script ?? '<script>';
696
644
  return (
697
645
  `${localeCode} has more than one real orthography and Champollion will not pick one ` +
698
- `for a community. Set ${opts} for ${localeCode} in champollion.config.json.`
646
+ `for a community. Choose ${opts}, on the pair in champollion.config.json, e.g.\n` +
647
+ ` "pairs": { "<source>:${localeCode}": { "script": "${first}" } }`
699
648
  );
700
649
  }
701
650
 
@@ -815,6 +764,9 @@ function unmappedLetters(text, converterKey) {
815
764
  const conv = SCRIPT_CONVERTERS[converterKey];
816
765
  if (!conv || typeof text !== 'string' || text === '') return [];
817
766
 
767
+ // Library-backed converters report their own leftovers.
768
+ if (conv.leftoverLetters) return conv.leftoverLetters(text);
769
+
818
770
  // Kryptonian is arithmetic over A–Z with no table; every letter outside
819
771
  // the basic Latin alphabet is unmapped.
820
772
  if (!conv.map) {
@@ -830,9 +782,7 @@ function unmappedLetters(text, converterKey) {
830
782
  const sequences = mappedSequences(converterKey);
831
783
  // Converters that lowercase (or uppercase) their input before matching must
832
784
  // be probed in that same normalised form, or every capital reads as unmapped.
833
- const probe = conv.converter === latinToTengwar || conv.converter === sroToSyllabics
834
- ? text.toLowerCase()
835
- : text;
785
+ const probe = conv.converter === latinToTengwar ? text.toLowerCase() : text;
836
786
 
837
787
  const out = [];
838
788
  let i = 0;
@@ -871,6 +821,11 @@ function reverseScript(text, converterKey) {
871
821
 
872
822
  const caseLossy = conv.converter !== romanizationToPiqad;
873
823
 
824
+ // Library-backed converters bring their own inverse (Cree: syllabics2sro).
825
+ if (conv.reverse) {
826
+ return { reversed: conv.reverse(text), caseLossy, unreversed: [] };
827
+ }
828
+
874
829
  if (!conv.map) {
875
830
  // Kryptonian: U+E100–E119 → A–Z.
876
831
  const [lo, hi] = conv.puaRange;
@@ -916,13 +871,136 @@ function isPrivateUse(codePoint) {
916
871
  || (codePoint >= 0x100000 && codePoint <= 0x10FFFD);
917
872
  }
918
873
 
874
+ // Tokens inside ordinary message text that are code, not language: double-
875
+ // brace and ${…} placeholders, printf tokens, HTML/XML tags, URLs, e-mail.
876
+ const FLAT_PROTECTED = new RegExp([
877
+ String.raw`\{\{[^{}]*\}\}`,
878
+ String.raw`\$\{[^{}]*\}`,
879
+ String.raw`%(?:\d+\$)?[-+ #0]*\d*(?:\.\d+)?[sdifuxXoeEgGc@]`,
880
+ String.raw`%\([A-Za-z_][A-Za-z0-9_]*\)[sd]`,
881
+ String.raw`<\/?[A-Za-z][^<>]*>`,
882
+ String.raw`(?:https?:\/\/|www\.)[^\s<>{}"']+`,
883
+ String.raw`[\w.+-]+@[\w-]+\.[\w.-]+`,
884
+ ].join('|'), 'g');
885
+
886
+ const ICU_BRANCHING = new Set(['plural', 'select', 'selectordinal']);
887
+
888
+ /**
889
+ * Split a UI string into the spans a script converter may touch and the spans
890
+ * it must leave byte-for-byte.
891
+ *
892
+ * WHY: converters used to run over the whole value, so "tânisi, {name}!"
893
+ * became "ᑖᓂᓯ, {ᓇᒣ}!" and Serbian "{name}" became "{наме}" — a broken
894
+ * placeholder that crashes the app at render time and that nothing caught.
895
+ *
896
+ * Protected: ICU argument names, types, styles and plural/select selectors
897
+ * (the TEXT of each branch stays convertible, recursively), `#` in plural
898
+ * branches, and the FLAT_PROTECTED tokens. Concatenating the segments'
899
+ * `text` always reproduces the input exactly.
900
+ *
901
+ * @param {string} text
902
+ * @returns {{ text: string, convert: boolean }[]}
903
+ */
904
+ function splitConvertible(text) {
905
+ const segments = [];
906
+ const push = (s, convert) => {
907
+ if (!s) return;
908
+ const last = segments[segments.length - 1];
909
+ if (last && last.convert === convert) last.text += s;
910
+ else segments.push({ text: s, convert });
911
+ };
912
+ const pushText = (s) => {
913
+ let at = 0;
914
+ for (const m of s.matchAll(FLAT_PROTECTED)) {
915
+ push(s.slice(at, m.index), true);
916
+ push(m[0], false);
917
+ at = m.index + m[0].length;
918
+ }
919
+ push(s.slice(at), true);
920
+ };
921
+ // Index of the brace that closes the one at `i`, or -1.
922
+ const matchingBrace = (i) => {
923
+ let depth = 0;
924
+ for (let k = i; k < text.length; k++) {
925
+ if (text[k] === '{') depth++;
926
+ else if (text[k] === '}' && --depth === 0) return k;
927
+ }
928
+ return -1;
929
+ };
930
+
931
+ // Message text from `i` until the end or (inBranch) the branch's closing
932
+ // brace; returns the index of that brace, or text.length.
933
+ function message(i, inBranch) {
934
+ let chunk = '';
935
+ while (i < text.length) {
936
+ const ch = text[i];
937
+ if (ch === '}' && inBranch) break;
938
+ if (ch === '{') {
939
+ pushText(chunk); chunk = '';
940
+ i = argument(i);
941
+ continue;
942
+ }
943
+ if (ch === '#' && inBranch) {
944
+ pushText(chunk); chunk = '';
945
+ push('#', false); i++;
946
+ continue;
947
+ }
948
+ chunk += ch; i++;
949
+ }
950
+ pushText(chunk);
951
+ return i;
952
+ }
953
+
954
+ // An ICU argument starting at the brace at `i`; returns the index after it.
955
+ function argument(i) {
956
+ const close = matchingBrace(i);
957
+ if (close === -1) { push(text[i], false); return i + 1; } // stray brace
958
+ // {{mustache}} and ${…} (the "$" was already pushed as text) stay whole.
959
+ if (text[i + 1] === '{' || text[i - 1] === '$') {
960
+ push(text.slice(i, close + 1), false);
961
+ return close + 1;
962
+ }
963
+ // Greedy name: a lazy one lets the optional ", plural," group match nothing.
964
+ const head = /^\{[^,{}]*(?:,\s*([A-Za-z]+)\s*,?)?/.exec(text.slice(i, close + 1));
965
+ const type = head?.[1];
966
+ if (!type || !ICU_BRANCHING.has(type)) {
967
+ push(text.slice(i, close + 1), false); // {name}, {n, number}, {d, date, short}
968
+ return close + 1;
969
+ }
970
+ // plural / select: protect the header and every selector, convert branches.
971
+ let k = i + head[0].length;
972
+ push(text.slice(i, k), false);
973
+ while (k < close) {
974
+ const sel = /^\s*[^\s{}]*\s*/.exec(text.slice(k, close))[0]; // "one ", "=0", "offset:1 "
975
+ push(sel, false); k += sel.length;
976
+ if (text[k] === '{') {
977
+ push('{', false);
978
+ k = message(k + 1, true);
979
+ push('}', false);
980
+ k += 1;
981
+ } else if (sel === '') {
982
+ break; // malformed; the remainder is protected below
983
+ }
984
+ }
985
+ push(text.slice(k, close + 1), false);
986
+ return close + 1;
987
+ }
988
+
989
+ message(0, false);
990
+ return segments;
991
+ }
992
+
919
993
  /**
920
994
  * Convert text using the registered converter for a locale.
921
995
  *
996
+ * Only the spans splitConvertible() marks as language are converted;
997
+ * placeholders, ICU syntax, tags and URLs come back byte-for-byte.
998
+ *
922
999
  * `unmapped` lists letters the converter could not translate and passed
923
- * through as-is. A non-empty `unmapped` means the input was not valid text in
924
- * the converter's working script, and the output is a mix of both scripts —
925
- * callers must treat it as a failure rather than writing it out.
1000
+ * through as-is, counted over the converted spans only. A non-empty
1001
+ * `unmapped` means the input was not valid text in the converter's working
1002
+ * script, and the output is a mix of both scripts — callers must treat it as
1003
+ * a failure rather than writing it out.
926
1004
  *
927
1005
  * @param {string} text - Text in the source script
928
1006
  * @param {string} localeCode - Locale code (e.g., "crk", "sr")
@@ -933,11 +1011,19 @@ function convertScript(text, localeCode) {
933
1011
  if (!converter) {
934
1012
  return { converted: text, converterUsed: null, unmapped: [] };
935
1013
  }
1014
+ const converterUsed = `${converter.from} → ${converter.to}`;
1015
+ if (typeof text !== 'string' || text === '') {
1016
+ return { converted: converter.converter(text ?? ''), converterUsed, unmapped: [] };
1017
+ }
936
1018
 
1019
+ const segments = splitConvertible(text);
937
1020
  return {
938
- converted: converter.converter(text),
939
- converterUsed: `${converter.from} → ${converter.to}`,
940
- unmapped: unmappedLetters(text, localeCode),
1021
+ converted: segments.map((s) => (s.convert ? converter.converter(s.text) : s.text)).join(''),
1022
+ converterUsed,
1023
+ unmapped: unmappedLetters(
1024
+ segments.filter((s) => s.convert).map((s) => s.text).join(' '),
1025
+ localeCode,
1026
+ ),
941
1027
  };
942
1028
  }
943
1029
 
@@ -990,5 +1076,6 @@ export {
990
1076
  unmappedLetters,
991
1077
  reverseScript,
992
1078
  isPrivateUse,
1079
+ splitConvertible,
993
1080
  SCRIPT_CONVERTERS,
994
1081
  };
package/lib/seal.mjs CHANGED
@@ -2,7 +2,7 @@
2
2
  * seal.mjs — CLIENT-SIDE encryption for the SEALED exposure tier.
3
3
  *
4
4
  * This is the cryptographic front door for the zero-knowledge sovereign-eval
5
- * PoC (docs/governance/OCAP_MULTISIG_PLAN.md, mechanism M1). A community that
5
+ * PoC (community-custodian multisig plan, docs/governance, mechanism M1). A community that
6
6
  * chooses to make a held-out set centrally evaluable-on-demand encrypts it
7
7
  * **on their own machine, before a single byte leaves**, under a *threshold
8
8
  * public key* the platform cannot decrypt alone. Champollion ever only receives
@@ -10,9 +10,10 @@
10
10
  * • ciphertext (an off-git, off-allowlist artifact), and
11
11
  * • a CONTENT-FREE metadata card (cipher id + custodian group + digest + AAD).
12
12
  *
13
- * The plaintext sentences are never read, uploaded, or hosted — exactly the L1
14
- * never-host-content doctrine (the data-boundaries doctrine), strengthened: not just
15
- * "no plaintext in the repo" but "no plaintext anywhere we hold, ever."
13
+ * The plaintext sentences are read only on the author's machine, to encrypt
14
+ * them; they are never uploaded or hosted — exactly the L1 never-host-content
15
+ * doctrine (the data-boundaries doctrine), strengthened: not just "no
16
+ * plaintext in the repo" but "no plaintext anywhere we hold, ever."
16
17
  *
17
18
  * ── SCHEME ────────────────────────────────────────────────────────────────
18
19
  * A standard hybrid "sealed box" (libsodium crypto_box_seal shape), built only
@@ -47,7 +48,7 @@
47
48
  * format and the card block do NOT change under either model; that is the
48
49
  * whole point of the seam. Historical note: earlier docs named FROST here —
49
50
  * FROST is threshold *signing*, the wrong primitive for this decryption path;
50
- * see docs/governance/OCAP_MULTISIG_PLAN.md for the corrected framing.
51
+ * see the community-custodian multisig plan (docs/governance) for the corrected framing.
51
52
  *
52
53
  * @module seal
53
54
  */
@@ -3,7 +3,7 @@
3
3
  *
4
4
  * A sealed (community-controlled secret) test set is precious: every evaluation
5
5
  * against it leaks a little signal, and custodian approval is a real,
6
- * low-frequency act (M-of-N, docs/governance/OCAP_MULTISIG_PLAN.md M2). So a
6
+ * low-frequency act (M-of-N, community-custodian multisig plan M2, docs/governance). So a
7
7
  * method must EARN the right to even *propose* a sealed run by first clearing a
8
8
  * disjoint, fully public twin — the **qualifier**.
9
9
  *
@@ -33,7 +33,7 @@ import { gatePublicRegistration } from './corpus-registration.mjs';
33
33
 
34
34
  /**
35
35
  * Default qualifier clearance threshold. Expressed on the public leaderboard's
36
- * composite 0–100 scale; communities/contests may override per qualifier. This
36
+ * chrF++ 0–100 scale; communities/contests may override per qualifier. This
37
37
  * is a floor placeholder, not a claim about any specific pair's difficulty —
38
38
  * callers should set a calibrated threshold for real qualifiers.
39
39
  */
package/lib/segment.js CHANGED
@@ -30,6 +30,7 @@
30
30
  */
31
31
 
32
32
  import { PLACEHOLDER_PREFIX } from './content.js';
33
+ import { nameRules } from './name-rules.js';
33
34
 
34
35
  // Block separator: a newline followed by one or more blank lines
35
36
  // (whitespace-only lines count as blank — CommonMark treats them as
@@ -193,7 +194,7 @@ Rules:
193
194
  - Translate ALL human-readable text in every segment.
194
195
  - Preserve ALL Markdown formatting: headers (#), bold (**), italic (*), links, images, lists, blockquotes, tables, admonitions (:::), etc.${placeholderRule}
195
196
  - Preserve line breaks and structure WITHIN each segment.
196
- - Proper nouns, product names, and technical terms should remain in the source language.
197
+ ${nameRules(options.protectedTerms)}
197
198
  - Translate link text but preserve link URLs. For example: [Read more](url) → [Lire la suite](url)
198
199
  - Echo each ${SEGMENT_MARKER_PREFIX}N${SEGMENT_MARKER_SUFFIX} marker on its own line, EXACTLY as given, before its translated segment.
199
200
  - Do NOT merge, drop, reorder, renumber, or add segments.
package/lib/seo.js CHANGED
@@ -93,6 +93,22 @@ function generateHreflangTags({ baseUrl, urlPattern, inputLocale, locales, pageP
93
93
  return lines.join('\n');
94
94
  }
95
95
 
96
+ /**
97
+ * The target codes in `languages`, in either documented form — a list
98
+ * (["fr","de"]) or an object keyed by code ({ "fr": "formal-vous", … }, the
99
+ * form `champollion init` writes when it records each language's register).
100
+ * The object form used to be skipped here, so its locales got no hreflang.
101
+ *
102
+ * @param {object} config
103
+ * @returns {string[]}
104
+ */
105
+ function configLanguageCodes(config) {
106
+ const langs = config.languages;
107
+ if (Array.isArray(langs)) return [...langs];
108
+ if (langs && typeof langs === 'object') return Object.keys(langs);
109
+ return [];
110
+ }
111
+
96
112
  /**
97
113
  * Generate hreflang tags for all configured pages.
98
114
  *
@@ -108,9 +124,7 @@ function generateAllHreflangTags(config) {
108
124
 
109
125
  // Collect all target locales from config
110
126
  const locales = [];
111
- if (config.languages && Array.isArray(config.languages)) {
112
- locales.push(...config.languages);
113
- }
127
+ locales.push(...configLanguageCodes(config));
114
128
  if (config.pairs) {
115
129
  for (const [pairKey] of Object.entries(config.pairs)) {
116
130
  const { target } = parsePairKey(pairKey);
@@ -163,9 +177,7 @@ function generateSitemap(config) {
163
177
 
164
178
  // Collect all locales
165
179
  const targetLocales = [];
166
- if (config.languages && Array.isArray(config.languages)) {
167
- targetLocales.push(...config.languages);
168
- }
180
+ targetLocales.push(...configLanguageCodes(config));
169
181
  if (config.pairs) {
170
182
  for (const [pairKey] of Object.entries(config.pairs)) {
171
183
  const { target } = parsePairKey(pairKey);
@@ -224,9 +236,7 @@ function generateJsonLd(config) {
224
236
 
225
237
  // Collect all locales
226
238
  const targetLocales = [];
227
- if (config.languages && Array.isArray(config.languages)) {
228
- targetLocales.push(...config.languages);
229
- }
239
+ targetLocales.push(...configLanguageCodes(config));
230
240
  if (config.pairs) {
231
241
  for (const [pairKey] of Object.entries(config.pairs)) {
232
242
  const { target } = parsePairKey(pairKey);
package/lib/serve.js CHANGED
@@ -20,8 +20,10 @@
20
20
  *
21
21
  * NO FORKED TRANSLATION LOGIC:
22
22
  * Requests run through the SAME pipeline `sync` uses — resolveRuntime for
23
- * config/plugin/preflight resolution and translateAndValidate for the
24
- * TM partition → API call → deterministic quality gate → TM store sequence.
23
+ * config/plugin/preflight resolution and translateWithFallback for the
24
+ * TM partition → API call → deterministic quality gate → TM store sequence
25
+ * (plus the pair's fallback method, when one is configured, for what the
26
+ * pair's own method could not translate safely).
25
27
  * TM hits are served from cache at $0; gate failures come back as
26
28
  * structured per-key errors, never as silently degraded output.
27
29
  *
@@ -49,7 +51,8 @@ import { createRequire } from 'node:module';
49
51
  import { resolveConfig } from './config.js';
50
52
  import { resolveRuntime } from './sync.js';
51
53
  import { loadTM, saveTM, isTMDirty, partitionByTM, tmMethodKey } from './tm.js';
52
- import { translateAndValidate } from './translate-pair.js';
54
+ import { translateWithFallback } from './translate-pair.js';
55
+ import { createFallbackBudget, fallbackSummary } from './fallback.js';
53
56
  import { estimateCost, QUALITY_TIERS } from './pairs.js';
54
57
  import { getMethod } from './translate.js';
55
58
  import { isUnsafeKey } from './security.js';
@@ -502,7 +505,7 @@ async function handleTranslate(ctx, body, res) {
502
505
  const { misses } = partitionByTM(runtime.tm, payload, keyNames, targetLocale, tmKey);
503
506
  let estimate = null;
504
507
  if (misses.length > 0) {
505
- estimate = await estimateCost(misses.length, pairConfig);
508
+ estimate = await estimateCost(misses.length, pairConfig, { cwd: runtime.cwd });
506
509
  const estimated = estimate?.estimatedCost ?? null;
507
510
 
508
511
  if (maxCostPerRequest !== null || maxSessionCost !== null) {
@@ -525,26 +528,47 @@ async function handleTranslate(ctx, body, res) {
525
528
  }
526
529
  }
527
530
 
531
+ // ── The pair's fallback method, under the same caps ────────────────
532
+ // What the pair's own method cannot translate safely goes to its
533
+ // fallback (if configured). Under a cap, that batch is priced before it
534
+ // runs and must fit in what the primary's estimate left of the tighter of
535
+ // the per-request cap and the remaining session ceiling (lib/fallback.js).
536
+ const capsLeft = [
537
+ maxCostPerRequest,
538
+ maxSessionCost !== null ? maxSessionCost - session.spendUsd : null,
539
+ ].filter(v => v !== null);
540
+ const primaryEstimate = misses.length > 0 ? (estimate?.estimatedCost ?? 0) : 0;
541
+ const fallbackBudget = createFallbackBudget({
542
+ maxCost: capsLeft.length > 0 ? Math.min(...capsLeft) : null,
543
+ committed: primaryEstimate,
544
+ cwd: runtime.cwd,
545
+ });
546
+
528
547
  // ── The real pipeline: TM partition → API → quality gate → TM store ──
529
548
  let result;
530
549
  try {
531
- result = await translateAndValidate(keyNames, payload, pairConfig, pairKey, {
550
+ result = await translateWithFallback(keyNames, payload, pairConfig, pairKey, {
532
551
  apiKey: runtime.apiKey,
552
+ cwd: runtime.cwd,
533
553
  tm: runtime.tm,
534
554
  targetCode: targetLocale,
535
555
  onProgress: null,
556
+ budget: fallbackBudget,
536
557
  });
537
558
  } catch (err) {
538
559
  output.error(`[serve] ${pairKey}: pipeline error — ${err.message}`);
539
560
  return sendError(res, 500, 'internal_error', 'Translation pipeline failed. See server logs.');
540
561
  }
541
- if (result.apiCalled) output.progress(' done\n');
562
+ if (result.apiCalled) output.progressDone(`[serve] ${pairKey}`, 'done');
542
563
 
543
564
  // Session ledger: count the estimate for any request that actually hit
544
565
  // the upstream API (conservative — estimates deliberately err high).
545
566
  if (result.apiCalled && estimate?.estimatedCost != null) {
546
567
  session.spendUsd += estimate.estimatedCost;
547
568
  }
569
+ // Fallback batches approved under a cap carry their estimate too.
570
+ const fallbackSpend = fallbackBudget.committed - primaryEstimate;
571
+ if (fallbackSpend > 0) session.spendUsd += fallbackSpend;
548
572
 
549
573
  // Persist TM mutations (stores AND evictions) after every request so a
550
574
  // crash never loses paid, gate-validated translations. Atomic write.
@@ -616,6 +640,17 @@ async function handleTranslate(ctx, body, res) {
616
640
  costUsd = estimate.estimatedCost;
617
641
  costBasis = 'estimate';
618
642
  }
643
+ // A fallback that ran adds its own spend: priced when a cap made us price
644
+ // it, otherwise unknown — never silently left out of a known figure.
645
+ if (result.fallback?.attempted > 0) {
646
+ if (fallbackBudget.maxCost !== null && costUsd !== null) {
647
+ costUsd += fallbackSpend;
648
+ costBasis = 'estimate';
649
+ } else {
650
+ costUsd = null;
651
+ costBasis = 'unknown';
652
+ }
653
+ }
619
654
 
620
655
  const meta = {
621
656
  served_by: `champollion-serve/${CLI_VERSION}`,
@@ -629,6 +664,8 @@ async function handleTranslate(ctx, body, res) {
629
664
  translated: translatedCount,
630
665
  failed: errorCount,
631
666
  tm_hits: result.tmHitCount,
667
+ // The pair's fallback method and what it did (pairs with a fallback).
668
+ ...(result.fallback && { fallback: fallbackSummary(result.fallback) }),
632
669
  // Script conversion status for this pair: which script was written and
633
670
  // how many values stayed in the working script (unmapped letters).
634
671
  script_conversion: {