champollion 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -37
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +51 -3
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +289 -88
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +649 -130
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +16 -10
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +197 -38
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +193 -106
- package/lib/seal.mjs +6 -5
- package/lib/sealed-qualifier.mjs +2 -2
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +3 -2
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/DATA-SOVEREIGNTY.md +19 -20
- package/shared/LANGUAGE-CARD-FIELDS.md +1 -1
- package/shared/cards-fallback.json +1 -1
- package/shared/catalogue/card-config.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/docent/faq.en.json +14 -16
- package/shared/docent/system-prompt.md +17 -19
- package/shared/explainers/tc-features.json +15 -15
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/human-services.json +1 -1
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +20 -10
- package/shared/schemas/human-services.schema.json +2 -2
- package/shared/schemas/language-card.schema.json +1 -1
- package/shared/schemas/method-card.schema.json +1 -1
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11333
package/lib/scripts.js
CHANGED
|
@@ -12,9 +12,11 @@
|
|
|
12
12
|
* - Japanese: Romaji → Hiragana/Katakana
|
|
13
13
|
* - Hindi: Romanized → Devanagari
|
|
14
14
|
*
|
|
15
|
-
* All converters here are DETERMINISTIC — no LLM needed
|
|
16
|
-
*
|
|
17
|
-
* convert to display
|
|
15
|
+
* All converters here are DETERMINISTIC — no LLM needed: lookup tables, or a
|
|
16
|
+
* community-maintained converter where one exists (Plains Cree). They run as a
|
|
17
|
+
* post-translation hook: translate in working script, then convert to display
|
|
18
|
+
* script — and only over the spans of a string that are language, never over
|
|
19
|
+
* placeholders, ICU syntax, tags or URLs (see splitConvertible).
|
|
18
20
|
*
|
|
19
21
|
* ADDING A NEW CONVERTER:
|
|
20
22
|
* 1. Add the conversion map below
|
|
@@ -23,115 +25,53 @@
|
|
|
23
25
|
* 4. Add the `scripts` field to the language's register entry in registers.js
|
|
24
26
|
*/
|
|
25
27
|
|
|
28
|
+
import creeSroSyllabics from 'cree-sro-syllabics';
|
|
29
|
+
|
|
26
30
|
// -----------------------------------------------------------------
|
|
27
31
|
// Plains Cree: SRO → Syllabics
|
|
28
32
|
// -----------------------------------------------------------------
|
|
29
33
|
|
|
30
34
|
/**
|
|
31
|
-
* SRO
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
+
* Plains Cree SRO ⇄ syllabics is delegated to `cree-sro-syllabics` (MIT;
|
|
36
|
+
* Eddie Antonio Santos, NRC — the converter ALTLab's itwêwina uses). The
|
|
37
|
+
* hand-written table this replaced mapped 62 of the 64 consonant+w+vowel
|
|
38
|
+
* syllables to the wrong glyph and had no bare w-series at all, so
|
|
39
|
+
* "nêhiyawêwin" came out as ᓀᐦᐃᔭᐤᐁᐤᐃᐣ instead of ᓀᐦᐃᔭᐍᐏᐣ, and no test
|
|
40
|
+
* checked a single real spelling. Cree spelling is the community converter's
|
|
41
|
+
* call; this module only decides which spans of a UI string it may touch
|
|
42
|
+
* (see splitConvertible / convertScript).
|
|
35
43
|
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* ORDER MATTERS: Longer sequences must be matched before shorter ones
|
|
41
|
-
* (e.g., "twê" before "tw" before "t").
|
|
44
|
+
* Library conventions are kept as they are: a hyphen between preverb and
|
|
45
|
+
* verb becomes U+202F NARROW NO-BREAK SPACE, and a full stop after a syllabic
|
|
46
|
+
* becomes ᙮ (U+166E CANADIAN SYLLABICS FULL STOP).
|
|
42
47
|
*/
|
|
43
|
-
const
|
|
44
|
-
// Long vowels with w-glide (must come before short vowel w-glide)
|
|
45
|
-
['pwê', 'ᐻ'], ['pwî', 'ᐽ'], ['pwô', 'ᐿ'], ['pwâ', 'ᑁ'],
|
|
46
|
-
['twê', 'ᑗ'], ['twî', 'ᑙ'], ['twô', 'ᑛ'], ['twâ', 'ᑝ'],
|
|
47
|
-
['kwê', 'ᑵ'], ['kwî', 'ᑷ'], ['kwô', 'ᑹ'], ['kwâ', 'ᑻ'],
|
|
48
|
-
['cwê', 'ᒑ'], ['cwî', 'ᒓ'], ['cwô', 'ᒕ'], ['cwâ', 'ᒗ'],
|
|
49
|
-
['mwê', 'ᒫ'], ['mwî', 'ᒭ'], ['mwô', 'ᒯ'], ['mwâ', 'ᒱ'],
|
|
50
|
-
['nwê', 'ᓇ'], ['nwî', 'ᓉ'], ['nwô', 'ᓋ'], ['nwâ', 'ᓍ'],
|
|
51
|
-
['swê', 'ᓭ'], ['swî', 'ᓯ'], ['swô', 'ᓱ'], ['swâ', 'ᓳ'],
|
|
52
|
-
['ywê', 'ᔋ'], ['ywî', 'ᔍ'], ['ywô', 'ᔏ'], ['ywâ', 'ᔑ'],
|
|
53
|
-
|
|
54
|
-
// Short vowels with w-glide
|
|
55
|
-
['pwe', 'ᐺ'], ['pwi', 'ᐼ'], ['pwo', 'ᐾ'], ['pwa', 'ᑀ'],
|
|
56
|
-
['twe', 'ᑖ'], ['twi', 'ᑘ'], ['two', 'ᑚ'], ['twa', 'ᑜ'],
|
|
57
|
-
['kwe', 'ᑴ'], ['kwi', 'ᑶ'], ['kwo', 'ᑸ'], ['kwa', 'ᑺ'],
|
|
58
|
-
['cwe', 'ᒐ'], ['cwi', 'ᒒ'], ['cwo', 'ᒔ'], ['cwa', 'ᒖ'],
|
|
59
|
-
['mwe', 'ᒪ'], ['mwi', 'ᒬ'], ['mwo', 'ᒮ'], ['mwa', 'ᒰ'],
|
|
60
|
-
['nwe', 'ᓈ'], ['nwi', 'ᓊ'], ['nwo', 'ᓌ'], ['nwa', 'ᓎ'],
|
|
61
|
-
['swe', 'ᓬ'], ['swi', 'ᓮ'], ['swo', 'ᓰ'], ['swa', 'ᓲ'],
|
|
62
|
-
['ywe', 'ᔊ'], ['ywi', 'ᔌ'], ['ywo', 'ᔎ'], ['ywa', 'ᔐ'],
|
|
63
|
-
|
|
64
|
-
// Long vowels (macron forms — these must come before short vowels)
|
|
65
|
-
['pê', 'ᐯ'], ['pî', 'ᐲ'], ['pô', 'ᐴ'], ['pâ', 'ᐹ'],
|
|
66
|
-
['tê', 'ᑌ'], ['tî', 'ᑏ'], ['tô', 'ᑑ'], ['tâ', 'ᑖ'],
|
|
67
|
-
['kê', 'ᑫ'], ['kî', 'ᑮ'], ['kô', 'ᑰ'], ['kâ', 'ᑳ'],
|
|
68
|
-
['cê', 'ᒉ'], ['cî', 'ᒌ'], ['cô', 'ᒎ'], ['câ', 'ᒑ'],
|
|
69
|
-
['mê', 'ᒣ'], ['mî', 'ᒦ'], ['mô', 'ᒨ'], ['mâ', 'ᒫ'],
|
|
70
|
-
['nê', 'ᓀ'], ['nî', 'ᓃ'], ['nô', 'ᓅ'], ['nâ', 'ᓈ'],
|
|
71
|
-
['sê', 'ᓭ'], ['sî', 'ᓰ'], ['sô', 'ᓲ'], ['sâ', 'ᓵ'],
|
|
72
|
-
['yê', 'ᔦ'], ['yî', 'ᔩ'], ['yô', 'ᔫ'], ['yâ', 'ᔮ'],
|
|
73
|
-
|
|
74
|
-
// Short vowels (consonant+vowel)
|
|
75
|
-
['pe', 'ᐯ'], ['pi', 'ᐱ'], ['po', 'ᐳ'], ['pa', 'ᐸ'],
|
|
76
|
-
['te', 'ᑌ'], ['ti', 'ᑎ'], ['to', 'ᑐ'], ['ta', 'ᑕ'],
|
|
77
|
-
['ke', 'ᑫ'], ['ki', 'ᑭ'], ['ko', 'ᑯ'], ['ka', 'ᑲ'],
|
|
78
|
-
['ce', 'ᒉ'], ['ci', 'ᒋ'], ['co', 'ᒍ'], ['ca', 'ᒐ'],
|
|
79
|
-
['me', 'ᒣ'], ['mi', 'ᒥ'], ['mo', 'ᒧ'], ['ma', 'ᒪ'],
|
|
80
|
-
['ne', 'ᓀ'], ['ni', 'ᓂ'], ['no', 'ᓄ'], ['na', 'ᓇ'],
|
|
81
|
-
['se', 'ᓭ'], ['si', 'ᓯ'], ['so', 'ᓱ'], ['sa', 'ᓴ'],
|
|
82
|
-
['ye', 'ᔦ'], ['yi', 'ᔨ'], ['yo', 'ᔪ'], ['ya', 'ᔭ'],
|
|
83
|
-
|
|
84
|
-
// Standalone vowels (long first)
|
|
85
|
-
['ê', 'ᐁ'], ['î', 'ᐄ'], ['ô', 'ᐆ'], ['â', 'ᐋ'],
|
|
86
|
-
['e', 'ᐁ'], ['i', 'ᐃ'], ['o', 'ᐅ'], ['a', 'ᐊ'],
|
|
87
|
-
|
|
88
|
-
// Digraphs (must come before single-char finals)
|
|
89
|
-
['th', 'ᖧ'],
|
|
90
|
-
|
|
91
|
-
// Finals (standalone consonants — no following vowel)
|
|
92
|
-
['p', 'ᑊ'], ['t', 'ᐟ'], ['k', 'ᐠ'], ['c', 'ᐨ'],
|
|
93
|
-
['m', 'ᒼ'], ['n', 'ᐣ'], ['s', 'ᐢ'], ['y', 'ᐩ'],
|
|
94
|
-
|
|
95
|
-
// Special characters
|
|
96
|
-
['h', 'ᐦ'], ['w', 'ᐤ'], ['l', 'ᓬ'], ['r', 'ᕒ'],
|
|
97
|
-
];
|
|
48
|
+
const { sro2syllabics, syllabics2sro } = creeSroSyllabics;
|
|
98
49
|
|
|
99
50
|
/**
|
|
100
|
-
* Convert SRO text to Cree Syllabics.
|
|
101
|
-
*
|
|
102
|
-
* This is a greedy left-to-right scan: at each position, try the longest
|
|
103
|
-
* possible match first. Characters that don't match any pattern (spaces,
|
|
104
|
-
* punctuation, numbers) pass through unchanged.
|
|
51
|
+
* Convert SRO text to Cree Syllabics. Letters the library does not recognise
|
|
52
|
+
* as SRO pass through unchanged; creeLeftoverLetters() reports them.
|
|
105
53
|
*
|
|
106
54
|
* @param {string} sro - SRO text to convert
|
|
107
55
|
* @returns {string} Syllabics text
|
|
108
56
|
*/
|
|
109
57
|
function sroToSyllabics(sro) {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
let i = 0;
|
|
58
|
+
return sro2syllabics(sro);
|
|
59
|
+
}
|
|
113
60
|
|
|
114
|
-
|
|
115
|
-
|
|
61
|
+
/** The Unified Canadian Aboriginal Syllabics blocks (base and extended). */
|
|
62
|
+
const SYLLABICS_CHAR = /[\u1400-\u167F\u18B0-\u18FF]/u;
|
|
116
63
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// No match — pass character through (space, punctuation, etc.)
|
|
128
|
-
if (!matched) {
|
|
129
|
-
result += input[i];
|
|
130
|
-
i++;
|
|
131
|
-
}
|
|
64
|
+
/**
|
|
65
|
+
* Letters the Cree converter left unconverted, derived from its behaviour:
|
|
66
|
+
* convert, then report every letter in the output that is not syllabics.
|
|
67
|
+
* Coverage therefore cannot drift from what the library actually does.
|
|
68
|
+
*/
|
|
69
|
+
function creeLeftoverLetters(text) {
|
|
70
|
+
const out = [];
|
|
71
|
+
for (const ch of sroToSyllabics(text)) {
|
|
72
|
+
if (/\p{L}/u.test(ch) && !SYLLABICS_CHAR.test(ch) && !out.includes(ch)) out.push(ch);
|
|
132
73
|
}
|
|
133
|
-
|
|
134
|
-
return result;
|
|
74
|
+
return out;
|
|
135
75
|
}
|
|
136
76
|
|
|
137
77
|
// -----------------------------------------------------------------
|
|
@@ -456,9 +396,13 @@ const SCRIPT_CONVERTERS = {
|
|
|
456
396
|
fromScript: 'Latn',
|
|
457
397
|
toScript: 'Cans',
|
|
458
398
|
type: 'deterministic',
|
|
459
|
-
|
|
399
|
+
// No table of our own: conversion, coverage and reversal all come from
|
|
400
|
+
// the cree-sro-syllabics library (see the Plains Cree section above).
|
|
401
|
+
map: null,
|
|
460
402
|
puaRange: null,
|
|
461
403
|
converter: sroToSyllabics,
|
|
404
|
+
leftoverLetters: creeLeftoverLetters,
|
|
405
|
+
reverse: syllabics2sro,
|
|
462
406
|
},
|
|
463
407
|
sr: {
|
|
464
408
|
from: 'Latin',
|
|
@@ -693,9 +637,14 @@ function formatScriptChoiceError(localeCode, resolution) {
|
|
|
693
637
|
const opts = resolution.choices
|
|
694
638
|
.map(c => `"script": "${c.script}" (${c.label})`)
|
|
695
639
|
.join(' or ');
|
|
640
|
+
// Show WHERE it goes: "for crk in champollion.config.json" sent people
|
|
641
|
+
// hunting (Round 1, Cree school). The pair entry works with either form of
|
|
642
|
+
// `languages`.
|
|
643
|
+
const first = resolution.choices[0]?.script ?? '<script>';
|
|
696
644
|
return (
|
|
697
645
|
`${localeCode} has more than one real orthography and Champollion will not pick one ` +
|
|
698
|
-
`for a community.
|
|
646
|
+
`for a community. Choose ${opts}, on the pair in champollion.config.json, e.g.\n` +
|
|
647
|
+
` "pairs": { "<source>:${localeCode}": { "script": "${first}" } }`
|
|
699
648
|
);
|
|
700
649
|
}
|
|
701
650
|
|
|
@@ -815,6 +764,9 @@ function unmappedLetters(text, converterKey) {
|
|
|
815
764
|
const conv = SCRIPT_CONVERTERS[converterKey];
|
|
816
765
|
if (!conv || typeof text !== 'string' || text === '') return [];
|
|
817
766
|
|
|
767
|
+
// Library-backed converters report their own leftovers.
|
|
768
|
+
if (conv.leftoverLetters) return conv.leftoverLetters(text);
|
|
769
|
+
|
|
818
770
|
// Kryptonian is arithmetic over A–Z with no table; every letter outside
|
|
819
771
|
// the basic Latin alphabet is unmapped.
|
|
820
772
|
if (!conv.map) {
|
|
@@ -830,9 +782,7 @@ function unmappedLetters(text, converterKey) {
|
|
|
830
782
|
const sequences = mappedSequences(converterKey);
|
|
831
783
|
// Converters that lowercase (or uppercase) their input before matching must
|
|
832
784
|
// be probed in that same normalised form, or every capital reads as unmapped.
|
|
833
|
-
const probe = conv.converter === latinToTengwar
|
|
834
|
-
? text.toLowerCase()
|
|
835
|
-
: text;
|
|
785
|
+
const probe = conv.converter === latinToTengwar ? text.toLowerCase() : text;
|
|
836
786
|
|
|
837
787
|
const out = [];
|
|
838
788
|
let i = 0;
|
|
@@ -871,6 +821,11 @@ function reverseScript(text, converterKey) {
|
|
|
871
821
|
|
|
872
822
|
const caseLossy = conv.converter !== romanizationToPiqad;
|
|
873
823
|
|
|
824
|
+
// Library-backed converters bring their own inverse (Cree: syllabics2sro).
|
|
825
|
+
if (conv.reverse) {
|
|
826
|
+
return { reversed: conv.reverse(text), caseLossy, unreversed: [] };
|
|
827
|
+
}
|
|
828
|
+
|
|
874
829
|
if (!conv.map) {
|
|
875
830
|
// Kryptonian: U+E100–E119 → A–Z.
|
|
876
831
|
const [lo, hi] = conv.puaRange;
|
|
@@ -916,13 +871,136 @@ function isPrivateUse(codePoint) {
|
|
|
916
871
|
|| (codePoint >= 0x100000 && codePoint <= 0x10FFFD);
|
|
917
872
|
}
|
|
918
873
|
|
|
874
|
+
// Tokens inside ordinary message text that are code, not language: double-
|
|
875
|
+
// brace and ${…} placeholders, printf tokens, HTML/XML tags, URLs, e-mail.
|
|
876
|
+
const FLAT_PROTECTED = new RegExp([
|
|
877
|
+
String.raw`\{\{[^{}]*\}\}`,
|
|
878
|
+
String.raw`\$\{[^{}]*\}`,
|
|
879
|
+
String.raw`%(?:\d+\$)?[-+ #0]*\d*(?:\.\d+)?[sdifuxXoeEgGc@]`,
|
|
880
|
+
String.raw`%\([A-Za-z_][A-Za-z0-9_]*\)[sd]`,
|
|
881
|
+
String.raw`<\/?[A-Za-z][^<>]*>`,
|
|
882
|
+
String.raw`(?:https?:\/\/|www\.)[^\s<>{}"']+`,
|
|
883
|
+
String.raw`[\w.+-]+@[\w-]+\.[\w.-]+`,
|
|
884
|
+
].join('|'), 'g');
|
|
885
|
+
|
|
886
|
+
const ICU_BRANCHING = new Set(['plural', 'select', 'selectordinal']);
|
|
887
|
+
|
|
888
|
+
/**
|
|
889
|
+
* Split a UI string into the spans a script converter may touch and the spans
|
|
890
|
+
* it must leave byte-for-byte.
|
|
891
|
+
*
|
|
892
|
+
* WHY: converters used to run over the whole value, so "tânisi, {name}!"
|
|
893
|
+
* became "ᑖᓂᓯ, {ᓇᒣ}!" and Serbian "{name}" became "{наме}" — a broken
|
|
894
|
+
* placeholder that crashes the app at render time and that nothing caught.
|
|
895
|
+
*
|
|
896
|
+
* Protected: ICU argument names, types, styles and plural/select selectors
|
|
897
|
+
* (the TEXT of each branch stays convertible, recursively), `#` in plural
|
|
898
|
+
* branches, and the FLAT_PROTECTED tokens. Concatenating the segments'
|
|
899
|
+
* `text` always reproduces the input exactly.
|
|
900
|
+
*
|
|
901
|
+
* @param {string} text
|
|
902
|
+
* @returns {{ text: string, convert: boolean }[]}
|
|
903
|
+
*/
|
|
904
|
+
function splitConvertible(text) {
|
|
905
|
+
const segments = [];
|
|
906
|
+
const push = (s, convert) => {
|
|
907
|
+
if (!s) return;
|
|
908
|
+
const last = segments[segments.length - 1];
|
|
909
|
+
if (last && last.convert === convert) last.text += s;
|
|
910
|
+
else segments.push({ text: s, convert });
|
|
911
|
+
};
|
|
912
|
+
const pushText = (s) => {
|
|
913
|
+
let at = 0;
|
|
914
|
+
for (const m of s.matchAll(FLAT_PROTECTED)) {
|
|
915
|
+
push(s.slice(at, m.index), true);
|
|
916
|
+
push(m[0], false);
|
|
917
|
+
at = m.index + m[0].length;
|
|
918
|
+
}
|
|
919
|
+
push(s.slice(at), true);
|
|
920
|
+
};
|
|
921
|
+
// Index of the brace that closes the one at `i`, or -1.
|
|
922
|
+
const matchingBrace = (i) => {
|
|
923
|
+
let depth = 0;
|
|
924
|
+
for (let k = i; k < text.length; k++) {
|
|
925
|
+
if (text[k] === '{') depth++;
|
|
926
|
+
else if (text[k] === '}' && --depth === 0) return k;
|
|
927
|
+
}
|
|
928
|
+
return -1;
|
|
929
|
+
};
|
|
930
|
+
|
|
931
|
+
// Message text from `i` until the end or (inBranch) the branch's closing
|
|
932
|
+
// brace; returns the index of that brace, or text.length.
|
|
933
|
+
function message(i, inBranch) {
|
|
934
|
+
let chunk = '';
|
|
935
|
+
while (i < text.length) {
|
|
936
|
+
const ch = text[i];
|
|
937
|
+
if (ch === '}' && inBranch) break;
|
|
938
|
+
if (ch === '{') {
|
|
939
|
+
pushText(chunk); chunk = '';
|
|
940
|
+
i = argument(i);
|
|
941
|
+
continue;
|
|
942
|
+
}
|
|
943
|
+
if (ch === '#' && inBranch) {
|
|
944
|
+
pushText(chunk); chunk = '';
|
|
945
|
+
push('#', false); i++;
|
|
946
|
+
continue;
|
|
947
|
+
}
|
|
948
|
+
chunk += ch; i++;
|
|
949
|
+
}
|
|
950
|
+
pushText(chunk);
|
|
951
|
+
return i;
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
// An ICU argument starting at the brace at `i`; returns the index after it.
|
|
955
|
+
function argument(i) {
|
|
956
|
+
const close = matchingBrace(i);
|
|
957
|
+
if (close === -1) { push(text[i], false); return i + 1; } // stray brace
|
|
958
|
+
// {{mustache}} and ${…} (the "$" was already pushed as text) stay whole.
|
|
959
|
+
if (text[i + 1] === '{' || text[i - 1] === '$') {
|
|
960
|
+
push(text.slice(i, close + 1), false);
|
|
961
|
+
return close + 1;
|
|
962
|
+
}
|
|
963
|
+
// Greedy name: a lazy one lets the optional ", plural," group match nothing.
|
|
964
|
+
const head = /^\{[^,{}]*(?:,\s*([A-Za-z]+)\s*,?)?/.exec(text.slice(i, close + 1));
|
|
965
|
+
const type = head?.[1];
|
|
966
|
+
if (!type || !ICU_BRANCHING.has(type)) {
|
|
967
|
+
push(text.slice(i, close + 1), false); // {name}, {n, number}, {d, date, short}
|
|
968
|
+
return close + 1;
|
|
969
|
+
}
|
|
970
|
+
// plural / select: protect the header and every selector, convert branches.
|
|
971
|
+
let k = i + head[0].length;
|
|
972
|
+
push(text.slice(i, k), false);
|
|
973
|
+
while (k < close) {
|
|
974
|
+
const sel = /^\s*[^\s{}]*\s*/.exec(text.slice(k, close))[0]; // "one ", "=0", "offset:1 "
|
|
975
|
+
push(sel, false); k += sel.length;
|
|
976
|
+
if (text[k] === '{') {
|
|
977
|
+
push('{', false);
|
|
978
|
+
k = message(k + 1, true);
|
|
979
|
+
push('}', false);
|
|
980
|
+
k += 1;
|
|
981
|
+
} else if (sel === '') {
|
|
982
|
+
break; // malformed; the remainder is protected below
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
push(text.slice(k, close + 1), false);
|
|
986
|
+
return close + 1;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
message(0, false);
|
|
990
|
+
return segments;
|
|
991
|
+
}
|
|
992
|
+
|
|
919
993
|
/**
|
|
920
994
|
* Convert text using the registered converter for a locale.
|
|
921
995
|
*
|
|
996
|
+
* Only the spans splitConvertible() marks as language are converted;
|
|
997
|
+
* placeholders, ICU syntax, tags and URLs come back byte-for-byte.
|
|
998
|
+
*
|
|
922
999
|
* `unmapped` lists letters the converter could not translate and passed
|
|
923
|
-
* through as-is
|
|
924
|
-
*
|
|
925
|
-
*
|
|
1000
|
+
* through as-is, counted over the converted spans only. A non-empty
|
|
1001
|
+
* `unmapped` means the input was not valid text in the converter's working
|
|
1002
|
+
* script, and the output is a mix of both scripts — callers must treat it as
|
|
1003
|
+
* a failure rather than writing it out.
|
|
926
1004
|
*
|
|
927
1005
|
* @param {string} text - Text in the source script
|
|
928
1006
|
* @param {string} localeCode - Locale code (e.g., "crk", "sr")
|
|
@@ -933,11 +1011,19 @@ function convertScript(text, localeCode) {
|
|
|
933
1011
|
if (!converter) {
|
|
934
1012
|
return { converted: text, converterUsed: null, unmapped: [] };
|
|
935
1013
|
}
|
|
1014
|
+
const converterUsed = `${converter.from} → ${converter.to}`;
|
|
1015
|
+
if (typeof text !== 'string' || text === '') {
|
|
1016
|
+
return { converted: converter.converter(text ?? ''), converterUsed, unmapped: [] };
|
|
1017
|
+
}
|
|
936
1018
|
|
|
1019
|
+
const segments = splitConvertible(text);
|
|
937
1020
|
return {
|
|
938
|
-
converted: converter.converter(text),
|
|
939
|
-
converterUsed
|
|
940
|
-
unmapped: unmappedLetters(
|
|
1021
|
+
converted: segments.map((s) => (s.convert ? converter.converter(s.text) : s.text)).join(''),
|
|
1022
|
+
converterUsed,
|
|
1023
|
+
unmapped: unmappedLetters(
|
|
1024
|
+
segments.filter((s) => s.convert).map((s) => s.text).join(' '),
|
|
1025
|
+
localeCode,
|
|
1026
|
+
),
|
|
941
1027
|
};
|
|
942
1028
|
}
|
|
943
1029
|
|
|
@@ -990,5 +1076,6 @@ export {
|
|
|
990
1076
|
unmappedLetters,
|
|
991
1077
|
reverseScript,
|
|
992
1078
|
isPrivateUse,
|
|
1079
|
+
splitConvertible,
|
|
993
1080
|
SCRIPT_CONVERTERS,
|
|
994
1081
|
};
|
package/lib/seal.mjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* seal.mjs — CLIENT-SIDE encryption for the SEALED exposure tier.
|
|
3
3
|
*
|
|
4
4
|
* This is the cryptographic front door for the zero-knowledge sovereign-eval
|
|
5
|
-
* PoC (docs/governance
|
|
5
|
+
* PoC (community-custodian multisig plan, docs/governance, mechanism M1). A community that
|
|
6
6
|
* chooses to make a held-out set centrally evaluable-on-demand encrypts it
|
|
7
7
|
* **on their own machine, before a single byte leaves**, under a *threshold
|
|
8
8
|
* public key* the platform cannot decrypt alone. Champollion ever only receives
|
|
@@ -10,9 +10,10 @@
|
|
|
10
10
|
* • ciphertext (an off-git, off-allowlist artifact), and
|
|
11
11
|
* • a CONTENT-FREE metadata card (cipher id + custodian group + digest + AAD).
|
|
12
12
|
*
|
|
13
|
-
* The plaintext sentences are
|
|
14
|
-
* never
|
|
15
|
-
*
|
|
13
|
+
* The plaintext sentences are read only on the author's machine, to encrypt
|
|
14
|
+
* them; they are never uploaded or hosted — exactly the L1 never-host-content
|
|
15
|
+
* doctrine (the data-boundaries doctrine), strengthened: not just "no
|
|
16
|
+
* plaintext in the repo" but "no plaintext anywhere we hold, ever."
|
|
16
17
|
*
|
|
17
18
|
* ── SCHEME ────────────────────────────────────────────────────────────────
|
|
18
19
|
* A standard hybrid "sealed box" (libsodium crypto_box_seal shape), built only
|
|
@@ -47,7 +48,7 @@
|
|
|
47
48
|
* format and the card block do NOT change under either model; that is the
|
|
48
49
|
* whole point of the seam. Historical note: earlier docs named FROST here —
|
|
49
50
|
* FROST is threshold *signing*, the wrong primitive for this decryption path;
|
|
50
|
-
* see docs/governance
|
|
51
|
+
* see the community-custodian multisig plan (docs/governance) for the corrected framing.
|
|
51
52
|
*
|
|
52
53
|
* @module seal
|
|
53
54
|
*/
|
package/lib/sealed-qualifier.mjs
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* A sealed (community-controlled secret) test set is precious: every evaluation
|
|
5
5
|
* against it leaks a little signal, and custodian approval is a real,
|
|
6
|
-
* low-frequency act (M-of-N, docs/governance
|
|
6
|
+
* low-frequency act (M-of-N, community-custodian multisig plan M2, docs/governance). So a
|
|
7
7
|
* method must EARN the right to even *propose* a sealed run by first clearing a
|
|
8
8
|
* disjoint, fully public twin — the **qualifier**.
|
|
9
9
|
*
|
|
@@ -33,7 +33,7 @@ import { gatePublicRegistration } from './corpus-registration.mjs';
|
|
|
33
33
|
|
|
34
34
|
/**
|
|
35
35
|
* Default qualifier clearance threshold. Expressed on the public leaderboard's
|
|
36
|
-
*
|
|
36
|
+
* chrF++ 0–100 scale; communities/contests may override per qualifier. This
|
|
37
37
|
* is a floor placeholder, not a claim about any specific pair's difficulty —
|
|
38
38
|
* callers should set a calibrated threshold for real qualifiers.
|
|
39
39
|
*/
|
package/lib/segment.js
CHANGED
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
import { PLACEHOLDER_PREFIX } from './content.js';
|
|
33
|
+
import { nameRules } from './name-rules.js';
|
|
33
34
|
|
|
34
35
|
// Block separator: a newline followed by one or more blank lines
|
|
35
36
|
// (whitespace-only lines count as blank — CommonMark treats them as
|
|
@@ -193,7 +194,7 @@ Rules:
|
|
|
193
194
|
- Translate ALL human-readable text in every segment.
|
|
194
195
|
- Preserve ALL Markdown formatting: headers (#), bold (**), italic (*), links, images, lists, blockquotes, tables, admonitions (:::), etc.${placeholderRule}
|
|
195
196
|
- Preserve line breaks and structure WITHIN each segment.
|
|
196
|
-
|
|
197
|
+
${nameRules(options.protectedTerms)}
|
|
197
198
|
- Translate link text but preserve link URLs. For example: [Read more](url) → [Lire la suite](url)
|
|
198
199
|
- Echo each ${SEGMENT_MARKER_PREFIX}N${SEGMENT_MARKER_SUFFIX} marker on its own line, EXACTLY as given, before its translated segment.
|
|
199
200
|
- Do NOT merge, drop, reorder, renumber, or add segments.
|
package/lib/seo.js
CHANGED
|
@@ -93,6 +93,22 @@ function generateHreflangTags({ baseUrl, urlPattern, inputLocale, locales, pageP
|
|
|
93
93
|
return lines.join('\n');
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/**
|
|
97
|
+
* The target codes in `languages`, in either documented form — a list
|
|
98
|
+
* (["fr","de"]) or an object keyed by code ({ "fr": "formal-vous", … }, the
|
|
99
|
+
* form `champollion init` writes when it records each language's register).
|
|
100
|
+
* The object form used to be skipped here, so its locales got no hreflang.
|
|
101
|
+
*
|
|
102
|
+
* @param {object} config
|
|
103
|
+
* @returns {string[]}
|
|
104
|
+
*/
|
|
105
|
+
function configLanguageCodes(config) {
|
|
106
|
+
const langs = config.languages;
|
|
107
|
+
if (Array.isArray(langs)) return [...langs];
|
|
108
|
+
if (langs && typeof langs === 'object') return Object.keys(langs);
|
|
109
|
+
return [];
|
|
110
|
+
}
|
|
111
|
+
|
|
96
112
|
/**
|
|
97
113
|
* Generate hreflang tags for all configured pages.
|
|
98
114
|
*
|
|
@@ -108,9 +124,7 @@ function generateAllHreflangTags(config) {
|
|
|
108
124
|
|
|
109
125
|
// Collect all target locales from config
|
|
110
126
|
const locales = [];
|
|
111
|
-
|
|
112
|
-
locales.push(...config.languages);
|
|
113
|
-
}
|
|
127
|
+
locales.push(...configLanguageCodes(config));
|
|
114
128
|
if (config.pairs) {
|
|
115
129
|
for (const [pairKey] of Object.entries(config.pairs)) {
|
|
116
130
|
const { target } = parsePairKey(pairKey);
|
|
@@ -163,9 +177,7 @@ function generateSitemap(config) {
|
|
|
163
177
|
|
|
164
178
|
// Collect all locales
|
|
165
179
|
const targetLocales = [];
|
|
166
|
-
|
|
167
|
-
targetLocales.push(...config.languages);
|
|
168
|
-
}
|
|
180
|
+
targetLocales.push(...configLanguageCodes(config));
|
|
169
181
|
if (config.pairs) {
|
|
170
182
|
for (const [pairKey] of Object.entries(config.pairs)) {
|
|
171
183
|
const { target } = parsePairKey(pairKey);
|
|
@@ -224,9 +236,7 @@ function generateJsonLd(config) {
|
|
|
224
236
|
|
|
225
237
|
// Collect all locales
|
|
226
238
|
const targetLocales = [];
|
|
227
|
-
|
|
228
|
-
targetLocales.push(...config.languages);
|
|
229
|
-
}
|
|
239
|
+
targetLocales.push(...configLanguageCodes(config));
|
|
230
240
|
if (config.pairs) {
|
|
231
241
|
for (const [pairKey] of Object.entries(config.pairs)) {
|
|
232
242
|
const { target } = parsePairKey(pairKey);
|
package/lib/serve.js
CHANGED
|
@@ -20,8 +20,10 @@
|
|
|
20
20
|
*
|
|
21
21
|
* NO FORKED TRANSLATION LOGIC:
|
|
22
22
|
* Requests run through the SAME pipeline `sync` uses — resolveRuntime for
|
|
23
|
-
* config/plugin/preflight resolution and
|
|
24
|
-
* TM partition → API call → deterministic quality gate → TM store sequence
|
|
23
|
+
* config/plugin/preflight resolution and translateWithFallback for the
|
|
24
|
+
* TM partition → API call → deterministic quality gate → TM store sequence
|
|
25
|
+
* (plus the pair's fallback method, when one is configured, for what the
|
|
26
|
+
* pair's own method could not translate safely).
|
|
25
27
|
* TM hits are served from cache at $0; gate failures come back as
|
|
26
28
|
* structured per-key errors, never as silently degraded output.
|
|
27
29
|
*
|
|
@@ -49,7 +51,8 @@ import { createRequire } from 'node:module';
|
|
|
49
51
|
import { resolveConfig } from './config.js';
|
|
50
52
|
import { resolveRuntime } from './sync.js';
|
|
51
53
|
import { loadTM, saveTM, isTMDirty, partitionByTM, tmMethodKey } from './tm.js';
|
|
52
|
-
import {
|
|
54
|
+
import { translateWithFallback } from './translate-pair.js';
|
|
55
|
+
import { createFallbackBudget, fallbackSummary } from './fallback.js';
|
|
53
56
|
import { estimateCost, QUALITY_TIERS } from './pairs.js';
|
|
54
57
|
import { getMethod } from './translate.js';
|
|
55
58
|
import { isUnsafeKey } from './security.js';
|
|
@@ -502,7 +505,7 @@ async function handleTranslate(ctx, body, res) {
|
|
|
502
505
|
const { misses } = partitionByTM(runtime.tm, payload, keyNames, targetLocale, tmKey);
|
|
503
506
|
let estimate = null;
|
|
504
507
|
if (misses.length > 0) {
|
|
505
|
-
estimate = await estimateCost(misses.length, pairConfig);
|
|
508
|
+
estimate = await estimateCost(misses.length, pairConfig, { cwd: runtime.cwd });
|
|
506
509
|
const estimated = estimate?.estimatedCost ?? null;
|
|
507
510
|
|
|
508
511
|
if (maxCostPerRequest !== null || maxSessionCost !== null) {
|
|
@@ -525,26 +528,47 @@ async function handleTranslate(ctx, body, res) {
|
|
|
525
528
|
}
|
|
526
529
|
}
|
|
527
530
|
|
|
531
|
+
// ── The pair's fallback method, under the same caps ────────────────
|
|
532
|
+
// What the pair's own method cannot translate safely goes to its
|
|
533
|
+
// fallback (if configured). Under a cap, that batch is priced before it
|
|
534
|
+
// runs and must fit in what the primary's estimate left of the tighter of
|
|
535
|
+
// the per-request cap and the remaining session ceiling (lib/fallback.js).
|
|
536
|
+
const capsLeft = [
|
|
537
|
+
maxCostPerRequest,
|
|
538
|
+
maxSessionCost !== null ? maxSessionCost - session.spendUsd : null,
|
|
539
|
+
].filter(v => v !== null);
|
|
540
|
+
const primaryEstimate = misses.length > 0 ? (estimate?.estimatedCost ?? 0) : 0;
|
|
541
|
+
const fallbackBudget = createFallbackBudget({
|
|
542
|
+
maxCost: capsLeft.length > 0 ? Math.min(...capsLeft) : null,
|
|
543
|
+
committed: primaryEstimate,
|
|
544
|
+
cwd: runtime.cwd,
|
|
545
|
+
});
|
|
546
|
+
|
|
528
547
|
// ── The real pipeline: TM partition → API → quality gate → TM store ──
|
|
529
548
|
let result;
|
|
530
549
|
try {
|
|
531
|
-
result = await
|
|
550
|
+
result = await translateWithFallback(keyNames, payload, pairConfig, pairKey, {
|
|
532
551
|
apiKey: runtime.apiKey,
|
|
552
|
+
cwd: runtime.cwd,
|
|
533
553
|
tm: runtime.tm,
|
|
534
554
|
targetCode: targetLocale,
|
|
535
555
|
onProgress: null,
|
|
556
|
+
budget: fallbackBudget,
|
|
536
557
|
});
|
|
537
558
|
} catch (err) {
|
|
538
559
|
output.error(`[serve] ${pairKey}: pipeline error — ${err.message}`);
|
|
539
560
|
return sendError(res, 500, 'internal_error', 'Translation pipeline failed. See server logs.');
|
|
540
561
|
}
|
|
541
|
-
if (result.apiCalled) output.
|
|
562
|
+
if (result.apiCalled) output.progressDone(`[serve] ${pairKey}`, 'done');
|
|
542
563
|
|
|
543
564
|
// Session ledger: count the estimate for any request that actually hit
|
|
544
565
|
// the upstream API (conservative — estimates deliberately err high).
|
|
545
566
|
if (result.apiCalled && estimate?.estimatedCost != null) {
|
|
546
567
|
session.spendUsd += estimate.estimatedCost;
|
|
547
568
|
}
|
|
569
|
+
// Fallback batches approved under a cap carry their estimate too.
|
|
570
|
+
const fallbackSpend = fallbackBudget.committed - primaryEstimate;
|
|
571
|
+
if (fallbackSpend > 0) session.spendUsd += fallbackSpend;
|
|
548
572
|
|
|
549
573
|
// Persist TM mutations (stores AND evictions) after every request so a
|
|
550
574
|
// crash never loses paid, gate-validated translations. Atomic write.
|
|
@@ -616,6 +640,17 @@ async function handleTranslate(ctx, body, res) {
|
|
|
616
640
|
costUsd = estimate.estimatedCost;
|
|
617
641
|
costBasis = 'estimate';
|
|
618
642
|
}
|
|
643
|
+
// A fallback that ran adds its own spend: priced when a cap made us price
|
|
644
|
+
// it, otherwise unknown — never silently left out of a known figure.
|
|
645
|
+
if (result.fallback?.attempted > 0) {
|
|
646
|
+
if (fallbackBudget.maxCost !== null && costUsd !== null) {
|
|
647
|
+
costUsd += fallbackSpend;
|
|
648
|
+
costBasis = 'estimate';
|
|
649
|
+
} else {
|
|
650
|
+
costUsd = null;
|
|
651
|
+
costBasis = 'unknown';
|
|
652
|
+
}
|
|
653
|
+
}
|
|
619
654
|
|
|
620
655
|
const meta = {
|
|
621
656
|
served_by: `champollion-serve/${CLI_VERSION}`,
|
|
@@ -629,6 +664,8 @@ async function handleTranslate(ctx, body, res) {
|
|
|
629
664
|
translated: translatedCount,
|
|
630
665
|
failed: errorCount,
|
|
631
666
|
tm_hits: result.tmHitCount,
|
|
667
|
+
// The pair's fallback method and what it did (pairs with a fallback).
|
|
668
|
+
...(result.fallback && { fallback: fallbackSummary(result.fallback) }),
|
|
632
669
|
// Script conversion status for this pair: which script was written and
|
|
633
670
|
// how many values stayed in the working script (unmapped letters).
|
|
634
671
|
script_conversion: {
|