champollion 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -37
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +51 -3
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +289 -88
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +649 -130
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +16 -10
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +197 -38
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +193 -106
- package/lib/seal.mjs +6 -5
- package/lib/sealed-qualifier.mjs +2 -2
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +3 -2
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/DATA-SOVEREIGNTY.md +19 -20
- package/shared/LANGUAGE-CARD-FIELDS.md +1 -1
- package/shared/cards-fallback.json +1 -1
- package/shared/catalogue/card-config.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/docent/faq.en.json +14 -16
- package/shared/docent/system-prompt.md +17 -19
- package/shared/explainers/tc-features.json +15 -15
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/human-services.json +1 -1
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +20 -10
- package/shared/schemas/human-services.schema.json +2 -2
- package/shared/schemas/language-card.schema.json +1 -1
- package/shared/schemas/method-card.schema.json +1 -1
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11333
package/lib/redo.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* --redo / --fresh: one way to say "translate this again".
|
|
3
|
+
*
|
|
4
|
+
* Six flags had grown up around re-translation, each with its own cache
|
|
5
|
+
* rule (--force, --force-keys, --force-content, --retranslate, --no-tm,
|
|
6
|
+
* --fresh-on-model-change), and people could not tell which one cost money.
|
|
7
|
+
* Two words now carry the whole idea:
|
|
8
|
+
*
|
|
9
|
+
* --redo <scope> queue this again. The cache still serves anything it
|
|
10
|
+
* already holds (gate-checked), so a redo is cheap.
|
|
11
|
+
* all | keys:<k1,k2> | content | files:<glob> | gaps
|
|
12
|
+
* (gaps: every plural message on disk that lacks a form
|
|
13
|
+
* its language uses for ordinary counts — asked from the
|
|
14
|
+
* model, never the cache, which holds the incomplete
|
|
15
|
+
* answer; lib/plural-gap-redo.js)
|
|
16
|
+
* (repeatable: --redo keys:a.b --redo files:docs/**)
|
|
17
|
+
* --fresh do not use the cache: what is queued is paid for again.
|
|
18
|
+
*
|
|
19
|
+
* `--redo files:X --fresh` is exactly the old `--retranslate X`. The old
|
|
20
|
+
* flags still work as written; this module only translates the new words
|
|
21
|
+
* into them, so sync itself has one set of semantics.
|
|
22
|
+
* --fresh-on-model-change stays separate: it is a policy for model
|
|
23
|
+
* switches, not a redo.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
const SCOPES = 'all | keys:<k1,k2> | content | files:<glob> | gaps';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Split a comma-separated key list, honouring `\,` for a comma INSIDE a key.
|
|
30
|
+
* gettext keys are whole sentences ("Welcome, %(name)s"), so a plain split
|
|
31
|
+
* made them impossible to name in --force-keys / --redo keys:.
|
|
32
|
+
* @param {string} list
|
|
33
|
+
* @returns {string[]}
|
|
34
|
+
*/
|
|
35
|
+
export function splitKeyList(list) {
|
|
36
|
+
return String(list ?? '')
|
|
37
|
+
.split(/(?<!\\),/)
|
|
38
|
+
.map((k) => k.replace(/\\,/g, ',').trim())
|
|
39
|
+
.filter(Boolean);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Inverse of splitKeyList for one key. */
|
|
43
|
+
const escapeKey = (k) => k.replace(/,/g, '\\,');
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Rewrite args in place. Returns an error message, or null.
|
|
47
|
+
* @param {object} args parsed CLI args
|
|
48
|
+
* @returns {string|null}
|
|
49
|
+
*/
|
|
50
|
+
export function applyRedo(args) {
|
|
51
|
+
const scopes = [].concat(args.redo || []).map((s) => String(s).trim()).filter(Boolean);
|
|
52
|
+
if (args.redo !== undefined && scopes.length === 0) {
|
|
53
|
+
return `--redo needs a scope: ${SCOPES}`;
|
|
54
|
+
}
|
|
55
|
+
const fresh = Boolean(args.fresh);
|
|
56
|
+
let keyScope = false;
|
|
57
|
+
|
|
58
|
+
for (const scope of scopes) {
|
|
59
|
+
const m = /^(all|content|keys|files|gaps)(?::(.*))?$/.exec(scope);
|
|
60
|
+
if (!m) return `--redo ${scope}: unknown scope. Use ${SCOPES}`;
|
|
61
|
+
const [, kind, value] = m;
|
|
62
|
+
if ((kind === 'keys' || kind === 'files') && !value) {
|
|
63
|
+
return `--redo ${kind}: needs a value, e.g. --redo ${kind === 'keys' ? 'keys:nav.home,nav.about' : 'files:docs/intro.md'}`;
|
|
64
|
+
}
|
|
65
|
+
if ((kind === 'all' || kind === 'content' || kind === 'gaps') && value) {
|
|
66
|
+
return `--redo ${kind} takes no value (got "${scope}")`;
|
|
67
|
+
}
|
|
68
|
+
if (kind === 'all') {
|
|
69
|
+
args.force = true;
|
|
70
|
+
keyScope = true;
|
|
71
|
+
} else if (kind === 'keys') {
|
|
72
|
+
const keys = [...splitKeyList(args['force-keys']), ...splitKeyList(value)];
|
|
73
|
+
args['force-keys'] = keys.map(escapeKey).join(',');
|
|
74
|
+
keyScope = true;
|
|
75
|
+
} else if (kind === 'gaps') {
|
|
76
|
+
args['redo-gaps'] = true;
|
|
77
|
+
keyScope = true;
|
|
78
|
+
} else if (kind === 'content') {
|
|
79
|
+
args['force-content'] = true;
|
|
80
|
+
keyScope = true; // content blocks are cache-served too; --fresh bills them
|
|
81
|
+
} else if (kind === 'files') {
|
|
82
|
+
if (fresh) {
|
|
83
|
+
args.retranslate = [...[].concat(args.retranslate || []), value];
|
|
84
|
+
} else {
|
|
85
|
+
args.files = [...[].concat(args.files || []), value];
|
|
86
|
+
args['force-content'] = true;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// --fresh on its own, or with a key/content scope: bypass the cache for
|
|
92
|
+
// this run. (A files scope under --fresh is already --retranslate.)
|
|
93
|
+
if (fresh && (scopes.length === 0 || keyScope)) args['no-tm'] = true;
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* refusal-category.js — why the pair's own method's answer was not used, in
|
|
3
|
+
* a few words that can be counted. Shared by the key-value pipeline
|
|
4
|
+
* (lib/translate-pair.js) and the content lanes (lib/fallback.js); no
|
|
5
|
+
* imports, so neither gets an import cycle.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A quality-gate refusal (or a missing answer) in a few words, so refusals
|
|
10
|
+
* can be counted: "length inflation (3.2x source, max 2.5x)" → "length
|
|
11
|
+
* inflation". The gate's own wording, without the figures that make each
|
|
12
|
+
* one unique.
|
|
13
|
+
*
|
|
14
|
+
* @param {string|null} reason - The gate's reason (null: no answer for it)
|
|
15
|
+
* @param {{ sharedOutput?: boolean, memorized?: boolean, heldBefore?: boolean, noAnswer?: boolean }} [flags]
|
|
16
|
+
* @returns {string}
|
|
17
|
+
*/
|
|
18
|
+
export function refusalCategory(reason, flags = {}) {
|
|
19
|
+
if (flags.heldBefore) return 'refused on an earlier sync, so not asked again';
|
|
20
|
+
if (flags.sharedOutput || flags.memorized || /memorized sentence/.test(String(reason || ''))) {
|
|
21
|
+
return 'a memorized sentence repeated for different source strings';
|
|
22
|
+
}
|
|
23
|
+
if (flags.noAnswer || !reason) return 'no answer';
|
|
24
|
+
const text = String(reason);
|
|
25
|
+
if (/^source echo|disguised|English in disguise/i.test(text)) return 'a copy of the source text';
|
|
26
|
+
if (/protected element/i.test(text)) return 'code, links or markup damaged or missing';
|
|
27
|
+
if (/content was lost|content preservation|orphaned placeholder/i.test(text)) return 'content lost or changed';
|
|
28
|
+
// The gate's words up to its figures or explanation: "length inflation",
|
|
29
|
+
// "repetition hallucination", "wrong script", "empty translation", …
|
|
30
|
+
return text.split(/ \(| — |: |; /)[0].trim() || 'refused by the quality gate';
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Count one unit the fallback was asked for, under why the primary's answer
|
|
35
|
+
* was not used (mutates `report.primaryReasons`).
|
|
36
|
+
*
|
|
37
|
+
* @param {object} report - A fallback report (lib/fallback.js newFallbackReport)
|
|
38
|
+
* @param {string} category - refusalCategory()
|
|
39
|
+
*/
|
|
40
|
+
export function notePrimaryReason(report, category) {
|
|
41
|
+
if (!report) return;
|
|
42
|
+
report.primaryReasons = report.primaryReasons || {};
|
|
43
|
+
report.primaryReasons[category] = (report.primaryReasons[category] || 0) + 1;
|
|
44
|
+
}
|
package/lib/registers.js
CHANGED
|
@@ -400,6 +400,19 @@ function _presetFor(code) {
|
|
|
400
400
|
* live corpus still does, and a card's own values must not be silently
|
|
401
401
|
* replaced by config. Once cutover lands, only the config path is live.
|
|
402
402
|
*/
|
|
403
|
+
/**
|
|
404
|
+
* CLI method name → key in the runtime `methodSupport` map. The service
|
|
405
|
+
* engines are the ones a card can say yes or no about; LLM methods are
|
|
406
|
+
* open-ended by construction (see `flat.llm` below) and have no entry.
|
|
407
|
+
*/
|
|
408
|
+
const METHOD_SUPPORT_KEYS = {
|
|
409
|
+
'google-translate': 'googleTranslate',
|
|
410
|
+
deepl: 'deepl',
|
|
411
|
+
'microsoft-translator': 'microsoftTranslator',
|
|
412
|
+
libretranslate: 'libreTranslate',
|
|
413
|
+
apertium: 'apertium',
|
|
414
|
+
};
|
|
415
|
+
|
|
403
416
|
function _withPromptConfig(card) {
|
|
404
417
|
if (!card) return card;
|
|
405
418
|
const preset = _presetFor(card.code);
|
|
@@ -438,17 +451,10 @@ function _withPromptConfig(card) {
|
|
|
438
451
|
// that absence answers that question honestly. This synthesizes the legacy
|
|
439
452
|
// map from them; the evidence stays on the card as methodSupportEvidence.
|
|
440
453
|
if (view.methodSupport && Array.isArray(view.methodSupport.named)) {
|
|
441
|
-
const LEGACY_KEYS = {
|
|
442
|
-
'google-translate': 'googleTranslate',
|
|
443
|
-
deepl: 'deepl',
|
|
444
|
-
'microsoft-translator': 'microsoftTranslator',
|
|
445
|
-
libretranslate: 'libreTranslate',
|
|
446
|
-
apertium: 'apertium',
|
|
447
|
-
};
|
|
448
454
|
const flat = {};
|
|
449
|
-
for (const key of Object.values(
|
|
455
|
+
for (const key of Object.values(METHOD_SUPPORT_KEYS)) flat[key] = { supported: false };
|
|
450
456
|
for (const n of view.methodSupport.named) {
|
|
451
|
-
const legacy =
|
|
457
|
+
const legacy = METHOD_SUPPORT_KEYS[n.variant];
|
|
452
458
|
if (legacy && n.value === 'service') flat[legacy] = { supported: true };
|
|
453
459
|
}
|
|
454
460
|
flat.deepl.formality = flat.deepl.supported && _deeplFormality.has(view.code);
|
|
@@ -602,6 +608,29 @@ _loadCards();
|
|
|
602
608
|
// Dynamic tier (packaged mode) — per-user cache + remote fetch
|
|
603
609
|
// -----------------------------------------------------------------
|
|
604
610
|
|
|
611
|
+
/**
|
|
612
|
+
* Locale-shaped codes ('aaa-NG', 'cmn-Hant') that no remote fetch can
|
|
613
|
+
* ever materialize.
|
|
614
|
+
*
|
|
615
|
+
* A locale card is a PROJECTION of its language's card, so the
|
|
616
|
+
* publication lane never emits one: prod's trading_card_index /
|
|
617
|
+
* trading_card_detail hold 8,681 rows and ZERO hyphenated codes
|
|
618
|
+
* (verified 2026-08-28). The bundle ships 840 locale deltas whose
|
|
619
|
+
* parent is in the core set; the remaining 7,840 manifest locales are
|
|
620
|
+
* names only — asking prod for one buys a round-trip, a miss, and a
|
|
621
|
+
* tombstone for a code that will never exist.
|
|
622
|
+
*
|
|
623
|
+
* Conlangs ('x-pirate') are hyphenated too, but card-config.json loads
|
|
624
|
+
* them into _cards in BOTH modes, so every caller of this predicate is
|
|
625
|
+
* already past a _cards hit by the time it runs.
|
|
626
|
+
*
|
|
627
|
+
* @param {string} code
|
|
628
|
+
* @returns {boolean}
|
|
629
|
+
*/
|
|
630
|
+
function _isUnfetchableLocale(code) {
|
|
631
|
+
return code.includes('-');
|
|
632
|
+
}
|
|
633
|
+
|
|
605
634
|
/**
|
|
606
635
|
* Try to materialize a card that isn't bundled: first from the
|
|
607
636
|
* per-user cache (~/.champollion/cards/), then via a synchronous
|
|
@@ -638,6 +667,9 @@ function _loadDynamicCard(code) {
|
|
|
638
667
|
*/
|
|
639
668
|
function _tryFetchRemoteSync(code) {
|
|
640
669
|
if (OFFLINE) return null;
|
|
670
|
+
// Prod publishes no locale codes, so there is nothing to ask for —
|
|
671
|
+
// and no tombstone worth writing for a code that can never exist.
|
|
672
|
+
if (_isUnfetchableLocale(code)) return null;
|
|
641
673
|
if (_fetchAttempted.has(code)) return null;
|
|
642
674
|
_fetchAttempted.add(code);
|
|
643
675
|
|
|
@@ -706,6 +738,12 @@ async function prefetchLanguageCards(codes) {
|
|
|
706
738
|
result.skipped.push(input);
|
|
707
739
|
continue;
|
|
708
740
|
}
|
|
741
|
+
if (_isUnfetchableLocale(code)) {
|
|
742
|
+
// Never published as a trading card (see _isUnfetchableLocale) —
|
|
743
|
+
// reported, not silently dropped, and not fetched.
|
|
744
|
+
result.missing.push(code);
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
709
747
|
try {
|
|
710
748
|
const remote = await fetchRemoteCard(code, { aliases: _manifest.get(code)?.a });
|
|
711
749
|
if (remote.missing) {
|
|
@@ -750,6 +788,11 @@ function getCardSourceInfo() {
|
|
|
750
788
|
* - Alias: 'fr' → 'fra', 'no' → 'nob', 'iw' → 'heb'
|
|
751
789
|
* - Base locale fallback: 'deu-AT' → 'deu' (base)
|
|
752
790
|
*
|
|
791
|
+
* The base fallback is what packaged mode leans on: repo checkouts hold
|
|
792
|
+
* a card for every locale, but the npm bundle ships only 840 locale
|
|
793
|
+
* deltas and prod publishes no locale codes, so the other 7,840 resolve
|
|
794
|
+
* to their language rather than to nothing.
|
|
795
|
+
*
|
|
753
796
|
* @param {string} code - Locale code to resolve
|
|
754
797
|
* @returns {string} Resolved primary code (ISO 639-3)
|
|
755
798
|
*/
|
|
@@ -760,9 +803,24 @@ function resolveCode(code) {
|
|
|
760
803
|
// Alias match
|
|
761
804
|
if (_aliases.has(code)) return _aliases.get(code);
|
|
762
805
|
|
|
806
|
+
// Flutter, gettext and Java write locales with '_' (pt_BR, zh_Hant); BCP 47
|
|
807
|
+
// and the cards use '-'. A Flutter "pt_BR" target matched no card, so it
|
|
808
|
+
// was synced as a language literally named "pt_BR" with no register.
|
|
809
|
+
if (typeof code === 'string' && code.includes('_')) {
|
|
810
|
+
return resolveCode(code.replace(/_/g, '-'));
|
|
811
|
+
}
|
|
812
|
+
|
|
763
813
|
// Manifest match (packaged mode) — the code is a known primary code
|
|
764
|
-
// even though its card isn't materialized locally yet
|
|
765
|
-
|
|
814
|
+
// even though its card isn't materialized locally yet.
|
|
815
|
+
//
|
|
816
|
+
// LOCALE CODES ARE EXCLUDED, and that exclusion is the whole point of
|
|
817
|
+
// the guard. The manifest lists all 8,680 locales, but only the 840
|
|
818
|
+
// whose parent is in the core set ship as bundled deltas (caught by
|
|
819
|
+
// the _cards check above) and prod publishes none of them at all. So
|
|
820
|
+
// a bare manifest hit used to resolve 'aaa-NG' to itself, hand it to
|
|
821
|
+
// _loadDynamicCard(), and return null for 7,840 codes the docblock
|
|
822
|
+
// above promises to fall back on. Fall through to the base instead.
|
|
823
|
+
if (_manifest.has(code) && !_isUnfetchableLocale(code)) return code;
|
|
766
824
|
|
|
767
825
|
// Base locale fallback — try stripping region (e.g., 'de-AT' → 'de')
|
|
768
826
|
const baseParts = code.split('-');
|
|
@@ -1014,6 +1072,34 @@ function getGenderGuidance(code) {
|
|
|
1014
1072
|
return card?.gender?.inclusiveGuidance || null;
|
|
1015
1073
|
}
|
|
1016
1074
|
|
|
1075
|
+
/**
|
|
1076
|
+
* The gender guidance a prompt carries, in one line a person can read: its
|
|
1077
|
+
* instruction (the first sentence that says what to do — "prefer écriture
|
|
1078
|
+
* inclusive with the interpunct …"), not the grammar note before it.
|
|
1079
|
+
*
|
|
1080
|
+
* @param {string|null} text - Guidance text (a pair's genderGuidance)
|
|
1081
|
+
* @param {number} [max=150]
|
|
1082
|
+
* @returns {string|null}
|
|
1083
|
+
*/
|
|
1084
|
+
function summarizeGenderGuidance(text, max = 150) {
|
|
1085
|
+
if (typeof text !== 'string' || !text.trim()) return null;
|
|
1086
|
+
const sentences = text.trim().split(/(?<=[.!?])\s+(?=[A-Z])/);
|
|
1087
|
+
const pick = sentences.find(x => /\b(prefer|use|avoid|write|keep|default)\b/i.test(x)) || sentences[0];
|
|
1088
|
+
return pick.length > max ? `${pick.slice(0, max - 1).trimEnd()}…` : pick;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
/**
|
|
1092
|
+
* Is this an ISO 639 private-use code (qaa–qtz) — the range kept for a
|
|
1093
|
+
* variety with no code yet, or one not confirmed (an unconfirmed "Ayta")?
|
|
1094
|
+
* The base subtag decides: "qaa", "qaa-PH", "qab_x" all are.
|
|
1095
|
+
*
|
|
1096
|
+
* @param {string} code
|
|
1097
|
+
* @returns {boolean}
|
|
1098
|
+
*/
|
|
1099
|
+
function isPrivateUseCode(code) {
|
|
1100
|
+
return /^q[a-t][a-z]$/i.test(String(code || '').split(/[-_]/)[0]);
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1017
1103
|
/**
|
|
1018
1104
|
* Get all loaded language codes (primary, not aliases).
|
|
1019
1105
|
*
|
|
@@ -1033,6 +1119,140 @@ function getAllLanguageCodes() {
|
|
|
1033
1119
|
return Array.from(codes);
|
|
1034
1120
|
}
|
|
1035
1121
|
|
|
1122
|
+
// -----------------------------------------------------------------
|
|
1123
|
+
// Language NAMES → codes (an agent or a person types "French")
|
|
1124
|
+
// -----------------------------------------------------------------
|
|
1125
|
+
|
|
1126
|
+
/** "Plains Créé" → "plains cree": case, accents and spacing never decide a match. */
|
|
1127
|
+
function _normalizeName(s) {
|
|
1128
|
+
return String(s ?? '').normalize('NFD').replace(/\p{M}/gu, '').toLowerCase().replace(/\s+/g, ' ').trim();
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
/** @type {Map<string, Array<{ code: string, name: string, aliases: string[] }>>|null} */
|
|
1132
|
+
let _nameIndex = null;
|
|
1133
|
+
|
|
1134
|
+
/**
|
|
1135
|
+
* normalized display name → the LANGUAGE cards that carry it. Locale cards
|
|
1136
|
+
* (`fra-CA`, a projection of French for Canada) are not languages and never
|
|
1137
|
+
* answer a name: in repo mode they are told by their `locale` block; in
|
|
1138
|
+
* packaged mode an unmaterialized manifest entry has no block, and a locale
|
|
1139
|
+
* entry is the one with a region/script subtag (the convention
|
|
1140
|
+
* _isUnfetchableLocale already applies to the same manifest).
|
|
1141
|
+
*/
|
|
1142
|
+
function _buildNameIndex() {
|
|
1143
|
+
const index = new Map();
|
|
1144
|
+
const add = (code, name, aliases) => {
|
|
1145
|
+
if (typeof name !== 'string' || !name.trim()) return;
|
|
1146
|
+
const key = _normalizeName(name);
|
|
1147
|
+
const list = index.get(key) || [];
|
|
1148
|
+
if (!list.some(e => e.code === code)) list.push({ code, name, aliases: Array.isArray(aliases) ? aliases : [] });
|
|
1149
|
+
index.set(key, list);
|
|
1150
|
+
};
|
|
1151
|
+
for (const [code, raw] of _cards) {
|
|
1152
|
+
if (!raw || raw.locale) continue;
|
|
1153
|
+
if (_mode === 'packaged' && _isUnfetchableLocale(code)) continue;
|
|
1154
|
+
const name = typeof raw.name === 'string' ? raw.name : normalizeCard({ name: raw.name }).name;
|
|
1155
|
+
add(code, name, raw.codeAliases ?? raw.aliases);
|
|
1156
|
+
}
|
|
1157
|
+
for (const [code, entry] of _manifest) {
|
|
1158
|
+
if (_cards.has(code) || _isUnfetchableLocale(code)) continue;
|
|
1159
|
+
add(code, entry?.n, entry?.a);
|
|
1160
|
+
}
|
|
1161
|
+
return index;
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
/**
|
|
1165
|
+
* The language cards whose display name is `name` (case-, accent- and
|
|
1166
|
+
* spacing-insensitive), never locale projections. Usually one; several when
|
|
1167
|
+
* the registries give two languages the same name (the caller must then ask
|
|
1168
|
+
* for a code — it never picks).
|
|
1169
|
+
*
|
|
1170
|
+
* @param {string} name
|
|
1171
|
+
* @returns {Array<{ code: string, name: string, tag: string }>} tag: the code
|
|
1172
|
+
* to write for it — BCP 47 uses the two-letter ISO 639-1 code where one
|
|
1173
|
+
* exists ("fr", not "fra"), so that is what a project file would carry
|
|
1174
|
+
*/
|
|
1175
|
+
function findLanguagesByName(name) {
|
|
1176
|
+
if (!_nameIndex) _nameIndex = _buildNameIndex();
|
|
1177
|
+
return (_nameIndex.get(_normalizeName(name)) || []).map(e => ({
|
|
1178
|
+
code: e.code,
|
|
1179
|
+
name: e.name,
|
|
1180
|
+
tag: e.aliases.find(a => typeof a === 'string' && /^[a-z]{2}$/.test(a)) || e.code,
|
|
1181
|
+
}));
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
/** Locale spelling that never decides a match: case, and `_` vs `-` (pt_BR). */
|
|
1185
|
+
const _sameLocale = (a, b) => String(a).toLowerCase().replace(/_/g, '-') === String(b).toLowerCase().replace(/_/g, '-');
|
|
1186
|
+
|
|
1187
|
+
/**
|
|
1188
|
+
* Turn what a caller typed for a language — a code ("fr", "fra", "pt_BR") or
|
|
1189
|
+
* a NAME ("French", "Plains Cree") — into the code to work in. With
|
|
1190
|
+
* `locales` (a project's own locale codes: its input locale and targets) the
|
|
1191
|
+
* answer is the project's spelling: "French" and "fra" both become the
|
|
1192
|
+
* project's "fr", exactly the code sync writes, caches and locks under.
|
|
1193
|
+
*
|
|
1194
|
+
* Never guesses. A name two languages share, or a language that matches two
|
|
1195
|
+
* of the project's locales ("French" in a project with fr and fr-CA), is an
|
|
1196
|
+
* error that lists the choices; so is a word that is neither a code nor a
|
|
1197
|
+
* language name. A code with no card ("x-pirate", "qaa") is returned as
|
|
1198
|
+
* written — the private-use and unlisted codes are legitimate.
|
|
1199
|
+
*
|
|
1200
|
+
* @param {string} input
|
|
1201
|
+
* @param {{ locales?: string[]|null }} [options]
|
|
1202
|
+
* @returns {{ code: string, how: 'locale'|'code'|'name', input: string, name?: string }
|
|
1203
|
+
* | { error: string, input: string, candidates?: string[] }}
|
|
1204
|
+
* how: 'locale' = one of `locales` (by its code, another code for the same
|
|
1205
|
+
* language, or its name — `name` says which name matched); 'code' = a code
|
|
1206
|
+
* outside `locales`; 'name' = a language name, as its code
|
|
1207
|
+
*/
|
|
1208
|
+
function resolveLanguageInput(input, { locales = null } = {}) {
|
|
1209
|
+
const raw = String(input ?? '').trim();
|
|
1210
|
+
if (!raw) return { error: 'no language given', input: raw };
|
|
1211
|
+
const own = Array.isArray(locales) ? [...new Set(locales.filter(l => typeof l === 'string' && l))] : null;
|
|
1212
|
+
|
|
1213
|
+
if (own) {
|
|
1214
|
+
const exact = own.find(l => l === raw) || own.find(l => _sameLocale(l, raw));
|
|
1215
|
+
if (exact) return { code: exact, how: 'locale', input: raw };
|
|
1216
|
+
}
|
|
1217
|
+
const projectMatches = (languageCode) => (own ? own.filter(l => resolveCode(l) === languageCode) : []);
|
|
1218
|
+
const ambiguous = (what, list) => ({
|
|
1219
|
+
error: `${what} matches more than one of the project's locales (${list.join(', ')}) — pass the one you mean`,
|
|
1220
|
+
input: raw, candidates: list,
|
|
1221
|
+
});
|
|
1222
|
+
|
|
1223
|
+
// A code: shaped like a BCP 47 tag — a 2-3 letter language subtag, or a
|
|
1224
|
+
// private-use tag ("x-pirate"). "French" also parses as a (5-8 letter,
|
|
1225
|
+
// never registered) language subtag, so length decides, not the grammar.
|
|
1226
|
+
const shaped = raw.replace(/_/g, '-');
|
|
1227
|
+
const m = /^([A-Za-z]{2,3})(?:-[A-Za-z0-9]{1,8})*$/.exec(shaped) || /^[xX](?:-[A-Za-z0-9]{1,8})+$/.exec(shaped);
|
|
1228
|
+
if (m) {
|
|
1229
|
+
const same = projectMatches(resolveCode(raw));
|
|
1230
|
+
if (same.length === 1) return { code: same[0], how: 'locale', input: raw };
|
|
1231
|
+
if (same.length > 1) return ambiguous(`"${raw}"`, same);
|
|
1232
|
+
return { code: raw, how: 'code', input: raw };
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
// A name.
|
|
1236
|
+
const found = findLanguagesByName(raw);
|
|
1237
|
+
if (found.length === 0) {
|
|
1238
|
+
return {
|
|
1239
|
+
error: `"${raw}" is neither a language code nor the name of a language champollion has a card for — pass its code (e.g. "fr", "crk")`,
|
|
1240
|
+
input: raw,
|
|
1241
|
+
};
|
|
1242
|
+
}
|
|
1243
|
+
if (found.length > 1) {
|
|
1244
|
+
return {
|
|
1245
|
+
error: `"${raw}" is the name of ${found.length} languages (${found.map(f => f.code).join(', ')}) — pass the code of the one you mean`,
|
|
1246
|
+
input: raw, candidates: found.map(f => f.code),
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
const lang = found[0];
|
|
1250
|
+
const same = projectMatches(lang.code);
|
|
1251
|
+
if (same.length === 1) return { code: same[0], how: 'locale', input: raw, name: lang.name };
|
|
1252
|
+
if (same.length > 1) return ambiguous(`"${raw}" (${lang.code})`, same);
|
|
1253
|
+
return { code: lang.tag, how: 'name', input: raw, name: lang.name };
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1036
1256
|
/**
|
|
1037
1257
|
* Get method support flags for a locale.
|
|
1038
1258
|
*
|
|
@@ -1044,6 +1264,25 @@ function getMethodSupport(code) {
|
|
|
1044
1264
|
return card?.methodSupport || null;
|
|
1045
1265
|
}
|
|
1046
1266
|
|
|
1267
|
+
/**
|
|
1268
|
+
* Does the card say this service engine covers this language?
|
|
1269
|
+
*
|
|
1270
|
+
* Takes the CLI method name (`libretranslate`, `google-translate`, …) and
|
|
1271
|
+
* reads the entry's `supported` flag — entries are objects, never bare
|
|
1272
|
+
* booleans, so testing the entry itself against `false` can never fire.
|
|
1273
|
+
*
|
|
1274
|
+
* @param {string} code - Locale code
|
|
1275
|
+
* @param {string} method - CLI method name
|
|
1276
|
+
* @returns {boolean|null} true/false when the card answers; null when there
|
|
1277
|
+
* is no card, no entry, or the method is not a listed service (LLM lanes)
|
|
1278
|
+
*/
|
|
1279
|
+
function isMethodSupported(code, method) {
|
|
1280
|
+
const key = METHOD_SUPPORT_KEYS[method];
|
|
1281
|
+
if (!key) return null;
|
|
1282
|
+
const entry = getMethodSupport(code)?.[key];
|
|
1283
|
+
return typeof entry?.supported === 'boolean' ? entry.supported : null;
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1047
1286
|
// -----------------------------------------------------------------
|
|
1048
1287
|
// Reference tier — REMOVED (v6: unified cards)
|
|
1049
1288
|
// -----------------------------------------------------------------
|
|
@@ -1167,9 +1406,14 @@ export {
|
|
|
1167
1406
|
getRegisterPresets,
|
|
1168
1407
|
getFormality,
|
|
1169
1408
|
getGenderGuidance,
|
|
1409
|
+
summarizeGenderGuidance,
|
|
1410
|
+
isPrivateUseCode,
|
|
1170
1411
|
getAllLanguageCodes,
|
|
1171
1412
|
getMethodSupport,
|
|
1413
|
+
isMethodSupported,
|
|
1172
1414
|
resolveCode,
|
|
1415
|
+
findLanguagesByName,
|
|
1416
|
+
resolveLanguageInput,
|
|
1173
1417
|
DEFAULT_REGISTER_FALLBACK,
|
|
1174
1418
|
|
|
1175
1419
|
// Dynamic card tier (packaged installs)
|
package/lib/repair-script.js
CHANGED
|
@@ -42,9 +42,8 @@ import { getLanguageCard } from './registers.js';
|
|
|
42
42
|
import {
|
|
43
43
|
SCRIPT_CONVERTERS, reverseScript, isPrivateUse, converterKeyForLocale, getConverterInfo,
|
|
44
44
|
} from './scripts.js';
|
|
45
|
-
import {
|
|
46
|
-
|
|
47
|
-
} from './format.js';
|
|
45
|
+
import { readLocaleFile, writeLocaleFile, detectYAMLStyle, LOCALE_FILE_FORMATS } from './format.js';
|
|
46
|
+
import { discoverLocaleLayout } from './locale-layout.js';
|
|
48
47
|
import { discoverDocusaurusJSONFiles } from './docusaurus-sync.js';
|
|
49
48
|
import { visualize } from './integrity.js';
|
|
50
49
|
import { output } from './output.js';
|
|
@@ -107,7 +106,10 @@ function runRepairScript({ cwd = process.cwd(), cliArgs = {} } = {}) {
|
|
|
107
106
|
const dry = !!cliArgs.dry;
|
|
108
107
|
const localeFilter = cliArgs.locale || null;
|
|
109
108
|
|
|
110
|
-
const pairs = resolvePairs(config);
|
|
109
|
+
const pairs = resolvePairs(config, { cwd });
|
|
110
|
+
// Key-value projects: every locale's files from the ONE layout module
|
|
111
|
+
// (flat file, folder of namespace files, or localesPattern).
|
|
112
|
+
const layout = config.format === 'docusaurus' ? null : discoverLocaleLayout(config, { cwd });
|
|
111
113
|
const report = [];
|
|
112
114
|
const totals = { filesScanned: 0, valuesRepaired: 0, caseLossy: 0, unreversedCodepoints: 0 };
|
|
113
115
|
let hadWriteFailure = false;
|
|
@@ -134,16 +136,18 @@ function runRepairScript({ cwd = process.cwd(), cliArgs = {} } = {}) {
|
|
|
134
136
|
continue;
|
|
135
137
|
}
|
|
136
138
|
|
|
137
|
-
// Discover this locale's files
|
|
138
|
-
//
|
|
139
|
+
// Discover this locale's files: Docusaurus keeps a directory per locale
|
|
140
|
+
// of {message} JSON; key-value projects ask the layout (one file, or
|
|
141
|
+
// one per namespace). Each entry carries its own real format.
|
|
139
142
|
const files = [];
|
|
140
|
-
let flatFormat = null;
|
|
141
143
|
if (config.format === 'docusaurus') {
|
|
142
|
-
|
|
144
|
+
for (const p of discoverDocusaurusJSONFiles(path.join(config.localesDir, locale))) {
|
|
145
|
+
files.push({ path: p, format: 'json' });
|
|
146
|
+
}
|
|
143
147
|
} else {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
148
|
+
for (const f of layout.filesFor(locale)) {
|
|
149
|
+
if (fs.existsSync(f.path)) files.push({ path: f.path, format: f.format });
|
|
150
|
+
}
|
|
147
151
|
}
|
|
148
152
|
|
|
149
153
|
const localeReport = {
|
|
@@ -152,12 +156,15 @@ function runRepairScript({ cwd = process.cwd(), cliArgs = {} } = {}) {
|
|
|
152
156
|
files: [],
|
|
153
157
|
};
|
|
154
158
|
|
|
155
|
-
for (const filePath of files) {
|
|
159
|
+
for (const { path: filePath, format: flatFormat } of files) {
|
|
156
160
|
totals.filesScanned++;
|
|
157
161
|
const stats = { repaired: 0, caseLossy: 0, unreversed: new Set(), samples: [] };
|
|
158
162
|
|
|
159
163
|
try {
|
|
160
|
-
if (
|
|
164
|
+
if (!LOCALE_FILE_FORMATS.includes(flatFormat)) {
|
|
165
|
+
throw new Error(`locale format "${flatFormat}" is not supported by this version of champollion`);
|
|
166
|
+
}
|
|
167
|
+
if (flatFormat === 'json') {
|
|
161
168
|
const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
162
169
|
const repaired = repairNode(parsed, converterKey, stats);
|
|
163
170
|
if (stats.repaired > 0 && !dry) {
|