champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
package/lib/content.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Markdown content translation — translates
|
|
2
|
+
* Markdown content translation — translates the Markdown/MDX files in a
|
|
3
|
+
* `contentDir`: a Hugo site's content/ folder, or any folder of Markdown
|
|
4
|
+
* (a Next.js app's newsletters/, a docs/ folder…). The naming rule below is
|
|
5
|
+
* the same for both; detectContentSite() only labels which one it is.
|
|
3
6
|
*
|
|
4
7
|
* WHY: Hugo stores page content as Markdown files with YAML front matter.
|
|
5
8
|
* Unlike i18n string files (key→value pairs), content files need:
|
|
@@ -9,16 +12,18 @@
|
|
|
9
12
|
* 3. Body translation — send the protected Markdown to the LLM
|
|
10
13
|
* 4. Reassembly — restore protected blocks and rebuild the file
|
|
11
14
|
*
|
|
12
|
-
* Hugo's translation-by-filename convention:
|
|
15
|
+
* Hugo's translation-by-filename convention (used for every contentDir):
|
|
13
16
|
* content/posts/my-post.md → default language
|
|
14
17
|
* content/posts/my-post.fr.md → French
|
|
15
18
|
* content/posts/my-post.ja.md → Japanese
|
|
19
|
+
* newsletters/2026-10.md → newsletters/2026-10.crk.md (Plains Cree)
|
|
16
20
|
*
|
|
17
21
|
* This module handles the parse→protect→translate→restore→write pipeline.
|
|
18
22
|
*/
|
|
19
23
|
|
|
20
24
|
import fs from 'node:fs';
|
|
21
25
|
import path from 'node:path';
|
|
26
|
+
import { nameRules } from './name-rules.js';
|
|
22
27
|
|
|
23
28
|
// Sentinel used for protected block placeholders. Uses Unicode brackets
|
|
24
29
|
// that are extremely unlikely to appear in real content, making them
|
|
@@ -481,7 +486,7 @@ function hasOrphanedPlaceholders(text) {
|
|
|
481
486
|
// -----------------------------------------------------------------
|
|
482
487
|
|
|
483
488
|
/**
|
|
484
|
-
* Scan a Hugo content
|
|
489
|
+
* Scan a content directory (Hugo's content/ or any folder) for source-language Markdown files.
|
|
485
490
|
*
|
|
486
491
|
* Uses Hugo's filename convention: files without a language suffix
|
|
487
492
|
* (e.g., my-post.md) or with the source language suffix
|
|
@@ -523,14 +528,95 @@ function discoverContentFiles(contentDir, sourceLocale) {
|
|
|
523
528
|
}
|
|
524
529
|
|
|
525
530
|
/**
|
|
526
|
-
* Check if a string looks like a language code
|
|
527
|
-
* optionally
|
|
531
|
+
* Check if a string looks like a language code: 2-3 lowercase letters,
|
|
532
|
+
* optionally a script (Hant, Latn) and/or a region (TW, BR, 419) —
|
|
533
|
+
* the shapes a target code takes in a translated file's name.
|
|
528
534
|
*
|
|
529
535
|
* WHY: We need to distinguish "my-post.md" (no lang suffix) from
|
|
530
536
|
* "my-post.fr.md" (French) and "version.2.md" (not a lang code).
|
|
537
|
+
* Script and numeric-region codes matter: before they were recognised,
|
|
538
|
+
* "my-post.zh-Hant.md" (written by sync for a zh-Hant target) was taken
|
|
539
|
+
* for a SOURCE file on the next run and translated into every target.
|
|
531
540
|
*/
|
|
532
541
|
function isLikelyLangCode(str) {
|
|
533
|
-
return /^[a-z]{2,3}(-[A-Z]{2})?$/.test(str);
|
|
542
|
+
return /^[a-z]{2,3}(?:-[A-Z][a-z]{3})?(?:-(?:[A-Z]{2}|\d{3}))?$/.test(str);
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// -----------------------------------------------------------------
|
|
546
|
+
// Which kind of site a content directory belongs to
|
|
547
|
+
// -----------------------------------------------------------------
|
|
548
|
+
|
|
549
|
+
// Hugo's own configuration file names (Hugo ≥ 0.110 prefers hugo.*).
|
|
550
|
+
const HUGO_CONFIG_FILES = ['hugo.toml', 'hugo.yaml', 'hugo.yml', 'hugo.json'];
|
|
551
|
+
// Older Hugo sites use config.*, but so do many other tools — such a file
|
|
552
|
+
// counts as Hugo evidence only when it carries a setting only Hugo uses.
|
|
553
|
+
const GENERIC_CONFIG_FILES = ['config.toml', 'config.yaml', 'config.yml', 'config.json'];
|
|
554
|
+
const HUGO_SETTING_REGEX = /^[ \t]*["']?(?:baseURL|baseurl|defaultContentLanguage|languageCode|disableKinds|enableRobotsTXT|hasCJKLanguage|enableGitInfo|enableEmoji|summaryLength|pluralizeListTitles)["']?[ \t]*[:=]/m;
|
|
555
|
+
|
|
556
|
+
function isFile(p) {
|
|
557
|
+
try { return fs.statSync(p).isFile(); } catch { return false; }
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
function isDirectory(p) {
|
|
561
|
+
try { return fs.statSync(p).isDirectory(); } catch { return false; }
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Say what a content directory belongs to — for the sync banner, which used
|
|
566
|
+
* to announce "Detected framework: Hugo" for ANY contentDir, including a
|
|
567
|
+
* plain folder of Markdown newsletters in a Next.js app. Hugo is reported
|
|
568
|
+
* only on real Hugo evidence, looked for in the project root and in the
|
|
569
|
+
* folder that holds the content directory (Hugo's own layout: site/content):
|
|
570
|
+
*
|
|
571
|
+
* - a hugo.toml / hugo.yaml / hugo.yml / hugo.json,
|
|
572
|
+
* - Hugo's config directory (config/_default/…),
|
|
573
|
+
* - a config.toml / .yaml / .yml / .json carrying a Hugo-only setting
|
|
574
|
+
* (baseURL, defaultContentLanguage, languageCode, …),
|
|
575
|
+
* - an archetypes/ folder, or a layouts/ folder of Hugo templates
|
|
576
|
+
* (layouts/_default/, or .html files).
|
|
577
|
+
*
|
|
578
|
+
* Detection changes no behaviour: every content directory is translated the
|
|
579
|
+
* same way — each translation written beside its source as
|
|
580
|
+
* <name>.<locale>.md (or .mdx).
|
|
581
|
+
*
|
|
582
|
+
* @param {string} contentDir - The configured content directory
|
|
583
|
+
* @param {string} [cwd] - Project root
|
|
584
|
+
* @returns {{ site: 'hugo'|'markdown', evidence: string|null }}
|
|
585
|
+
* evidence: the project-relative path that identified a Hugo site.
|
|
586
|
+
*/
|
|
587
|
+
function detectContentSite(contentDir, cwd = process.cwd()) {
|
|
588
|
+
const root = path.resolve(cwd);
|
|
589
|
+
const roots = [...new Set([root, path.dirname(path.resolve(cwd, contentDir))])];
|
|
590
|
+
const rel = (p) => path.relative(root, p) || '.';
|
|
591
|
+
|
|
592
|
+
for (const dir of roots) {
|
|
593
|
+
for (const name of HUGO_CONFIG_FILES) {
|
|
594
|
+
const p = path.join(dir, name);
|
|
595
|
+
if (isFile(p)) return { site: 'hugo', evidence: rel(p) };
|
|
596
|
+
}
|
|
597
|
+
for (const name of [...HUGO_CONFIG_FILES, ...GENERIC_CONFIG_FILES]) {
|
|
598
|
+
const p = path.join(dir, 'config', '_default', name);
|
|
599
|
+
if (isFile(p)) return { site: 'hugo', evidence: rel(p) };
|
|
600
|
+
}
|
|
601
|
+
for (const name of GENERIC_CONFIG_FILES) {
|
|
602
|
+
const p = path.join(dir, name);
|
|
603
|
+
if (!isFile(p)) continue;
|
|
604
|
+
let text = '';
|
|
605
|
+
try { text = fs.readFileSync(p, 'utf-8'); } catch { continue; }
|
|
606
|
+
if (HUGO_SETTING_REGEX.test(text)) return { site: 'hugo', evidence: `${rel(p)} (Hugo settings)` };
|
|
607
|
+
}
|
|
608
|
+
const archetypes = path.join(dir, 'archetypes');
|
|
609
|
+
if (isDirectory(archetypes)) return { site: 'hugo', evidence: `${rel(archetypes)}/` };
|
|
610
|
+
const layouts = path.join(dir, 'layouts');
|
|
611
|
+
if (isDirectory(layouts)) {
|
|
612
|
+
let hugoTemplates = isDirectory(path.join(layouts, '_default'));
|
|
613
|
+
if (!hugoTemplates) {
|
|
614
|
+
try { hugoTemplates = fs.readdirSync(layouts).some(n => n.endsWith('.html')); } catch { hugoTemplates = false; }
|
|
615
|
+
}
|
|
616
|
+
if (hugoTemplates) return { site: 'hugo', evidence: `${rel(layouts)}/` };
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
return { site: 'markdown', evidence: null };
|
|
534
620
|
}
|
|
535
621
|
|
|
536
622
|
/**
|
|
@@ -596,7 +682,7 @@ Rules:
|
|
|
596
682
|
- Translate ALL human-readable text in the Markdown.
|
|
597
683
|
- Preserve ALL Markdown formatting: headers (#), bold (**), italic (*), links, images, lists, blockquotes, etc.${placeholderRule}
|
|
598
684
|
- Preserve all line breaks, paragraph spacing, and document structure.
|
|
599
|
-
|
|
685
|
+
${nameRules(options.protectedTerms)}
|
|
600
686
|
- Translate link text but preserve link URLs. For example: [Read more](url) → [Lire la suite](url)
|
|
601
687
|
- Return ONLY the translated Markdown. No code fences, no explanation, no preamble.
|
|
602
688
|
|
|
@@ -724,6 +810,7 @@ export {
|
|
|
724
810
|
buildContentPrompt,
|
|
725
811
|
reassembleContentFile,
|
|
726
812
|
isLikelyLangCode,
|
|
813
|
+
detectContentSite,
|
|
727
814
|
DEFAULT_TRANSLATABLE_FIELDS,
|
|
728
815
|
NON_TRANSLATABLE_NESTED_FIELDS,
|
|
729
816
|
PLACEHOLDER_PREFIX,
|
|
@@ -34,8 +34,9 @@ import { classifyLicense, TIERS } from './license-gate.mjs';
|
|
|
34
34
|
|
|
35
35
|
// ---------------------------------------------------------------------------
|
|
36
36
|
// LICENSE OPTIONS — the plain-language picklist. `commercial` / `redistribution`
|
|
37
|
-
// are the booleans written onto the card's `license` block
|
|
38
|
-
// terms
|
|
37
|
+
// are the booleans written onto the card's `license` block: the licence's OWN
|
|
38
|
+
// terms — what it lets others do — never which of our lanes the corpus may
|
|
39
|
+
// enter. That is `publicEligible`, the author-facing hint; the real
|
|
39
40
|
// gate is gatePublicRegistration(), which cross-checks license-gate.mjs and
|
|
40
41
|
// fails safe to blocked.
|
|
41
42
|
// ---------------------------------------------------------------------------
|
|
@@ -72,9 +73,12 @@ export const LICENSE_OPTIONS = [
|
|
|
72
73
|
label: 'CC-BY-NC-4.0',
|
|
73
74
|
spdx: 'CC-BY-NC-4.0',
|
|
74
75
|
commercial: false,
|
|
75
|
-
|
|
76
|
+
// CC BY-NC 4.0 §2(a)(1)(B): the licence lets anyone share the material
|
|
77
|
+
// for non-commercial purposes. It said `false` here — a statement about
|
|
78
|
+
// OUR public lane written as a licence term (the lane is publicEligible).
|
|
79
|
+
redistribution: true,
|
|
76
80
|
publicEligible: false,
|
|
77
|
-
explanation: 'Free for non-commercial use only. Cannot enter the public ranked or commercial lanes — register privately or local-only.',
|
|
81
|
+
explanation: 'Free for non-commercial use and sharing only. Cannot enter the public ranked or commercial lanes — register privately or local-only.',
|
|
78
82
|
},
|
|
79
83
|
{
|
|
80
84
|
key: 'cc-by-nc-sa-4.0',
|
|
@@ -234,23 +238,75 @@ export function resolveLicense(input) {
|
|
|
234
238
|
);
|
|
235
239
|
if (hit) return hit;
|
|
236
240
|
|
|
237
|
-
// Anything else:
|
|
238
|
-
//
|
|
241
|
+
// Anything else: an SPDX id the author typed. A standard licence the gate
|
|
242
|
+
// recognises (CC-BY-2.0, MIT, CC-BY-NC-SA-3.0 …) is recorded as the
|
|
243
|
+
// standard licence it is, with that licence's own terms; only an id the
|
|
244
|
+
// gate does not know — or a LicenseRef-, which is never a standard SPDX
|
|
245
|
+
// id — is custom/unconfirmed and fails safe. CC-BY-2.0 used to be called
|
|
246
|
+
// "Custom/unconfirmed" on the card (Round 14, researcher persona).
|
|
239
247
|
const cls = classifyLicense({ license_spdx: raw });
|
|
240
|
-
const
|
|
241
|
-
|
|
248
|
+
const terms = /^LicenseRef-/i.test(raw) ? null : STANDARD_TIER_TERMS[cls.tier];
|
|
249
|
+
if (terms) {
|
|
250
|
+
return {
|
|
251
|
+
key: raw.toLowerCase(),
|
|
252
|
+
label: raw,
|
|
253
|
+
spdx: raw,
|
|
254
|
+
commercial: terms.commercial,
|
|
255
|
+
redistribution: terms.redistribution,
|
|
256
|
+
publicEligible: terms.publicEligible,
|
|
257
|
+
explanation: `${raw} — a standard licence (${cls.tier}): ${terms.plain}`,
|
|
258
|
+
standard: true,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
242
261
|
return {
|
|
243
262
|
key: 'custom',
|
|
244
263
|
label: raw,
|
|
245
264
|
spdx: raw,
|
|
246
|
-
commercial:
|
|
247
|
-
redistribution:
|
|
248
|
-
publicEligible:
|
|
249
|
-
explanation: `Custom license "${raw}"
|
|
265
|
+
commercial: false,
|
|
266
|
+
redistribution: false,
|
|
267
|
+
publicEligible: false,
|
|
268
|
+
explanation: `Custom license "${raw}" (${cls.tier} to the license gate) — treated as not-redistributable until its terms are confirmed.`,
|
|
250
269
|
custom: true,
|
|
251
270
|
};
|
|
252
271
|
}
|
|
253
272
|
|
|
273
|
+
/**
|
|
274
|
+
* A standard licence's own terms, by the license gate's tier: what the
|
|
275
|
+
* licence lets others do (`commercial`, `redistribution`) — never a statement
|
|
276
|
+
* about which of OUR lanes it may enter (that is `publicEligible`, and the
|
|
277
|
+
* public gate re-checks it).
|
|
278
|
+
*/
|
|
279
|
+
const STANDARD_TIER_TERMS = Object.freeze({
|
|
280
|
+
[TIERS.PERMISSIVE]: { commercial: true, redistribution: true, publicEligible: true,
|
|
281
|
+
plain: 'anyone may use and share it, commercially too, under its terms (attribution where it asks).' },
|
|
282
|
+
[TIERS.SHAREALIKE]: { commercial: true, redistribution: true, publicEligible: true,
|
|
283
|
+
plain: 'anyone may use and share it, commercially too; what is built on it is shared under the same terms.' },
|
|
284
|
+
[TIERS.NONCOMMERCIAL]: { commercial: false, redistribution: true, publicEligible: false,
|
|
285
|
+
plain: 'non-commercial use and sharing only, under the same terms.' },
|
|
286
|
+
[TIERS.NODERIVATIVES]: { commercial: true, redistribution: true, publicEligible: false,
|
|
287
|
+
plain: 'may be shared verbatim under its terms; no modified versions may be distributed.' },
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* The licence's redistribution term, in the card's usageRestrictions enum:
|
|
292
|
+
* 'prohibited' when the licence does not allow it, 'permitted' for a
|
|
293
|
+
* permissive licence, 'same-terms' for any other that allows it (share-alike,
|
|
294
|
+
* non-commercial, no-derivatives: shared only under the licence's own terms).
|
|
295
|
+
* From the LICENCE only — never from the exposure tier: a local-only card
|
|
296
|
+
* used to say "prohibited" here beside license.redistribution: true, and
|
|
297
|
+
* `mt-eval contest prepare` then told the organizer not to release a
|
|
298
|
+
* CC-BY-2.0 dev set (Round 14, researcher persona). Where the steward lets
|
|
299
|
+
* the text travel is the separate `transmission` mark.
|
|
300
|
+
*
|
|
301
|
+
* @param {{spdx: string, redistribution: boolean}} license - the card's license block
|
|
302
|
+
* @returns {'prohibited'|'permitted'|'same-terms'}
|
|
303
|
+
*/
|
|
304
|
+
export function redistributionTerm(license) {
|
|
305
|
+
if (license.redistribution !== true) return 'prohibited';
|
|
306
|
+
const tier = classifyLicense({ license_spdx: license.spdx }).tier;
|
|
307
|
+
return tier === TIERS.PERMISSIVE ? 'permitted' : 'same-terms';
|
|
308
|
+
}
|
|
309
|
+
|
|
254
310
|
/**
|
|
255
311
|
* Resolve an exposure-tier choice to a tier object. Accepts a key, a label, a
|
|
256
312
|
* 1-based index, or the aliases 'local'/'secret'/'sovereign'. Defaults to the
|
|
@@ -292,7 +348,9 @@ export function deriveLicenseBlock(option) {
|
|
|
292
348
|
commercial: option.commercial,
|
|
293
349
|
redistribution: option.redistribution,
|
|
294
350
|
aiTraining: option.aiTraining === undefined ? null : option.aiTraining,
|
|
295
|
-
|
|
351
|
+
// Only a licence whose terms are not known: an id the gate does not
|
|
352
|
+
// recognise, or the picklist's "Other / custom (unconfirmed)".
|
|
353
|
+
notes: option.custom || option.key === 'other' ? 'Custom/unconfirmed license — verify terms before any commercial or redistribution use.' : null,
|
|
296
354
|
};
|
|
297
355
|
}
|
|
298
356
|
|
|
@@ -396,15 +454,58 @@ export function slugify(text) {
|
|
|
396
454
|
}
|
|
397
455
|
|
|
398
456
|
/**
|
|
399
|
-
*
|
|
400
|
-
*
|
|
457
|
+
* What a registered set is FOR, as the author states it (`--role`). Optional:
|
|
458
|
+
* the card id carries a role segment only when the author gave one.
|
|
459
|
+
*
|
|
460
|
+
* WHY optional: the id scheme does not require a role — the corpora-card
|
|
461
|
+
* schema's id pattern is `^(ref|eval)-[a-z0-9][a-z0-9-]*$`, and tracked cards
|
|
462
|
+
* already exist without one (eval-in22-gen-v1, eval-smol-doc-v1). No consumer
|
|
463
|
+
* (registry builder, harness, MCP server, database) parses the role out of an
|
|
464
|
+
* id. Stamping `dev` on every registration (2026-10-03 hospital persona) told
|
|
465
|
+
* readers a nurse-checked held-out TEST set was a development split.
|
|
466
|
+
*/
|
|
467
|
+
export const CORPUS_ROLES = Object.freeze(['test', 'dev', 'train']);
|
|
468
|
+
|
|
469
|
+
/** Normalise a --role value: lower-cased + trimmed, or null when not given. */
|
|
470
|
+
export function normalizeRole(value) {
|
|
471
|
+
if (value === undefined || value === null || value === false) return null;
|
|
472
|
+
const s = String(value).trim().toLowerCase();
|
|
473
|
+
return s ? s : null;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* The id slug: from the corpus NAME, falling back to the publisher only when
|
|
478
|
+
* the name has no a–z/0–9 characters (e.g. a name written in syllabics), then
|
|
479
|
+
* to 'custom'. Before 2026-10-03 the publisher won whenever one was given, so
|
|
480
|
+
* "Hospital nurse-checked test" by "Hospital clinical informatics" became
|
|
481
|
+
* eval-…-hospital-clinical-informatics-dev-v1.
|
|
401
482
|
*/
|
|
402
|
-
export function
|
|
483
|
+
export function deriveIdSlug({ name, publisher } = {}) {
|
|
484
|
+
return slugify(name) || slugify(publisher) || 'custom';
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* Derive a NEW card id: eval-<src>-<tgt>-<slug>[-<role>]-v<major>.
|
|
489
|
+
*
|
|
490
|
+
* The role segment appears only when the author stated one, and is not
|
|
491
|
+
* repeated when the slug already ends with it ("… test" + --role test →
|
|
492
|
+
* …-test-v1, not …-test-test-v1). Only ever used for a registration that has
|
|
493
|
+
* no id yet — an id already registered is never re-derived.
|
|
494
|
+
*/
|
|
495
|
+
export function deriveCardId({ source, target, slug, role = null, version = '1' }) {
|
|
403
496
|
const major = String(version).split('.')[0] || '1';
|
|
404
497
|
const tag = slug && slug.length ? slug : 'custom';
|
|
405
|
-
|
|
498
|
+
const r = normalizeRole(role);
|
|
499
|
+
if (r !== null && !CORPUS_ROLES.includes(r)) {
|
|
500
|
+
throw new Error(`Unknown role '${role}'. Use one of: ${CORPUS_ROLES.join(', ')} (or none).`);
|
|
501
|
+
}
|
|
502
|
+
const withRole = r === null || tag === r || tag.endsWith(`-${r}`) ? tag : `${tag}-${r}`;
|
|
503
|
+
return `eval-${source}-${target}-${withRole}-v${major}`;
|
|
406
504
|
}
|
|
407
505
|
|
|
506
|
+
/** The corpora-card schema's id pattern (cli/shared/schemas/corpora-card.schema.json). */
|
|
507
|
+
export const CARD_ID_PATTERN = /^(ref|eval)-[a-z0-9][a-z0-9-]*$/;
|
|
508
|
+
|
|
408
509
|
/**
|
|
409
510
|
* Build a corpora card object for a registration.
|
|
410
511
|
*
|
|
@@ -421,12 +522,17 @@ export function deriveCardId({ source, target, slug, version = '1' }) {
|
|
|
421
522
|
* @param {string|null} [o.sha256] built-corpus hash (optional)
|
|
422
523
|
* @param {object} o.licenseOption a resolveLicense() result
|
|
423
524
|
* @param {object} o.tier a resolveTier() result
|
|
424
|
-
* @param {string} [o.contaminationRisk] NONE|LOW|MEDIUM|HIGH
|
|
525
|
+
* @param {string} [o.contaminationRisk] NONE|LOW|MEDIUM|HIGH, or UNCHECKED (the
|
|
526
|
+
* file could not be compared with the public corpora and no grade was stated)
|
|
527
|
+
* @param {string|null} [o.role] test|dev|train — `test` writes the size under `test`
|
|
528
|
+
* @param {boolean} [o.dataRead] a local data file was read (to count + checksum)
|
|
425
529
|
* @param {string} [o.contaminationReasoning]
|
|
426
530
|
* @param {number} o.size
|
|
427
531
|
* @param {string} [o.sizeUnit]
|
|
428
532
|
* @param {string} o.domain
|
|
429
533
|
* @param {boolean} [o.doNotTrain]
|
|
534
|
+
* @param {string|null} [o.transmission] 'local-only' when the data file is already
|
|
535
|
+
* marked local-only (its sidecar); a local-only tier is marked regardless
|
|
430
536
|
* @param {string} o.addedAt ISO date (injected for determinism)
|
|
431
537
|
* @param {string} [o.populatedFrom]
|
|
432
538
|
* @param {object} [o.sealed] content-free sealed block (sealed tier
|
|
@@ -434,6 +540,21 @@ export function deriveCardId({ source, target, slug, version = '1' }) {
|
|
|
434
540
|
* ciphertextDigest/aad + key + qualifier refs). NEVER contains text.
|
|
435
541
|
* @returns {object} a corpora card (metadata only — never contains text)
|
|
436
542
|
*/
|
|
543
|
+
/**
|
|
544
|
+
* What registration did with the corpus text, truthfully (Round 9, hospital
|
|
545
|
+
* persona: a local-only card said "corpus content was never read" while the
|
|
546
|
+
* command had just read the file to count and checksum it).
|
|
547
|
+
*/
|
|
548
|
+
function readNote(o) {
|
|
549
|
+
if (o.tier.key === 'sealed') {
|
|
550
|
+
return 'Metadata only — the plaintext was read on this machine only to encrypt it; only ciphertext and this content-free card were written, and no text was uploaded or hosted.';
|
|
551
|
+
}
|
|
552
|
+
if (o.dataRead) {
|
|
553
|
+
return 'Metadata only — the text was read on this machine only to count its entries and compute its sha256; none of it was uploaded or hosted, and none of it left the machine.';
|
|
554
|
+
}
|
|
555
|
+
return 'Metadata only — corpus content was never read, uploaded, or hosted.';
|
|
556
|
+
}
|
|
557
|
+
|
|
437
558
|
export function buildCorpusCard(o) {
|
|
438
559
|
const tierKey = o.tier.key;
|
|
439
560
|
const isPublic = tierKey === 'public';
|
|
@@ -464,30 +585,54 @@ export function buildCorpusCard(o) {
|
|
|
464
585
|
if (o.licenseUrl) source.license_url = o.licenseUrl;
|
|
465
586
|
}
|
|
466
587
|
|
|
467
|
-
//
|
|
468
|
-
//
|
|
469
|
-
//
|
|
470
|
-
|
|
588
|
+
// The split block — size, unit and domain, under the split the author
|
|
589
|
+
// named: `test` for a test set (the schema's public test split), `dev`
|
|
590
|
+
// otherwise (a dev set, a role not stated, and `train`, which the schema
|
|
591
|
+
// has no block for). A test set used to be written under `dev` (Round 9,
|
|
592
|
+
// hospital persona). Readers of `dev` (arena build_registry, corpus_fetch)
|
|
593
|
+
// fall back to `test`. dataFile (a hosted-content path) is set ONLY for the
|
|
594
|
+
// public tier, where the builder reproduces it from source;
|
|
595
|
+
// private/local-only never name a data file. It is named after the card
|
|
596
|
+
// id: a pair-only name stamped a `dev` role the author never stated, and
|
|
597
|
+
// two public cards for one pair shared one path.
|
|
598
|
+
const splitKey = o.role === 'test' ? 'test' : 'dev';
|
|
599
|
+
const split = {
|
|
471
600
|
size: o.size,
|
|
472
601
|
sizeUnit: o.sizeUnit || 'entries',
|
|
473
602
|
domain: o.domain,
|
|
474
603
|
};
|
|
475
604
|
if (isPublic) {
|
|
476
|
-
|
|
477
|
-
|
|
605
|
+
split.dataFile = `curated/${o.id}.json`;
|
|
606
|
+
split.format = 'harness-json';
|
|
478
607
|
}
|
|
479
608
|
|
|
480
|
-
|
|
481
|
-
|
|
609
|
+
// Training: ONE source of truth, doNotTrain (default true — an evaluation
|
|
610
|
+
// set is not trained on). A licence that itself refuses training (the
|
|
611
|
+
// steward-terms options: license.aiTraining false) makes it true.
|
|
612
|
+
// usageRestrictions.training says only WHO set it, and agrees with it.
|
|
613
|
+
// It used to say "discouraged" (not prohibited) beside doNotTrain: true
|
|
614
|
+
// ("must not"), and "prohibited-by-license" for any non-commercial licence
|
|
615
|
+
// although NC says nothing about training (Round 14, researcher persona).
|
|
616
|
+
const licenceRefusesTraining = license.aiTraining === false;
|
|
617
|
+
const doNotTrain = o.doNotTrain !== false || licenceRefusesTraining;
|
|
618
|
+
|
|
619
|
+
// Each term is stated once, from where it comes from: commercial use and
|
|
620
|
+
// redistribution by the licence (license.*; commercialUse null = "defer to
|
|
621
|
+
// license.commercial", as the schema reads it); redistribution is repeated
|
|
622
|
+
// in the enum `mt-eval contest prepare` reads, from the licence only —
|
|
623
|
+
// never from the exposure tier (see redistributionTerm).
|
|
482
624
|
const usageRestrictions = {
|
|
483
|
-
training: doNotTrain ?
|
|
484
|
-
commercialUse:
|
|
485
|
-
redistribution:
|
|
486
|
-
? (o.licenseOption.spdx.toUpperCase().includes('-SA') ? 'same-terms' : 'permitted')
|
|
487
|
-
: 'prohibited',
|
|
625
|
+
training: !doNotTrain ? 'permitted' : (licenceRefusesTraining ? 'prohibited-by-license' : 'prohibited-by-community'),
|
|
626
|
+
commercialUse: null,
|
|
627
|
+
redistribution: redistributionTerm(license),
|
|
488
628
|
communityNotes: null,
|
|
489
629
|
};
|
|
490
630
|
|
|
631
|
+
// Where the steward lets the text travel — a mark, not a licence term
|
|
632
|
+
// (schema: `transmission`). A local-only set, or a file already marked
|
|
633
|
+
// local-only (its sidecar keeps the mark; the card says the same).
|
|
634
|
+
const localOnly = tierKey === 'local-only' || o.transmission === 'local-only';
|
|
635
|
+
|
|
491
636
|
const card = {
|
|
492
637
|
id: o.id,
|
|
493
638
|
type: 'eval',
|
|
@@ -501,16 +646,17 @@ export function buildCorpusCard(o) {
|
|
|
501
646
|
description: o.description,
|
|
502
647
|
source,
|
|
503
648
|
license,
|
|
504
|
-
|
|
649
|
+
[splitKey]: split,
|
|
505
650
|
contamination: { risk, reasoning },
|
|
506
651
|
doNotTrain,
|
|
507
652
|
exposureTier: tierKey,
|
|
653
|
+
...(localOnly ? { transmission: 'local-only' } : {}),
|
|
508
654
|
usageRestrictions,
|
|
509
655
|
_provenance: {
|
|
510
656
|
addedAt: o.addedAt,
|
|
511
657
|
populatedFrom:
|
|
512
658
|
o.populatedFrom ||
|
|
513
|
-
`Registered via 'champollion register-corpus' (exposureTier=${tierKey}).
|
|
659
|
+
`Registered via 'champollion register-corpus' (exposureTier=${tierKey}). ${readNote(o)}`,
|
|
514
660
|
},
|
|
515
661
|
};
|
|
516
662
|
|
|
@@ -569,11 +715,13 @@ export function resolveDestination({ tier, id, corporaCardsDir, localDir }) {
|
|
|
569
715
|
* @param {object} o.tier
|
|
570
716
|
* @param {object|null} o.licenseOption
|
|
571
717
|
* @param {{source?:string,target?:string}} o.pair
|
|
718
|
+
* @param {string|null} [o.pairError] why --pair could not be read (lib/language-pair.js), if it could not
|
|
572
719
|
* @param {string} o.name
|
|
573
720
|
* @param {string|null} [o.repoUrl]
|
|
574
721
|
* @param {string|null} [o.builder]
|
|
575
722
|
* @param {number|null} [o.size]
|
|
576
723
|
* @param {string} [o.domain]
|
|
724
|
+
* @param {string|null} [o.role] optional: test | dev | train (see CORPUS_ROLES)
|
|
577
725
|
* @param {string|null} [o.custodianGroupId] sealed tier: custodian group id
|
|
578
726
|
* @param {string|null} [o.thresholdPublicKey] sealed tier: threshold pubkey (path/value)
|
|
579
727
|
* @param {string|null} [o.sealInput] sealed tier: local corpus file to encrypt
|
|
@@ -583,11 +731,22 @@ export function resolveDestination({ tier, id, corporaCardsDir, localDir }) {
|
|
|
583
731
|
export function validateRegistration(o) {
|
|
584
732
|
const errors = [];
|
|
585
733
|
if (!o.name || !String(o.name).trim()) errors.push('A corpus name is required (--name).');
|
|
586
|
-
if (
|
|
587
|
-
|
|
734
|
+
if (o.pairError) {
|
|
735
|
+
// --pair was given but could not be read: say why, not "a code is required".
|
|
736
|
+
errors.push(o.pairError);
|
|
737
|
+
} else {
|
|
738
|
+
if (!o.pair || !o.pair.source) errors.push('A source language code is required (--pair "src>tgt" or src-tgt, or --source-lang).');
|
|
739
|
+
if (!o.pair || !o.pair.target) errors.push('A target language code is required (--pair "src>tgt" or src-tgt, or --target-lang).');
|
|
740
|
+
}
|
|
588
741
|
if (!o.licenseOption) errors.push('A license is required (--license). Run with --list to see options.');
|
|
589
742
|
if (!(Number(o.size) > 0)) errors.push('A positive corpus size is required (--size).');
|
|
590
743
|
if (!o.domain || !String(o.domain).trim()) errors.push('A domain is required (--domain), e.g. news, conversational, educational.');
|
|
744
|
+
const role = o.role === true ? null : normalizeRole(o.role);
|
|
745
|
+
if (o.role === true) {
|
|
746
|
+
errors.push(`--role needs a value: ${CORPUS_ROLES.join(', ')} (or leave --role out, and the id names no role).`);
|
|
747
|
+
} else if (role !== null && !CORPUS_ROLES.includes(role)) {
|
|
748
|
+
errors.push(`Unknown --role '${o.role}'. Use ${CORPUS_ROLES.join(', ')} — or leave --role out, and the id names no role.`);
|
|
749
|
+
}
|
|
591
750
|
|
|
592
751
|
if (o.tier.key === 'public') {
|
|
593
752
|
if (!o.repoUrl) errors.push('Public registration needs a fetch-from-source pointer (--repo-url) — we never host content.');
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One wording for a cost estimate, wherever a price is named beside a command
|
|
3
|
+
* or an answer: the redo notes of `sync`, the named-key cache note, and the
|
|
4
|
+
* MCP server's `translate` answer (Round 9: the MCP said "0 USD" for a model
|
|
5
|
+
* on this machine, where the CLI says "$0 API cost (runs on this machine)").
|
|
6
|
+
*
|
|
7
|
+
* Pure, no imports: the MCP server reads it from the package's exports.
|
|
8
|
+
*
|
|
9
|
+
* The estimate is the object a method's estimateCost() returns
|
|
10
|
+
* ({ estimatedCost: number|null, local?: true, … }):
|
|
11
|
+
* - a model served on this machine (local: true, $0) → "$0 API cost (runs on this machine)"
|
|
12
|
+
* - a known price → "est. ~$0.0123"
|
|
13
|
+
* - no published price (null), or no estimate at all → "cost unknown — no published price" / "cost unknown"
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** What a model on this machine costs, in the words every surface uses. */
|
|
17
|
+
export const LOCAL_COST_LABEL = '$0 API cost (runs on this machine)';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {{ estimatedCost?: number|null, local?: boolean }|null|undefined} estimate
|
|
21
|
+
* @returns {string}
|
|
22
|
+
*/
|
|
23
|
+
export function costLabel(estimate) {
|
|
24
|
+
if (!estimate || typeof estimate !== 'object') return 'cost unknown';
|
|
25
|
+
const cost = estimate.estimatedCost;
|
|
26
|
+
if (estimate.local && cost === 0) return LOCAL_COST_LABEL;
|
|
27
|
+
if (typeof cost === 'number' && Number.isFinite(cost)) return `est. ~$${cost.toFixed(4)}`;
|
|
28
|
+
return 'cost unknown — no published price';
|
|
29
|
+
}
|