champollion 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +83 -36
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1689 -87
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +196 -14
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +270 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1127 -344
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +726 -78
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +965 -253
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +964 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +571 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +252 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +68 -128
  70. package/lib/methods/llm.js +80 -31
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +11 -0
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +2 -1
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +880 -17
  117. package/lib/verify.js +1296 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/method-registry.json +2 -0
  126. package/shared/metric-registry.json +96 -18
  127. package/shared/schemas/champollion-plugin.schema.json +4 -0
  128. package/shared/schemas/corpora-card.schema.json +8 -2
  129. package/shared/schemas/method-index-record.schema.json +67 -0
  130. package/shared/schemas/method-registry.schema.json +4 -0
  131. package/shared/schemas/metric-registry.schema.json +55 -1
  132. package/shared/docent/corpus.json +0 -11739
package/lib/content.js CHANGED
@@ -1,5 +1,8 @@
1
1
  /**
2
- * Markdown content translation — translates Hugo content files.
2
+ * Markdown content translation — translates the Markdown/MDX files in a
3
+ * `contentDir`: a Hugo site's content/ folder, or any folder of Markdown
4
+ * (a Next.js app's newsletters/, a docs/ folder…). The naming rule below is
5
+ * the same for both; detectContentSite() only labels which one it is.
3
6
  *
4
7
  * WHY: Hugo stores page content as Markdown files with YAML front matter.
5
8
  * Unlike i18n string files (key→value pairs), content files need:
@@ -9,16 +12,18 @@
9
12
  * 3. Body translation — send the protected Markdown to the LLM
10
13
  * 4. Reassembly — restore protected blocks and rebuild the file
11
14
  *
12
- * Hugo's translation-by-filename convention:
15
+ * Hugo's translation-by-filename convention (used for every contentDir):
13
16
  * content/posts/my-post.md → default language
14
17
  * content/posts/my-post.fr.md → French
15
18
  * content/posts/my-post.ja.md → Japanese
19
+ * newsletters/2026-10.md → newsletters/2026-10.crk.md (Plains Cree)
16
20
  *
17
21
  * This module handles the parse→protect→translate→restore→write pipeline.
18
22
  */
19
23
 
20
24
  import fs from 'node:fs';
21
25
  import path from 'node:path';
26
+ import { nameRules } from './name-rules.js';
22
27
 
23
28
  // Sentinel used for protected block placeholders. Uses Unicode brackets
24
29
  // that are extremely unlikely to appear in real content, making them
@@ -481,7 +486,7 @@ function hasOrphanedPlaceholders(text) {
481
486
  // -----------------------------------------------------------------
482
487
 
483
488
  /**
484
- * Scan a Hugo content directory for source language Markdown files.
489
+ * Scan a content directory (Hugo's content/ or any folder) for source-language Markdown files.
485
490
  *
486
491
  * Uses Hugo's filename convention: files without a language suffix
487
492
  * (e.g., my-post.md) or with the source language suffix
@@ -523,14 +528,95 @@ function discoverContentFiles(contentDir, sourceLocale) {
523
528
  }
524
529
 
525
530
  /**
526
- * Check if a string looks like a language code (2-3 lowercase letters,
527
- * optionally with a region suffix like zh-TW).
531
+ * Check if a string looks like a language code: 2-3 lowercase letters,
532
+ * optionally a script (Hant, Latn) and/or a region (TW, BR, 419) —
533
+ * the shapes a target code takes in a translated file's name.
528
534
  *
529
535
  * WHY: We need to distinguish "my-post.md" (no lang suffix) from
530
536
  * "my-post.fr.md" (French) and "version.2.md" (not a lang code).
537
+ * Script and numeric-region codes matter: before they were recognised,
538
+ * "my-post.zh-Hant.md" (written by sync for a zh-Hant target) was taken
539
+ * for a SOURCE file on the next run and translated into every target.
531
540
  */
532
541
  function isLikelyLangCode(str) {
533
- return /^[a-z]{2,3}(-[A-Z]{2})?$/.test(str);
542
+ return /^[a-z]{2,3}(?:-[A-Z][a-z]{3})?(?:-(?:[A-Z]{2}|\d{3}))?$/.test(str);
543
+ }
544
+
545
+ // -----------------------------------------------------------------
546
+ // Which kind of site a content directory belongs to
547
+ // -----------------------------------------------------------------
548
+
549
+ // Hugo's own configuration file names (Hugo ≥ 0.110 prefers hugo.*).
550
+ const HUGO_CONFIG_FILES = ['hugo.toml', 'hugo.yaml', 'hugo.yml', 'hugo.json'];
551
+ // Older Hugo sites use config.*, but so do many other tools — such a file
552
+ // counts as Hugo evidence only when it carries a setting only Hugo uses.
553
+ const GENERIC_CONFIG_FILES = ['config.toml', 'config.yaml', 'config.yml', 'config.json'];
554
+ const HUGO_SETTING_REGEX = /^[ \t]*["']?(?:baseURL|baseurl|defaultContentLanguage|languageCode|disableKinds|enableRobotsTXT|hasCJKLanguage|enableGitInfo|enableEmoji|summaryLength|pluralizeListTitles)["']?[ \t]*[:=]/m;
555
+
556
+ function isFile(p) {
557
+ try { return fs.statSync(p).isFile(); } catch { return false; }
558
+ }
559
+
560
+ function isDirectory(p) {
561
+ try { return fs.statSync(p).isDirectory(); } catch { return false; }
562
+ }
563
+
564
+ /**
565
+ * Say what a content directory belongs to — for the sync banner, which used
566
+ * to announce "Detected framework: Hugo" for ANY contentDir, including a
567
+ * plain folder of Markdown newsletters in a Next.js app. Hugo is reported
568
+ * only on real Hugo evidence, looked for in the project root and in the
569
+ * folder that holds the content directory (Hugo's own layout: site/content):
570
+ *
571
+ * - a hugo.toml / hugo.yaml / hugo.yml / hugo.json,
572
+ * - Hugo's config directory (config/_default/…),
573
+ * - a config.toml / .yaml / .yml / .json carrying a Hugo-only setting
574
+ * (baseURL, defaultContentLanguage, languageCode, …),
575
+ * - an archetypes/ folder, or a layouts/ folder of Hugo templates
576
+ * (layouts/_default/, or .html files).
577
+ *
578
+ * Detection changes no behaviour: every content directory is translated the
579
+ * same way — each translation written beside its source as
580
+ * <name>.<locale>.md (or .mdx).
581
+ *
582
+ * @param {string} contentDir - The configured content directory
583
+ * @param {string} [cwd] - Project root
584
+ * @returns {{ site: 'hugo'|'markdown', evidence: string|null }}
585
+ * evidence: the project-relative path that identified a Hugo site.
586
+ */
587
+ function detectContentSite(contentDir, cwd = process.cwd()) {
588
+ const root = path.resolve(cwd);
589
+ const roots = [...new Set([root, path.dirname(path.resolve(cwd, contentDir))])];
590
+ const rel = (p) => path.relative(root, p) || '.';
591
+
592
+ for (const dir of roots) {
593
+ for (const name of HUGO_CONFIG_FILES) {
594
+ const p = path.join(dir, name);
595
+ if (isFile(p)) return { site: 'hugo', evidence: rel(p) };
596
+ }
597
+ for (const name of [...HUGO_CONFIG_FILES, ...GENERIC_CONFIG_FILES]) {
598
+ const p = path.join(dir, 'config', '_default', name);
599
+ if (isFile(p)) return { site: 'hugo', evidence: rel(p) };
600
+ }
601
+ for (const name of GENERIC_CONFIG_FILES) {
602
+ const p = path.join(dir, name);
603
+ if (!isFile(p)) continue;
604
+ let text = '';
605
+ try { text = fs.readFileSync(p, 'utf-8'); } catch { continue; }
606
+ if (HUGO_SETTING_REGEX.test(text)) return { site: 'hugo', evidence: `${rel(p)} (Hugo settings)` };
607
+ }
608
+ const archetypes = path.join(dir, 'archetypes');
609
+ if (isDirectory(archetypes)) return { site: 'hugo', evidence: `${rel(archetypes)}/` };
610
+ const layouts = path.join(dir, 'layouts');
611
+ if (isDirectory(layouts)) {
612
+ let hugoTemplates = isDirectory(path.join(layouts, '_default'));
613
+ if (!hugoTemplates) {
614
+ try { hugoTemplates = fs.readdirSync(layouts).some(n => n.endsWith('.html')); } catch { hugoTemplates = false; }
615
+ }
616
+ if (hugoTemplates) return { site: 'hugo', evidence: `${rel(layouts)}/` };
617
+ }
618
+ }
619
+ return { site: 'markdown', evidence: null };
534
620
  }
535
621
 
536
622
  /**
@@ -596,7 +682,7 @@ Rules:
596
682
  - Translate ALL human-readable text in the Markdown.
597
683
  - Preserve ALL Markdown formatting: headers (#), bold (**), italic (*), links, images, lists, blockquotes, etc.${placeholderRule}
598
684
  - Preserve all line breaks, paragraph spacing, and document structure.
599
- - Proper nouns, product names, and technical terms should remain in the source language.
685
+ ${nameRules(options.protectedTerms)}
600
686
  - Translate link text but preserve link URLs. For example: [Read more](url) → [Lire la suite](url)
601
687
  - Return ONLY the translated Markdown. No code fences, no explanation, no preamble.
602
688
 
@@ -724,6 +810,7 @@ export {
724
810
  buildContentPrompt,
725
811
  reassembleContentFile,
726
812
  isLikelyLangCode,
813
+ detectContentSite,
727
814
  DEFAULT_TRANSLATABLE_FIELDS,
728
815
  NON_TRANSLATABLE_NESTED_FIELDS,
729
816
  PLACEHOLDER_PREFIX,
@@ -34,8 +34,9 @@ import { classifyLicense, TIERS } from './license-gate.mjs';
34
34
 
35
35
  // ---------------------------------------------------------------------------
36
36
  // LICENSE OPTIONS — the plain-language picklist. `commercial` / `redistribution`
37
- // are the booleans written onto the card's `license` block (the license's plain
38
- // terms for OUR lanes). `publicEligible` is the author-facing hint; the real
37
+ // are the booleans written onto the card's `license` block: the licence's OWN
38
+ // terms — what it lets others do — never which of our lanes the corpus may
39
+ // enter. That is `publicEligible`, the author-facing hint; the real
39
40
  // gate is gatePublicRegistration(), which cross-checks license-gate.mjs and
40
41
  // fails safe to blocked.
41
42
  // ---------------------------------------------------------------------------
@@ -72,9 +73,12 @@ export const LICENSE_OPTIONS = [
72
73
  label: 'CC-BY-NC-4.0',
73
74
  spdx: 'CC-BY-NC-4.0',
74
75
  commercial: false,
75
- redistribution: false,
76
+ // CC BY-NC 4.0 §2(a)(1)(B): the licence lets anyone share the material
77
+ // for non-commercial purposes. It said `false` here — a statement about
78
+ // OUR public lane written as a licence term (the lane is publicEligible).
79
+ redistribution: true,
76
80
  publicEligible: false,
77
- explanation: 'Free for non-commercial use only. Cannot enter the public ranked or commercial lanes — register privately or local-only.',
81
+ explanation: 'Free for non-commercial use and sharing only. Cannot enter the public ranked or commercial lanes — register privately or local-only.',
78
82
  },
79
83
  {
80
84
  key: 'cc-by-nc-sa-4.0',
@@ -234,23 +238,75 @@ export function resolveLicense(input) {
234
238
  );
235
239
  if (hit) return hit;
236
240
 
237
- // Anything else: a raw SPDX the author typed. Classify it through the gate so
238
- // a real permissive/share-alike id still works, but unknowns fail safe.
241
+ // Anything else: an SPDX id the author typed. A standard licence the gate
242
+ // recognises (CC-BY-2.0, MIT, CC-BY-NC-SA-3.0 …) is recorded as the
243
+ // standard licence it is, with that licence's own terms; only an id the
244
+ // gate does not know — or a LicenseRef-, which is never a standard SPDX
245
+ // id — is custom/unconfirmed and fails safe. CC-BY-2.0 used to be called
246
+ // "Custom/unconfirmed" on the card (Round 14, researcher persona).
239
247
  const cls = classifyLicense({ license_spdx: raw });
240
- const permissive = cls.tier === TIERS.PERMISSIVE;
241
- const shareAlike = cls.tier === TIERS.SHAREALIKE;
248
+ const terms = /^LicenseRef-/i.test(raw) ? null : STANDARD_TIER_TERMS[cls.tier];
249
+ if (terms) {
250
+ return {
251
+ key: raw.toLowerCase(),
252
+ label: raw,
253
+ spdx: raw,
254
+ commercial: terms.commercial,
255
+ redistribution: terms.redistribution,
256
+ publicEligible: terms.publicEligible,
257
+ explanation: `${raw} — a standard licence (${cls.tier}): ${terms.plain}`,
258
+ standard: true,
259
+ };
260
+ }
242
261
  return {
243
262
  key: 'custom',
244
263
  label: raw,
245
264
  spdx: raw,
246
- commercial: permissive || shareAlike,
247
- redistribution: permissive || shareAlike,
248
- publicEligible: permissive || shareAlike,
249
- explanation: `Custom license "${raw}" classified as ${cls.tier} by the license gate.`,
265
+ commercial: false,
266
+ redistribution: false,
267
+ publicEligible: false,
268
+ explanation: `Custom license "${raw}" (${cls.tier} to the license gate) — treated as not-redistributable until its terms are confirmed.`,
250
269
  custom: true,
251
270
  };
252
271
  }
253
272
 
273
+ /**
274
+ * A standard licence's own terms, by the license gate's tier: what the
275
+ * licence lets others do (`commercial`, `redistribution`) — never a statement
276
+ * about which of OUR lanes it may enter (that is `publicEligible`, and the
277
+ * public gate re-checks it).
278
+ */
279
+ const STANDARD_TIER_TERMS = Object.freeze({
280
+ [TIERS.PERMISSIVE]: { commercial: true, redistribution: true, publicEligible: true,
281
+ plain: 'anyone may use and share it, commercially too, under its terms (attribution where it asks).' },
282
+ [TIERS.SHAREALIKE]: { commercial: true, redistribution: true, publicEligible: true,
283
+ plain: 'anyone may use and share it, commercially too; what is built on it is shared under the same terms.' },
284
+ [TIERS.NONCOMMERCIAL]: { commercial: false, redistribution: true, publicEligible: false,
285
+ plain: 'non-commercial use and sharing only, under the same terms.' },
286
+ [TIERS.NODERIVATIVES]: { commercial: true, redistribution: true, publicEligible: false,
287
+ plain: 'may be shared verbatim under its terms; no modified versions may be distributed.' },
288
+ });
289
+
290
+ /**
291
+ * The licence's redistribution term, in the card's usageRestrictions enum:
292
+ * 'prohibited' when the licence does not allow it, 'permitted' for a
293
+ * permissive licence, 'same-terms' for any other that allows it (share-alike,
294
+ * non-commercial, no-derivatives: shared only under the licence's own terms).
295
+ * From the LICENCE only — never from the exposure tier: a local-only card
296
+ * used to say "prohibited" here beside license.redistribution: true, and
297
+ * `mt-eval contest prepare` then told the organizer not to release a
298
+ * CC-BY-2.0 dev set (Round 14, researcher persona). Where the steward lets
299
+ * the text travel is the separate `transmission` mark.
300
+ *
301
+ * @param {{spdx: string, redistribution: boolean}} license - the card's license block
302
+ * @returns {'prohibited'|'permitted'|'same-terms'}
303
+ */
304
+ export function redistributionTerm(license) {
305
+ if (license.redistribution !== true) return 'prohibited';
306
+ const tier = classifyLicense({ license_spdx: license.spdx }).tier;
307
+ return tier === TIERS.PERMISSIVE ? 'permitted' : 'same-terms';
308
+ }
309
+
254
310
  /**
255
311
  * Resolve an exposure-tier choice to a tier object. Accepts a key, a label, a
256
312
  * 1-based index, or the aliases 'local'/'secret'/'sovereign'. Defaults to the
@@ -292,7 +348,9 @@ export function deriveLicenseBlock(option) {
292
348
  commercial: option.commercial,
293
349
  redistribution: option.redistribution,
294
350
  aiTraining: option.aiTraining === undefined ? null : option.aiTraining,
295
- notes: option.custom ? 'Custom/unconfirmed license — verify terms before any commercial or redistribution use.' : null,
351
+ // Only a licence whose terms are not known: an id the gate does not
352
+ // recognise, or the picklist's "Other / custom (unconfirmed)".
353
+ notes: option.custom || option.key === 'other' ? 'Custom/unconfirmed license — verify terms before any commercial or redistribution use.' : null,
296
354
  };
297
355
  }
298
356
 
@@ -396,15 +454,58 @@ export function slugify(text) {
396
454
  }
397
455
 
398
456
  /**
399
- * Derive a default card id from the pair + a name/publisher slug.
400
- * Shape mirrors existing cards: eval-<src>-<tgt>-<slug>-dev-v<major>.
457
+ * What a registered set is FOR, as the author states it (`--role`). Optional:
458
+ * the card id carries a role segment only when the author gave one.
459
+ *
460
+ * WHY optional: the id scheme does not require a role — the corpora-card
461
+ * schema's id pattern is `^(ref|eval)-[a-z0-9][a-z0-9-]*$`, and tracked cards
462
+ * already exist without one (eval-in22-gen-v1, eval-smol-doc-v1). No consumer
463
+ * (registry builder, harness, MCP server, database) parses the role out of an
464
+ * id. Stamping `dev` on every registration (2026-10-03 hospital persona) told
465
+ * readers a nurse-checked held-out TEST set was a development split.
466
+ */
467
+ export const CORPUS_ROLES = Object.freeze(['test', 'dev', 'train']);
468
+
469
+ /** Normalise a --role value: lower-cased + trimmed, or null when not given. */
470
+ export function normalizeRole(value) {
471
+ if (value === undefined || value === null || value === false) return null;
472
+ const s = String(value).trim().toLowerCase();
473
+ return s ? s : null;
474
+ }
475
+
476
+ /**
477
+ * The id slug: from the corpus NAME, falling back to the publisher only when
478
+ * the name has no a–z/0–9 characters (e.g. a name written in syllabics), then
479
+ * to 'custom'. Before 2026-10-03 the publisher won whenever one was given, so
480
+ * "Hospital nurse-checked test" by "Hospital clinical informatics" became
481
+ * eval-…-hospital-clinical-informatics-dev-v1.
401
482
  */
402
- export function deriveCardId({ source, target, slug, version = '1' }) {
483
+ export function deriveIdSlug({ name, publisher } = {}) {
484
+ return slugify(name) || slugify(publisher) || 'custom';
485
+ }
486
+
487
+ /**
488
+ * Derive a NEW card id: eval-<src>-<tgt>-<slug>[-<role>]-v<major>.
489
+ *
490
+ * The role segment appears only when the author stated one, and is not
491
+ * repeated when the slug already ends with it ("… test" + --role test →
492
+ * …-test-v1, not …-test-test-v1). Only ever used for a registration that has
493
+ * no id yet — an id already registered is never re-derived.
494
+ */
495
+ export function deriveCardId({ source, target, slug, role = null, version = '1' }) {
403
496
  const major = String(version).split('.')[0] || '1';
404
497
  const tag = slug && slug.length ? slug : 'custom';
405
- return `eval-${source}-${target}-${tag}-dev-v${major}`;
498
+ const r = normalizeRole(role);
499
+ if (r !== null && !CORPUS_ROLES.includes(r)) {
500
+ throw new Error(`Unknown role '${role}'. Use one of: ${CORPUS_ROLES.join(', ')} (or none).`);
501
+ }
502
+ const withRole = r === null || tag === r || tag.endsWith(`-${r}`) ? tag : `${tag}-${r}`;
503
+ return `eval-${source}-${target}-${withRole}-v${major}`;
406
504
  }
407
505
 
506
+ /** The corpora-card schema's id pattern (cli/shared/schemas/corpora-card.schema.json). */
507
+ export const CARD_ID_PATTERN = /^(ref|eval)-[a-z0-9][a-z0-9-]*$/;
508
+
408
509
  /**
409
510
  * Build a corpora card object for a registration.
410
511
  *
@@ -421,12 +522,17 @@ export function deriveCardId({ source, target, slug, version = '1' }) {
421
522
  * @param {string|null} [o.sha256] built-corpus hash (optional)
422
523
  * @param {object} o.licenseOption a resolveLicense() result
423
524
  * @param {object} o.tier a resolveTier() result
424
- * @param {string} [o.contaminationRisk] NONE|LOW|MEDIUM|HIGH
525
+ * @param {string} [o.contaminationRisk] NONE|LOW|MEDIUM|HIGH, or UNCHECKED (the
526
+ * file could not be compared with the public corpora and no grade was stated)
527
+ * @param {string|null} [o.role] test|dev|train — `test` writes the size under `test`
528
+ * @param {boolean} [o.dataRead] a local data file was read (to count + checksum)
425
529
  * @param {string} [o.contaminationReasoning]
426
530
  * @param {number} o.size
427
531
  * @param {string} [o.sizeUnit]
428
532
  * @param {string} o.domain
429
533
  * @param {boolean} [o.doNotTrain]
534
+ * @param {string|null} [o.transmission] 'local-only' when the data file is already
535
+ * marked local-only (its sidecar); a local-only tier is marked regardless
430
536
  * @param {string} o.addedAt ISO date (injected for determinism)
431
537
  * @param {string} [o.populatedFrom]
432
538
  * @param {object} [o.sealed] content-free sealed block (sealed tier
@@ -434,6 +540,21 @@ export function deriveCardId({ source, target, slug, version = '1' }) {
434
540
  * ciphertextDigest/aad + key + qualifier refs). NEVER contains text.
435
541
  * @returns {object} a corpora card (metadata only — never contains text)
436
542
  */
543
+ /**
544
+ * What registration did with the corpus text, truthfully (Round 9, hospital
545
+ * persona: a local-only card said "corpus content was never read" while the
546
+ * command had just read the file to count and checksum it).
547
+ */
548
+ function readNote(o) {
549
+ if (o.tier.key === 'sealed') {
550
+ return 'Metadata only — the plaintext was read on this machine only to encrypt it; only ciphertext and this content-free card were written, and no text was uploaded or hosted.';
551
+ }
552
+ if (o.dataRead) {
553
+ return 'Metadata only — the text was read on this machine only to count its entries and compute its sha256; none of it was uploaded or hosted, and none of it left the machine.';
554
+ }
555
+ return 'Metadata only — corpus content was never read, uploaded, or hosted.';
556
+ }
557
+
437
558
  export function buildCorpusCard(o) {
438
559
  const tierKey = o.tier.key;
439
560
  const isPublic = tierKey === 'public';
@@ -464,30 +585,54 @@ export function buildCorpusCard(o) {
464
585
  if (o.licenseUrl) source.license_url = o.licenseUrl;
465
586
  }
466
587
 
467
- // dev block — records the consumer-reports length/size + domain. dataFile
468
- // (a hosted-content path) is set ONLY for the public tier, where the builder
469
- // reproduces it from source; private/local-only never name a data file.
470
- const dev = {
588
+ // The split block — size, unit and domain, under the split the author
589
+ // named: `test` for a test set (the schema's public test split), `dev`
590
+ // otherwise (a dev set, a role not stated, and `train`, which the schema
591
+ // has no block for). A test set used to be written under `dev` (Round 9,
592
+ // hospital persona). Readers of `dev` (arena build_registry, corpus_fetch)
593
+ // fall back to `test`. dataFile (a hosted-content path) is set ONLY for the
594
+ // public tier, where the builder reproduces it from source;
595
+ // private/local-only never name a data file. It is named after the card
596
+ // id: a pair-only name stamped a `dev` role the author never stated, and
597
+ // two public cards for one pair shared one path.
598
+ const splitKey = o.role === 'test' ? 'test' : 'dev';
599
+ const split = {
471
600
  size: o.size,
472
601
  sizeUnit: o.sizeUnit || 'entries',
473
602
  domain: o.domain,
474
603
  };
475
604
  if (isPublic) {
476
- dev.dataFile = `curated/${o.pair.source}-${o.pair.target}-dev-v1.json`;
477
- dev.format = 'harness-json';
605
+ split.dataFile = `curated/${o.id}.json`;
606
+ split.format = 'harness-json';
478
607
  }
479
608
 
480
- const doNotTrain = o.doNotTrain !== false; // default true (benchmark integrity)
481
-
609
+ // Training: ONE source of truth, doNotTrain (default true — an evaluation
610
+ // set is not trained on). A licence that itself refuses training (the
611
+ // steward-terms options: license.aiTraining false) makes it true.
612
+ // usageRestrictions.training says only WHO set it, and agrees with it.
613
+ // It used to say "discouraged" (not prohibited) beside doNotTrain: true
614
+ // ("must not"), and "prohibited-by-license" for any non-commercial licence
615
+ // although NC says nothing about training (Round 14, researcher persona).
616
+ const licenceRefusesTraining = license.aiTraining === false;
617
+ const doNotTrain = o.doNotTrain !== false || licenceRefusesTraining;
618
+
619
+ // Each term is stated once, from where it comes from: commercial use and
620
+ // redistribution by the licence (license.*; commercialUse null = "defer to
621
+ // license.commercial", as the schema reads it); redistribution is repeated
622
+ // in the enum `mt-eval contest prepare` reads, from the licence only —
623
+ // never from the exposure tier (see redistributionTerm).
482
624
  const usageRestrictions = {
483
- training: doNotTrain ? (license.commercial ? 'discouraged' : 'prohibited-by-license') : 'permitted',
484
- commercialUse: license.commercial ? 'permitted' : 'prohibited-by-license',
485
- redistribution: isPublic
486
- ? (o.licenseOption.spdx.toUpperCase().includes('-SA') ? 'same-terms' : 'permitted')
487
- : 'prohibited',
625
+ training: !doNotTrain ? 'permitted' : (licenceRefusesTraining ? 'prohibited-by-license' : 'prohibited-by-community'),
626
+ commercialUse: null,
627
+ redistribution: redistributionTerm(license),
488
628
  communityNotes: null,
489
629
  };
490
630
 
631
+ // Where the steward lets the text travel — a mark, not a licence term
632
+ // (schema: `transmission`). A local-only set, or a file already marked
633
+ // local-only (its sidecar keeps the mark; the card says the same).
634
+ const localOnly = tierKey === 'local-only' || o.transmission === 'local-only';
635
+
491
636
  const card = {
492
637
  id: o.id,
493
638
  type: 'eval',
@@ -501,16 +646,17 @@ export function buildCorpusCard(o) {
501
646
  description: o.description,
502
647
  source,
503
648
  license,
504
- dev,
649
+ [splitKey]: split,
505
650
  contamination: { risk, reasoning },
506
651
  doNotTrain,
507
652
  exposureTier: tierKey,
653
+ ...(localOnly ? { transmission: 'local-only' } : {}),
508
654
  usageRestrictions,
509
655
  _provenance: {
510
656
  addedAt: o.addedAt,
511
657
  populatedFrom:
512
658
  o.populatedFrom ||
513
- `Registered via 'champollion register-corpus' (exposureTier=${tierKey}). Metadata only — corpus content was never read, uploaded, or hosted.`,
659
+ `Registered via 'champollion register-corpus' (exposureTier=${tierKey}). ${readNote(o)}`,
514
660
  },
515
661
  };
516
662
 
@@ -569,11 +715,13 @@ export function resolveDestination({ tier, id, corporaCardsDir, localDir }) {
569
715
  * @param {object} o.tier
570
716
  * @param {object|null} o.licenseOption
571
717
  * @param {{source?:string,target?:string}} o.pair
718
+ * @param {string|null} [o.pairError] why --pair could not be read (lib/language-pair.js), if it could not
572
719
  * @param {string} o.name
573
720
  * @param {string|null} [o.repoUrl]
574
721
  * @param {string|null} [o.builder]
575
722
  * @param {number|null} [o.size]
576
723
  * @param {string} [o.domain]
724
+ * @param {string|null} [o.role] optional: test | dev | train (see CORPUS_ROLES)
577
725
  * @param {string|null} [o.custodianGroupId] sealed tier: custodian group id
578
726
  * @param {string|null} [o.thresholdPublicKey] sealed tier: threshold pubkey (path/value)
579
727
  * @param {string|null} [o.sealInput] sealed tier: local corpus file to encrypt
@@ -583,11 +731,22 @@ export function resolveDestination({ tier, id, corporaCardsDir, localDir }) {
583
731
  export function validateRegistration(o) {
584
732
  const errors = [];
585
733
  if (!o.name || !String(o.name).trim()) errors.push('A corpus name is required (--name).');
586
- if (!o.pair || !o.pair.source) errors.push('A source language code is required (--pair "src>tgt" or --source-lang).');
587
- if (!o.pair || !o.pair.target) errors.push('A target language code is required (--pair "src>tgt" or --target-lang).');
734
+ if (o.pairError) {
735
+ // --pair was given but could not be read: say why, not "a code is required".
736
+ errors.push(o.pairError);
737
+ } else {
738
+ if (!o.pair || !o.pair.source) errors.push('A source language code is required (--pair "src>tgt" or src-tgt, or --source-lang).');
739
+ if (!o.pair || !o.pair.target) errors.push('A target language code is required (--pair "src>tgt" or src-tgt, or --target-lang).');
740
+ }
588
741
  if (!o.licenseOption) errors.push('A license is required (--license). Run with --list to see options.');
589
742
  if (!(Number(o.size) > 0)) errors.push('A positive corpus size is required (--size).');
590
743
  if (!o.domain || !String(o.domain).trim()) errors.push('A domain is required (--domain), e.g. news, conversational, educational.');
744
+ const role = o.role === true ? null : normalizeRole(o.role);
745
+ if (o.role === true) {
746
+ errors.push(`--role needs a value: ${CORPUS_ROLES.join(', ')} (or leave --role out, and the id names no role).`);
747
+ } else if (role !== null && !CORPUS_ROLES.includes(role)) {
748
+ errors.push(`Unknown --role '${o.role}'. Use ${CORPUS_ROLES.join(', ')} — or leave --role out, and the id names no role.`);
749
+ }
591
750
 
592
751
  if (o.tier.key === 'public') {
593
752
  if (!o.repoUrl) errors.push('Public registration needs a fetch-from-source pointer (--repo-url) — we never host content.');
@@ -0,0 +1,29 @@
1
+ /**
2
+ * One wording for a cost estimate, wherever a price is named beside a command
3
+ * or an answer: the redo notes of `sync`, the named-key cache note, and the
4
+ * MCP server's `translate` answer (Round 9: the MCP said "0 USD" for a model
5
+ * on this machine, where the CLI says "$0 API cost (runs on this machine)").
6
+ *
7
+ * Pure, no imports: the MCP server reads it from the package's exports.
8
+ *
9
+ * The estimate is the object a method's estimateCost() returns
10
+ * ({ estimatedCost: number|null, local?: true, … }):
11
+ * - a model served on this machine (local: true, $0) → "$0 API cost (runs on this machine)"
12
+ * - a known price → "est. ~$0.0123"
13
+ * - no published price (null), or no estimate at all → "cost unknown — no published price" / "cost unknown"
14
+ */
15
+
16
+ /** What a model on this machine costs, in the words every surface uses. */
17
+ export const LOCAL_COST_LABEL = '$0 API cost (runs on this machine)';
18
+
19
+ /**
20
+ * @param {{ estimatedCost?: number|null, local?: boolean }|null|undefined} estimate
21
+ * @returns {string}
22
+ */
23
+ export function costLabel(estimate) {
24
+ if (!estimate || typeof estimate !== 'object') return 'cost unknown';
25
+ const cost = estimate.estimatedCost;
26
+ if (estimate.local && cost === 0) return LOCAL_COST_LABEL;
27
+ if (typeof cost === 'number' && Number.isFinite(cost)) return `est. ~$${cost.toFixed(4)}`;
28
+ return 'cost unknown — no published price';
29
+ }