champollion 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +83 -36
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1689 -87
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +196 -14
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +270 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1127 -344
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +726 -78
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +965 -253
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +964 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +571 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +252 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +68 -128
  70. package/lib/methods/llm.js +80 -31
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +11 -0
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +2 -1
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +880 -17
  117. package/lib/verify.js +1296 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/method-registry.json +2 -0
  126. package/shared/metric-registry.json +96 -18
  127. package/shared/schemas/champollion-plugin.schema.json +4 -0
  128. package/shared/schemas/corpora-card.schema.json +8 -2
  129. package/shared/schemas/method-index-record.schema.json +67 -0
  130. package/shared/schemas/method-registry.schema.json +4 -0
  131. package/shared/schemas/metric-registry.schema.json +55 -1
  132. package/shared/docent/corpus.json +0 -11739
@@ -13,16 +13,19 @@
13
13
  * text is never hosted by us. Gated by cli/lib/license-gate.mjs so NC /
14
14
  * no-redistribute / unconfirmed licenses cannot enter the public lane.
15
15
  *
16
- * We NEVER read, upload, or host corpus content in ANY tier — only metadata and
17
- * (for public) a pointer to where the data is fetched from. This command writes
18
- * a metadata card; it never touches your sentences.
16
+ * We NEVER upload or host corpus content in ANY tier — only metadata and (for
17
+ * public) a pointer to where the data is fetched from. This command writes a
18
+ * metadata card. A file it is given is read on this machine only: --data to
19
+ * count its entries and compute its sha256, --seal-input (sealed tier) to
20
+ * encrypt it. Nothing it reads is copied, uploaded or sent anywhere.
19
21
  *
20
22
  * Interactive wizard when stdin is a TTY; fully scriptable via flags otherwise
21
- * (or with --yes). See `champollion register-corpus --help`.
23
+ * (or with --yes). See `champollion network register-corpus --help`.
22
24
  *
23
25
  * Exit codes: 0 = registered/saved (or cancelled); 1 = invalid request / gate block.
24
26
  */
25
27
 
28
+ import crypto from 'node:crypto';
26
29
  import fs from 'node:fs';
27
30
  import path from 'node:path';
28
31
  import readline from 'node:readline';
@@ -37,6 +40,10 @@ import {
37
40
  gatePublicRegistration,
38
41
  buildCorpusCard,
39
42
  deriveCardId,
43
+ deriveIdSlug,
44
+ normalizeRole,
45
+ CORPUS_ROLES,
46
+ CARD_ID_PATTERN,
40
47
  slugify,
41
48
  resolveDestination,
42
49
  validateRegistration,
@@ -49,11 +56,280 @@ import {
49
56
  buildAad,
50
57
  } from '../seal.mjs';
51
58
  import { DEFAULT_QUALIFIER_THRESHOLD } from '../sealed-qualifier.mjs';
59
+ import { getLanguageCard } from '../registers.js';
60
+ import { findPublicCorporaBySha } from '../public-catalogue.js';
61
+ import { parseLanguagePair, formatLanguagePair } from '../language-pair.js';
62
+ import { showCommandHelp } from '../command-help.js';
63
+
64
+ /** A word as a POSIX shell reads it back exactly (quoted only when it must be). */
65
+ function shellQuote(word) {
66
+ const w = String(word);
67
+ return /^[A-Za-z0-9_.:/@%+=,-]+$/.test(w) ? w : `'${w.replace(/'/g, `'\\''`)}'`;
68
+ }
69
+
70
+ /**
71
+ * The language flags of the `mt-eval run` hint, filled from the card's pair
72
+ * (Round 7, hospital persona: the hint printed <src>/<tgt> although the card
73
+ * it had just written holds the pair). mt-eval's --source-lang/--target-lang
74
+ * are the NAMES the model is told (from the language card, else the
75
+ * project's own "name" for the code in champollion.config.json — a
76
+ * private-use code such as qaa has no card); --source-code/--target-lang-code
77
+ * are the codes the run is recorded under (--target-lang-code is mt-eval's
78
+ * documented spelling — --target-code is its alias, Round 12).
79
+ *
80
+ * @returns {{ flags: string, unnamed: string[], cardless: string[], names: Object<string, string|null> }}
81
+ * unnamed: codes with no name anywhere (the hint says to put the language's
82
+ * name in their flag); cardless: codes with no language card (nmt-forge init
83
+ * needs --no-card for them); names: the name found for each code, or null
84
+ */
85
+ function languageFlags(pair, cwd) {
86
+ let projectNames = {};
87
+ try {
88
+ const cfg = JSON.parse(fs.readFileSync(path.join(cwd, 'champollion.config.json'), 'utf-8'));
89
+ if (cfg && cfg.languages && typeof cfg.languages === 'object' && !Array.isArray(cfg.languages)) {
90
+ for (const [code, v] of Object.entries(cfg.languages)) if (v && typeof v.name === 'string') projectNames[code] = v.name;
91
+ }
92
+ } catch { projectNames = {}; }
93
+ const unnamed = [];
94
+ const cardless = [];
95
+ const names = {};
96
+ const nameOf = (code) => {
97
+ let card = null;
98
+ try { card = getLanguageCard(code); } catch { card = null; }
99
+ if (!card) cardless.push(code);
100
+ const name = (card && typeof card.name === 'string' && card.name) || projectNames[code] || null;
101
+ if (!name) unnamed.push(code);
102
+ names[code] = name;
103
+ return name || code;
104
+ };
105
+ const flags = `--source-lang ${shellQuote(nameOf(pair.source))} --target-lang ${shellQuote(nameOf(pair.target))} `
106
+ + `--source-code ${shellQuote(pair.source)} --target-lang-code ${shellQuote(pair.target)}`;
107
+ return { flags, unnamed, cardless, names };
108
+ }
109
+
110
+ /**
111
+ * Would someone train a model against this set? Then its first score must come
112
+ * AFTER nmt-forge has registered it and the predictions are written down.
113
+ * A test set — stated with --role test, or a local-only / private set whose
114
+ * role was not stated (the tiers a community's own held-out set uses) — may be
115
+ * trained against; a dev or train set is not what forge's predictions are
116
+ * judged on.
117
+ */
118
+ function mayBeTrainedAgainst(req) {
119
+ if (req.role === 'test') return true;
120
+ if (req.role) return false;
121
+ return req.tier.key === 'local-only' || req.tier.key === 'private';
122
+ }
123
+
124
+ /**
125
+ * The nmt-forge project already set up for this pair: this directory, or one
126
+ * folder below it, that holds a forge workspace (.forge/) and a config.json
127
+ * for the same source and target. Exactly one such folder, or null — with
128
+ * none (or several) the hint starts a new one with `nmt-forge init`.
129
+ */
130
+ function findForgeProject(cwd, pair) {
131
+ const candidates = [cwd];
132
+ let entries = [];
133
+ try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { entries = []; }
134
+ for (const e of entries) {
135
+ if (e.isDirectory() && !e.name.startsWith('.') && e.name !== 'node_modules') candidates.push(path.join(cwd, e.name));
136
+ }
137
+ const matches = candidates.filter((dir) => {
138
+ if (!fs.existsSync(path.join(dir, '.forge'))) return false;
139
+ let lang;
140
+ try { lang = JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf-8'))?.language; } catch { return false; }
141
+ return !!lang && lang.source === pair.source && lang.target === pair.target;
142
+ });
143
+ return matches.length === 1 ? matches[0] : null;
144
+ }
145
+
146
+ /**
147
+ * What an existing forge project already knows about this file: the name it
148
+ * is registered under (matched by sha256 — forge hashes the file's bytes, as
149
+ * describeDataFile does), the preregistrations bound to it, and the set names
150
+ * taken by other files.
151
+ */
152
+ function forgeKnows(projectDir, sha256) {
153
+ let sets = {};
154
+ try {
155
+ sets = JSON.parse(fs.readFileSync(path.join(projectDir, '.forge', 'eval-registry.json'), 'utf-8'))?.sets || {};
156
+ } catch { sets = {}; } // no registry yet: nothing is registered
157
+ const name = sha256 ? (Object.entries(sets).find(([, e]) => e && e.sha256 === sha256) || [null])[0] : null;
158
+ const preregs = [];
159
+ const preregIds = new Set();
160
+ const dir = path.join(projectDir, '.forge', 'preregistrations');
161
+ let files = [];
162
+ try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { files = []; } // none written yet
163
+ for (const f of files) {
164
+ const id = f.replace(/\.json$/, '');
165
+ preregIds.add(id);
166
+ try {
167
+ const pr = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf-8'));
168
+ if (sha256 && pr?.eval_set?.sha256 === sha256) preregs.push(pr.id || id);
169
+ } catch { /* not a preregistration forge wrote: its id is still taken */ }
170
+ }
171
+ return { name, preregs, preregIds, taken: new Set(Object.keys(sets)) };
172
+ }
173
+
174
+ /**
175
+ * The nmt-forge steps that must come before ANY score of a test set a model
176
+ * may be trained against — the build-MT guide's "register, screen, predict"
177
+ * (Round 10, Cree-school persona: the "Next:" line went straight to
178
+ * `mt-eval run`, a scoring read, and forge then refuses the predictions).
179
+ * Paths are written from inside the forge project, where forge runs. An
180
+ * existing project is read first, so no printed step is one forge refuses:
181
+ * a file it already holds keeps its name, and `project-test` is not reused
182
+ * for a different file.
183
+ *
184
+ * @returns {{lines: string[], done: boolean}} done: the file is registered
185
+ * and has predictions bound to it — the baseline can run now
186
+ */
187
+ function forgeFirstLines({ pair, dataFile, sha256, cardId, cwd, lang }) {
188
+ const existing = findForgeProject(cwd, pair);
189
+ const projectDir = existing || path.join(cwd, `${pair.target}-model`);
190
+ const rel = path.relative(cwd, projectDir);
191
+ const where = rel || '.';
192
+ const known = existing ? forgeKnows(projectDir, sha256) : { name: null, preregs: [], preregIds: new Set(), taken: new Set() };
193
+ if (known.name && known.preregs.length > 0) {
194
+ return {
195
+ done: true,
196
+ lines: [
197
+ ` nmt-forge already holds this file (as ${known.name}, in ${where}) with predictions written`,
198
+ ` down before any score (${known.preregs.join(', ')}). \`nmt-forge status\` there names the next step.`,
199
+ ],
200
+ };
201
+ }
202
+ const setName = known.name || (known.taken.has('project-test') ? cardId : 'project-test');
203
+ // forge refuses a prereg id that exists ("the old one stands as history").
204
+ let preregId = 'all-data';
205
+ for (let n = 2; known.preregIds.has(preregId); n += 1) preregId = `all-data-${n}`;
206
+ // …and `prereg template --out` will not overwrite a predictions file.
207
+ let predictions = preregId === 'all-data' ? 'predictions.json' : `predictions-${preregId}.json`;
208
+ for (let n = 2; existing && fs.existsSync(path.join(projectDir, predictions)); n += 1) predictions = `predictions-${preregId}-${n}.json`;
209
+ const testFile = dataFile ? shellQuote(path.relative(projectDir, dataFile)) : '<your test file>';
210
+ const lines = [
211
+ ` Training a model for ${formatLanguagePair(pair)}? Do this first, before anything scores this file.`,
212
+ ' A benchmark is a scoring read, and nmt-forge refuses predictions written after one:',
213
+ ];
214
+ if (!existing) {
215
+ const noCard = lang.cardless.includes(pair.target)
216
+ ? ` --no-card --name ${lang.names[pair.target] ? shellQuote(lang.names[pair.target]) : "'<the language name>'"}`
217
+ : '';
218
+ lines.push(` nmt-forge init ${shellQuote(pair.target)} --dir ${shellQuote(rel)}${noCard} # once: the training project (any folder name)`);
219
+ }
220
+ if (rel) lines.push(` cd ${shellQuote(rel)}`);
221
+ if (known.name) {
222
+ lines.push(` # already registered with nmt-forge as ${known.name}`);
223
+ } else {
224
+ lines.push(` nmt-forge registry add ${shellQuote(setName)} ${testFile} --role test`);
225
+ }
226
+ lines.push(' nmt-forge leak-audit <training corpus> --clean-to corpus.clean.jsonl');
227
+ lines.push(` nmt-forge prereg template --out ${predictions} # then edit it: what you expect, and why`);
228
+ lines.push(` nmt-forge prereg new ${preregId} --eval-set ${shellQuote(setName)} --predictions ${predictions}`);
229
+ if (rel) lines.push(' cd ..');
230
+ lines.push(' (One prediction file per model you plan to train, each named after it — leak-audit\'s');
231
+ lines.push(' verdict may call for a second, twin-free model.)');
232
+ return { lines, done: false };
233
+ }
234
+
235
+ /** The model for a local run: the project's own when it runs `local`, else a placeholder. */
236
+ function localModelFor(cwd) {
237
+ try {
238
+ const cfg = JSON.parse(fs.readFileSync(path.join(cwd, 'champollion.config.json'), 'utf-8'));
239
+ if (cfg?.defaultMethod === 'local' && typeof cfg.model === 'string' && cfg.model) return cfg.model;
240
+ } catch { /* no project config here */ }
241
+ return null;
242
+ }
52
243
 
53
244
  // Tracked corpora-cards SSOT, resolved relative to this module.
54
245
  const DEFAULT_CARDS_DIR = fileURLToPath(new URL('../../shared/corpora-cards/', import.meta.url));
55
246
 
56
247
  const CONTAMINATION_LEVELS = ['NONE', 'LOW', 'MEDIUM', 'HIGH'];
248
+ // Written (never typed): the file could not be compared with the public
249
+ // corpora and no grade was stated — see the contamination block in run().
250
+ const CONTAMINATION_UNCHECKED = 'UNCHECKED';
251
+
252
+ // The steward sidecar the harness reads next to a corpus file
253
+ // (mt_eval_harness.corpus_loader.SIDECAR_SUFFIX — keep the two in step).
254
+ const SIDECAR_SUFFIX = '.champollion.json';
255
+
256
+ /**
257
+ * Read the local data file a registration describes: its sha256 and how many
258
+ * sentence pairs it holds (TSV/text: non-empty lines that are not '# '
259
+ * comments; JSONL: non-empty lines; JSON: entries). The text itself is read
260
+ * only to hash and count it — it is never copied, uploaded or stored.
261
+ */
262
+ function describeDataFile(file) {
263
+ const bytes = fs.readFileSync(file);
264
+ const sha256 = crypto.createHash('sha256').update(bytes).digest('hex');
265
+ const text = bytes.toString('utf-8').replace(/^\uFEFF/, '');
266
+ let rows;
267
+ if (file.endsWith('.json')) {
268
+ const data = JSON.parse(text);
269
+ rows = Array.isArray(data) ? data.length
270
+ : Array.isArray(data?.entries) ? data.entries.length : 0;
271
+ } else {
272
+ rows = text.split('\n')
273
+ .map((l) => l.replace(/\r$/, ''))
274
+ .filter((l) => l.trim() && !/^#(\s|$)/.test(l)).length;
275
+ }
276
+ return { sha256, rows };
277
+ }
278
+
279
+ /**
280
+ * The public corpus a file is a byte-identical copy of, from the corpora
281
+ * cards this CLI can see (each pins its built corpus's sha256, and the
282
+ * upstream archive's) — or null. A private, local-only or sealed card claims
283
+ * text no model has seen; a file the cards show is public cannot claim that
284
+ * (Round 8, researcher persona: a copy of the public Tatoeba eng→sme set was
285
+ * graded "Contamination: NONE", caught only later by `contest prepare`).
286
+ *
287
+ * @param {string} sha256
288
+ * @param {string} cardsDir
289
+ * @returns {{ id: string, name: string, license: string|null, risk: string|null }|null}
290
+ */
291
+ function findPublicCopy(sha256, cardsDir) {
292
+ let files;
293
+ try { files = fs.readdirSync(cardsDir).filter((f) => f.endsWith('.json')); } catch { return null; }
294
+ for (const f of files) {
295
+ let card;
296
+ try { card = JSON.parse(fs.readFileSync(path.join(cardsDir, f), 'utf-8')); } catch { continue; }
297
+ const src = card?.source || {};
298
+ const tier = card?.exposureTier || (src.repo_url || src.url ? 'public' : null);
299
+ if (tier && tier !== 'public') continue;
300
+ if (src.sha256 === sha256 || src.archive_sha256 === sha256) {
301
+ return { id: card.id || f.replace(/\.json$/, ''), name: card.name || card.id, license: card.license?.spdx || src.license || null, risk: card.contamination?.risk || null };
302
+ }
303
+ }
304
+ return null;
305
+ }
306
+
307
+ /**
308
+ * Write (or update) the sidecar next to the data file so `mt-eval run
309
+ * --corpus <file>` knows what this registration said: its id, licence and
310
+ * checksum — and, for local-only, that no outside service may see it. The
311
+ * harness lets a sidecar only TIGHTEN what it does, and this never loosens an
312
+ * existing one: a file already marked local-only stays local-only.
313
+ */
314
+ function readSidecar(file) {
315
+ const sidecar = file + SIDECAR_SUFFIX;
316
+ if (!fs.existsSync(sidecar)) return {};
317
+ try {
318
+ return JSON.parse(fs.readFileSync(sidecar, 'utf-8'));
319
+ } catch (e) {
320
+ throw new Error(`${sidecar} exists but is not valid JSON (${e.message}). `
321
+ + 'Fix or remove it — it states how this corpus may be used.');
322
+ }
323
+ }
324
+
325
+ function writeSidecar(file, fields) {
326
+ const sidecar = file + SIDECAR_SUFFIX;
327
+ const existing = readSidecar(file);
328
+ const merged = { ...existing, ...fields };
329
+ if (existing.transmission === 'local-only') merged.transmission = 'local-only';
330
+ fs.writeFileSync(sidecar, JSON.stringify(merged, null, 2) + '\n', 'utf-8');
331
+ return { path: sidecar, transmission: merged.transmission || null };
332
+ }
57
333
 
58
334
  /**
59
335
  * Resolve a path to its REAL location, following symlinks. The destination
@@ -91,21 +367,30 @@ function isInteractive() {
91
367
  return process.stdin.isTTY === true;
92
368
  }
93
369
 
94
- function ask(rl, question, defaultValue) {
370
+ function ask(rl, question, defaultValue, shownDefault = defaultValue) {
95
371
  return new Promise((resolve) => {
96
- const suffix = defaultValue ? ` (${defaultValue})` : '';
372
+ // shownDefault: what Enter means, shown in the prompt — the caller may
373
+ // want to tell Enter apart from a typed value (defaultValue '').
374
+ const suffix = shownDefault ? ` (${shownDefault})` : '';
97
375
  rl.question(` ${question}${suffix}: `, (answer) => {
98
376
  resolve(answer.trim() || defaultValue || '');
99
377
  });
100
378
  });
101
379
  }
102
380
 
103
- /** Parse "src>tgt" / "src-tgt" / "src,tgt" into {source, target}, else null. */
104
- function parsePair(value) {
105
- if (!value) return null;
106
- const m = String(value).trim().split(/[>\-,\s]+/).filter(Boolean);
107
- if (m.length >= 2) return { source: m[0].toLowerCase(), target: m[1].toLowerCase() };
108
- return null;
381
+ /**
382
+ * Read --pair (any spelling lib/language-pair.js accepts: eng>crk, eng-crk,
383
+ * eng:crk, …). Codes are lower-cased: they name the card id, which is
384
+ * lower-case. The old reader split on every - and > and kept the first two
385
+ * pieces, so "eng>pt-BR" was registered as eng→pt without a word.
386
+ *
387
+ * @returns {{pair: {source: string, target: string}|null, pairError: string|null}}
388
+ */
389
+ function readPair(value) {
390
+ if (value === undefined || value === null || value === '') return { pair: null, pairError: null };
391
+ const p = parseLanguagePair(value, { label: '--pair' });
392
+ if (!p.ok) return { pair: null, pairError: p.error };
393
+ return { pair: { source: p.source.toLowerCase(), target: p.target.toLowerCase() }, pairError: null };
109
394
  }
110
395
 
111
396
  // ── Catalog printing (plain language for humans, machine-readable for agents) ──
@@ -119,24 +404,29 @@ function printCatalogs(asJson) {
119
404
  }, null, 2));
120
405
  return;
121
406
  }
407
+ // The id beside each label is what --license / --tier take (Round 8,
408
+ // school + hospital personas: the list showed numbered labels only).
122
409
  console.log('');
123
- console.log(' LICENSES — pick the terms others may use your corpus under:');
410
+ console.log(' LICENSES — pick the terms others may use your corpus under (pass the id to --license):');
124
411
  console.log('');
125
412
  LICENSE_OPTIONS.forEach((o, i) => {
126
- console.log(` ${i + 1}. ${o.label}`);
413
+ console.log(` ${i + 1}. ${o.key.padEnd(24)} ${o.label}`);
127
414
  console.log(` ${o.explanation}`);
128
415
  console.log(` commercial: ${o.commercial} · redistribution: ${o.redistribution} · public-lane: ${o.publicEligible ? 'eligible' : 'blocked'}`);
129
416
  });
130
417
  console.log('');
131
- console.log(' EXPOSURE TIERS — how far your corpus travels (default: most private):');
418
+ console.log(' --license also takes the list number, or an SPDX id (e.g. --license CC-BY-4.0).');
419
+ console.log('');
420
+ console.log(' EXPOSURE TIERS — how far your corpus travels (default: most private; pass the id to --tier):');
132
421
  console.log('');
133
422
  EXPOSURE_TIERS.forEach((t, i) => {
134
423
  const def = t.key === DEFAULT_TIER ? ' ★ default' : '';
135
- console.log(` ${i + 1}. ${t.label}${def}`);
424
+ console.log(` ${i + 1}. ${t.key.padEnd(11)} ${t.label}${def}`);
136
425
  console.log(` ${t.explanation}`);
137
426
  });
138
427
  console.log('');
139
- console.log(' Champollion never reads, uploads, or hosts your corpus text — in ANY tier.');
428
+ console.log(' Champollion never uploads or hosts your corpus text — in ANY tier. With --data, the');
429
+ console.log(' file is read on this machine only to count and checksum it; none of it leaves.');
140
430
  console.log('');
141
431
  }
142
432
 
@@ -148,18 +438,32 @@ async function runInteractive(rl, args) {
148
438
  console.log(' ════════════════════════════════════════════════');
149
439
  console.log('');
150
440
  console.log(' You choose the license and how far this corpus travels.');
151
- console.log(' We never read, upload, or host your text — only metadata,');
152
- console.log(' and (for public sets) a pointer to where the data is fetched from.');
441
+ console.log(' We never upload or host your text — only metadata, and (for public');
442
+ console.log(' sets) a pointer to where the data is fetched from. A file you name');
443
+ console.log(' is read on this machine only: --data to count and checksum it, and');
444
+ console.log(' a sealed set\'s file to encrypt it.');
153
445
 
154
446
  // Step 1 — basics
155
447
  console.log('');
156
448
  console.log(' Step 1/4 — Basics');
157
449
  console.log(' ────────────────────────────────────────────────');
158
450
  const name = await ask(rl, 'Corpus name', args.name || '');
159
- const pairStr = await ask(rl, 'Language pair (e.g. eng-crk, or "eng>crk")', args.pair || '');
160
- const pair = parsePair(pairStr);
451
+ // Asked again until it reads as a pair (or is left blank — validation then
452
+ // says a pair is required): an unreadable pair is said here, not at the end.
453
+ let pair = null;
454
+ let pairError = null;
455
+ for (let pairIn = args.pair || ''; ;) {
456
+ const pairStr = await ask(rl, 'Language pair, source>target (e.g. eng>crk — eng-crk works too)', pairIn);
457
+ ({ pair, pairError } = readPair(pairStr));
458
+ if (!pairError) break;
459
+ console.log(` ⚠ ${pairError}`);
460
+ pairIn = '';
461
+ }
161
462
  const publisher = await ask(rl, 'Publisher / your name or org', args.publisher || '');
162
463
  const description = await ask(rl, 'One-line description', args.description || `${name} evaluation corpus`);
464
+ const role = normalizeRole(await ask(rl,
465
+ `What is it for — ${CORPUS_ROLES.join(', ')}? (blank: don't say; the id then names no role)`,
466
+ normalizeRole(args.role) || ''));
163
467
 
164
468
  // Step 2 — license
165
469
  console.log('');
@@ -253,14 +557,17 @@ async function runInteractive(rl, args) {
253
557
  console.log('');
254
558
  console.log(' Contamination risk — is this text in known LLM training sets?');
255
559
  console.log(' NONE = private/unpublished · LOW = niche/recent · MEDIUM = public · HIGH = in major training sets');
256
- const riskIn = await ask(rl, 'Contamination risk', defaultRisk);
257
- const contaminationRisk = CONTAMINATION_LEVELS.includes(riskIn.toUpperCase()) ? riskIn.toUpperCase() : defaultRisk;
560
+ // Typed, or the default accepted? A typed grade is a statement; the default
561
+ // is not, and gives way to UNCHECKED when the file cannot be compared.
562
+ const riskTyped = (await ask(rl, 'Contamination risk', '', defaultRisk)).toUpperCase();
563
+ const contaminationStated = CONTAMINATION_LEVELS.includes(riskTyped);
564
+ const contaminationRisk = contaminationStated ? riskTyped : defaultRisk;
258
565
 
259
566
  return {
260
- name, pair, publisher, description,
567
+ name, pair, publisher, description, role,
261
568
  licenseOption, tier,
262
569
  repoUrl, builder, sourceUrl,
263
- size: Number(sizeStr), domain, contaminationRisk,
570
+ size: Number(sizeStr), domain, contaminationRisk, contaminationStated,
264
571
  custodianGroupId, thresholdPubkey, sealInput, sealOut,
265
572
  qualifierId, qualifierThreshold, keyScheme,
266
573
  };
@@ -269,8 +576,10 @@ async function runInteractive(rl, args) {
269
576
  // ── Non-interactive (flags) ─────────────────────────────────────────────────
270
577
 
271
578
  function fromFlags(args) {
272
- const pair = parsePair(args.pair) ||
273
- (args['source-lang'] && args['target-lang']
579
+ const read = args.pair !== undefined ? readPair(args.pair) : { pair: null, pairError: null };
580
+ const pairError = read.pairError;
581
+ const pair = read.pair ||
582
+ (!pairError && args['source-lang'] && args['target-lang']
274
583
  ? { source: String(args['source-lang']).toLowerCase(), target: String(args['target-lang']).toLowerCase() }
275
584
  : null);
276
585
  const licenseOption = resolveLicense(args.license);
@@ -279,8 +588,10 @@ function fromFlags(args) {
279
588
  return {
280
589
  name: args.name || '',
281
590
  pair,
282
- publisher: args.publisher || (args.name ? '' : ''),
591
+ pairError,
592
+ publisher: args.publisher || '',
283
593
  description: args.description || (args.name ? `${args.name} evaluation corpus` : ''),
594
+ role: args.role === true ? true : normalizeRole(args.role), // bare --role: validation says it needs a value
284
595
  licenseOption,
285
596
  tier,
286
597
  repoUrl: args['repo-url'] || null,
@@ -289,6 +600,9 @@ function fromFlags(args) {
289
600
  size: args.size != null ? Number(args.size) : NaN,
290
601
  domain: args.domain || '',
291
602
  contaminationRisk: riskIn && CONTAMINATION_LEVELS.includes(riskIn) ? riskIn : null,
603
+ // A grade you passed is your statement; it stands even when the file
604
+ // could not be compared with the public corpora.
605
+ contaminationStated: !!(riskIn && CONTAMINATION_LEVELS.includes(riskIn)),
292
606
  // sealed tier
293
607
  custodianGroupId: args['custodian-group'] || null,
294
608
  thresholdPubkey: args['threshold-pubkey'] || null,
@@ -421,16 +735,111 @@ async function run(args, cwd) {
421
735
  req = fromFlags(args);
422
736
  }
423
737
 
738
+ // --data: the local file this registration describes. Hashed and counted
739
+ // here (never copied or uploaded) so the card carries its checksum and the
740
+ // harness can find what was registered, through the sidecar written below.
741
+ let dataFile = null;
742
+ let dataInfo = null;
743
+ if (args.data) {
744
+ dataFile = path.resolve(cwd, String(args.data));
745
+ if (!fs.existsSync(dataFile) || !fs.statSync(dataFile).isFile()) {
746
+ console.error(`[ERR] --data: no such file: ${dataFile}`);
747
+ return 1;
748
+ }
749
+ if (req.tier && (req.tier.key === 'public' || req.tier.key === 'sealed')) {
750
+ console.error(`[ERR] --data is for local-only and private corpora. A ${req.tier.key} corpus is `
751
+ + (req.tier.key === 'public' ? 'fetched from its source (--repo-url / --builder).'
752
+ : 'encrypted from --seal-input.'));
753
+ return 1;
754
+ }
755
+ try {
756
+ dataInfo = describeDataFile(dataFile);
757
+ } catch (e) {
758
+ console.error(`[ERR] --data: could not read ${dataFile}: ${e.message}`);
759
+ return 1;
760
+ }
761
+ if (!Number.isFinite(req.size) || req.size <= 0) req.size = dataInfo.rows;
762
+ }
763
+
764
+ // A private / local-only / sealed registration of text that is public: never
765
+ // graded "Contamination: NONE". Compared with the corpora cards this CLI can
766
+ // see (a repo checkout), else — an npm install ships none — with the public
767
+ // corpus catalogue (lib/public-catalogue.js; only the public ids and
768
+ // checksums are downloaded, the file's own sha256 stays here). When neither
769
+ // can be read, the comparison was not made, and the grade says so
770
+ // (UNCHECKED) unless you state one with --contamination (Round 9,
771
+ // researcher persona: an npm install graded a byte-identical copy of a
772
+ // public benchmark NONE because it had nothing to compare with).
773
+ {
774
+ const cardsDir = args['cards-dir'] ? path.resolve(cwd, args['cards-dir']) : DEFAULT_CARDS_DIR;
775
+ const checked = [];
776
+ if (dataInfo) checked.push({ flag: '--data', file: dataFile, sha256: dataInfo.sha256 });
777
+ if (req.tier && req.tier.key === 'sealed' && req.sealInput) {
778
+ try {
779
+ const plain = path.resolve(cwd, String(req.sealInput));
780
+ checked.push({ flag: '--seal-input', file: plain, sha256: crypto.createHash('sha256').update(fs.readFileSync(plain)).digest('hex') });
781
+ } catch { /* the sealing step names an unreadable file */ }
782
+ }
783
+ if (checked.length > 0 && req.tier && req.tier.key !== 'public') {
784
+ let lookup;
785
+ if (fs.existsSync(cardsDir)) {
786
+ lookup = { ok: true, where: 'the corpora cards', find: (sha) => findPublicCopy(sha, cardsDir) };
787
+ } else {
788
+ const cat = await findPublicCorporaBySha(checked.map(c => c.sha256));
789
+ lookup = cat.ok
790
+ ? { ok: true, where: `the public corpus catalogue (${cat.compared} checksums)`, find: (sha) => (cat.hits.has(sha) ? { ...cat.hits.get(sha), risk: null } : null) }
791
+ : { ok: false, why: `this install ships no corpora cards, and ${cat.why}` };
792
+ }
793
+ if (!lookup.ok) {
794
+ // Not compared. NONE claims text no model has seen — it is never the
795
+ // default for a comparison that was not made.
796
+ const shown = checked.map(c => path.relative(cwd, c.file) || c.file).join(', ');
797
+ req.contaminationNote = `not compared with the public corpora — ${lookup.why}`;
798
+ if (req.contaminationStated) {
799
+ req.contaminationReasoning = `Stated at registration (${req.contaminationRisk}); ${shown} was not compared with the public corpora (${lookup.why}).`;
800
+ } else {
801
+ req.contaminationRisk = CONTAMINATION_UNCHECKED;
802
+ req.contaminationReasoning = `Not graded: ${shown} could not be compared with the public corpora at registration (${lookup.why}). `
803
+ + 'Re-register online to compare it, or state a grade with --contamination none|low|medium|high.';
804
+ }
805
+ } else {
806
+ req.contaminationCompared = lookup.where;
807
+ for (const c of checked) {
808
+ const hit = lookup.find(c.sha256);
809
+ if (!hit) continue;
810
+ const shown = path.relative(cwd, c.file) || c.file;
811
+ const why = `${shown} is a byte-identical copy of the public corpus ${hit.id} (${hit.name}${hit.license ? `, ${hit.license}` : ''}; same sha256 ${c.sha256.slice(0, 12)}…, found in ${lookup.where}).`;
812
+ if (req.tier.key === 'sealed') {
813
+ console.error(`[ERR] ${why}`);
814
+ console.error(' A sealed test set is text no model has seen; this text is public, so sealing it tests nothing.');
815
+ console.error(` Register it as public instead (--tier public, fetched from its source — ${hit.id} already is), or seal sentences that are genuinely private.`);
816
+ return 1;
817
+ }
818
+ if (!req.contaminationRisk || req.contaminationRisk === 'NONE') {
819
+ console.error(`[ERR] ${why}`);
820
+ console.error(` A ${req.tier.key} card would say "Contamination: NONE" (text no model has seen); this text is public${hit.risk ? ` — the public card rates it ${hit.risk}` : ''}.`);
821
+ console.error(` Register it as public (--tier public — ${hit.id} already describes it), use sentences that are genuinely private,`);
822
+ console.error(` or keep it ${req.tier.key} and state its exposure: --contamination ${hit.risk && hit.risk !== 'NONE' ? hit.risk : 'MEDIUM'}.`);
823
+ return 1;
824
+ }
825
+ req.contaminationReasoning = `${why} Registered ${req.tier.key} with the stated risk ${req.contaminationRisk}; the text itself is public.`;
826
+ }
827
+ }
828
+ }
829
+ }
830
+
424
831
  // Validate (collects every problem, fails loud).
425
832
  const v = validateRegistration({
426
833
  tier: req.tier,
427
834
  licenseOption: req.licenseOption,
428
835
  pair: req.pair || {},
836
+ pairError: req.pairError || null,
429
837
  name: req.name,
430
838
  repoUrl: req.repoUrl,
431
839
  builder: req.builder,
432
840
  size: req.size,
433
841
  domain: req.domain,
842
+ role: req.role,
434
843
  custodianGroupId: req.custodianGroupId,
435
844
  thresholdPublicKey: req.thresholdPubkey,
436
845
  sealInput: req.sealInput,
@@ -440,17 +849,58 @@ async function run(args, cwd) {
440
849
  console.error('[ERR] Cannot register this corpus:');
441
850
  for (const e of v.errors) console.error(` • ${e}`);
442
851
  console.error('');
443
- console.error(' Run "champollion register-corpus --list" to see licenses + tiers,');
444
- console.error(' or "champollion register-corpus --help" for all flags.');
852
+ console.error(' Run "champollion network register-corpus --list" to see licenses + tiers,');
853
+ console.error(' or "champollion network register-corpus --help" for all flags.');
445
854
  return 1;
446
855
  }
447
856
 
448
857
  // Build the (content-free) card.
858
+ // The id: a full --id (eval-…/ref-…) is used exactly as given; otherwise it
859
+ // is derived from --name (publisher only as a fallback), with a role segment
860
+ // only when --role states one. See deriveCardId.
449
861
  const version = args.version || '1';
450
- const slug = slugify(args.id || req.publisher || req.name || 'custom');
451
- const id = args.id && /^(ref|eval)-/.test(args.id)
452
- ? args.id
453
- : deriveCardId({ source: req.pair.source, target: req.pair.target, slug, version });
862
+ let id;
863
+ if (args.id && /^(ref|eval)-/.test(args.id)) {
864
+ id = String(args.id);
865
+ if (!CARD_ID_PATTERN.test(id)) {
866
+ console.error(`[ERR] --id '${id}' is not a valid card id: after eval- or ref-, use only`);
867
+ console.error(' lower-case letters, digits and hyphens (it is also the card filename).');
868
+ return 1;
869
+ }
870
+ if (req.role && !new RegExp(`-${req.role}(-|$)`).test(id)) {
871
+ console.error(`[ERR] --role ${req.role} shapes a generated id, but --id is used exactly as given`);
872
+ console.error(` and '${id}' does not name that role. Put the role in --id, or drop --role.`);
873
+ return 1;
874
+ }
875
+ } else {
876
+ const slug = (args.id && slugify(args.id)) || deriveIdSlug({ name: req.name, publisher: req.publisher });
877
+ id = deriveCardId({ source: req.pair.source, target: req.pair.target, slug, role: req.role, version });
878
+ }
879
+
880
+ // A data file that is already registered keeps its id. Its sidecar records
881
+ // the id it was registered under; deriving a fresh one on a re-run would
882
+ // silently relink the file to a second card. Ids are never re-derived.
883
+ // A file already marked local-only stays local-only (writeSidecar never
884
+ // loosens a mark), and its card says so too.
885
+ let priorTransmission = null;
886
+ if (dataFile) {
887
+ let priorSidecar;
888
+ try {
889
+ priorSidecar = readSidecar(dataFile);
890
+ } catch (e) {
891
+ console.error(`[ERR] ${e.message}`);
892
+ return 1;
893
+ }
894
+ if (priorSidecar.transmission === 'local-only') priorTransmission = 'local-only';
895
+ if (priorSidecar.id && priorSidecar.id !== id && !args.id) {
896
+ const shown = path.relative(cwd, dataFile) || dataFile;
897
+ console.error(`[ERR] ${shown} is already registered as ${priorSidecar.id}`);
898
+ console.error(` (recorded in ${path.basename(dataFile)}${SIDECAR_SUFFIX}). A registered id is never re-derived.`);
899
+ console.error(` To keep it, add: --id ${priorSidecar.id}`);
900
+ console.error(` To register the file under a new id: --id ${id}`);
901
+ return 1;
902
+ }
903
+ }
454
904
 
455
905
  const addedAt = new Date().toISOString().slice(0, 10);
456
906
 
@@ -487,17 +937,28 @@ async function run(args, cwd) {
487
937
  licenseOption: req.licenseOption,
488
938
  tier: req.tier,
489
939
  contaminationRisk: req.contaminationRisk,
940
+ ...(req.contaminationReasoning && { contaminationReasoning: req.contaminationReasoning }),
941
+ role: req.role || null,
942
+ dataRead: !!dataInfo,
490
943
  size: req.size,
491
944
  domain: req.domain,
492
945
  doNotTrain: args['do-not-train'] !== false,
946
+ transmission: priorTransmission,
493
947
  addedAt,
494
948
  sealed: sealedBlock,
495
949
  });
496
950
 
497
951
  // Resolve destination — local-only NEVER lands in the tracked SSOT.
498
952
  const corporaCardsDir = args['cards-dir'] ? path.resolve(cwd, args['cards-dir']) : DEFAULT_CARDS_DIR;
499
- const localDir = args.out ? path.resolve(cwd, args.out) : cwd;
500
- const dest = resolveDestination({ tier: req.tier, id, corporaCardsDir, localDir });
953
+ const localDir = args.out ? path.resolve(cwd, args.out) : (dataFile ? path.dirname(dataFile) : cwd);
954
+ // An npm install ships no corpora-cards directory (it is the monorepo's
955
+ // tracked SSOT). Writing a "registered" card into node_modules put it where
956
+ // nothing reads it and the next reinstall deletes it. From an install, the
957
+ // card stays with you, and listing it is a review-gated submission.
958
+ const installMode = !args['cards-dir'] && !fs.existsSync(DEFAULT_CARDS_DIR);
959
+ const dest = installMode && req.tier.key !== 'local-only'
960
+ ? { registered: false, tracked: false, dir: localDir, filename: `${id}.json` }
961
+ : resolveDestination({ tier: req.tier, id, corporaCardsDir, localDir });
501
962
 
502
963
  // Guard: a local-only card must never land in the tracked corpora-cards SSOT.
503
964
  // path.resolve does NOT follow symlinks and only the exact dir was checked —
@@ -522,11 +983,32 @@ async function run(args, cwd) {
522
983
  fs.mkdirSync(dest.dir, { recursive: true });
523
984
  fs.writeFileSync(filePath, JSON.stringify(card, null, 2) + '\n', 'utf-8');
524
985
 
986
+ let sidecar = null;
987
+ if (dataFile) {
988
+ try {
989
+ sidecar = writeSidecar(dataFile, {
990
+ id,
991
+ name: req.name,
992
+ license: card.license.spdx,
993
+ tier: req.tier.key,
994
+ sha256: dataInfo.sha256,
995
+ rows: dataInfo.rows,
996
+ card: path.relative(path.dirname(dataFile), filePath) || path.basename(filePath),
997
+ registeredAt: addedAt,
998
+ ...(req.tier.key === 'local-only' ? { transmission: 'local-only' } : {}),
999
+ });
1000
+ } catch (e) {
1001
+ console.error(`[ERR] ${e.message}`);
1002
+ return 1;
1003
+ }
1004
+ }
1005
+
525
1006
  // ── Report ──
526
1007
  if (args.json) {
527
1008
  console.log(JSON.stringify({
528
1009
  ok: true,
529
1010
  id,
1011
+ role: req.role || null,
530
1012
  exposureTier: req.tier.key,
531
1013
  registered: dest.registered,
532
1014
  tracked: dest.tracked,
@@ -542,7 +1024,24 @@ async function run(args, cwd) {
542
1024
  qualifierThreshold: sealedBlock.qualifierThreshold,
543
1025
  } : {}),
544
1026
  license: card.license,
1027
+ contamination: {
1028
+ risk: card.contamination.risk,
1029
+ // Where the file was compared with the public corpora (null: it could not be).
1030
+ comparedWith: req.contaminationCompared || null,
1031
+ ...(req.contaminationNote && { note: req.contaminationNote }),
1032
+ },
545
1033
  path: filePath,
1034
+ ...(sidecar ? {
1035
+ data: dataFile,
1036
+ dataSha256: dataInfo.sha256,
1037
+ rows: dataInfo.rows,
1038
+ sidecar: sidecar.path,
1039
+ transmission: sidecar.transmission,
1040
+ } : {}),
1041
+ ...(installMode && req.tier.key !== 'local-only' ? {
1042
+ listed: false,
1043
+ toList: 'champollion network submit --type dataset',
1044
+ } : {}),
546
1045
  }, null, 2));
547
1046
  return 0;
548
1047
  }
@@ -551,12 +1050,50 @@ async function run(args, cwd) {
551
1050
  output.ok(`${req.tier.key === 'local-only' ? 'Saved' : 'Registered'} corpus card: ${id}`);
552
1051
  console.log('');
553
1052
  console.log(` Exposure: ${req.tier.label}`);
554
- console.log(` License: ${card.license.spdx} (commercial: ${card.license.commercial}, redistribution: ${card.license.redistribution})`);
555
- console.log(` Pair: ${req.pair.source}→${req.pair.target}`);
556
- console.log(` Size/domain: ${req.size} ${card.dev.sizeUnit} · ${card.dev.domain}`);
557
- console.log(` Contamination: ${card.contamination.risk}`);
1053
+ console.log(` Role: ${req.role || 'not stated (add --role test|dev|train to say what it is for)'}`);
1054
+ // Each term once, as the card states it (Round 14, researcher persona:
1055
+ // the card and this summary disagreed). The licence's terms; training
1056
+ // (doNotTrain); the transmission mark — a separate field, not a licence term.
1057
+ const yesNo = (b) => (b ? 'yes' : 'no');
1058
+ const custom = card.license.notes ? ' (custom/unconfirmed — verify its terms)' : '';
1059
+ const redistributed = { permitted: 'yes', 'same-terms': 'yes, under the same terms', prohibited: 'no' }[card.usageRestrictions.redistribution];
1060
+ console.log(` License: ${card.license.spdx}${custom} — commercial use ${yesNo(card.license.commercial)}, redistribution ${redistributed}`);
1061
+ console.log(` Training: ${card.doNotTrain
1062
+ ? `not permitted — doNotTrain: true (${card.usageRestrictions.training === 'prohibited-by-license' ? 'the licence refuses training' : 'set at registration: a test set is not trained on'})`
1063
+ : 'permitted — doNotTrain: false'}`);
1064
+ if (card.transmission === 'local-only') {
1065
+ console.log(' Transmission: local-only — only a model on this machine may see it (a mark, not a licence term)');
1066
+ }
1067
+ console.log(` Pair: ${formatLanguagePair(req.pair)}`);
1068
+ const splitBlock = card.test || card.dev;
1069
+ console.log(` Size/domain: ${req.size} ${splitBlock.sizeUnit} · ${splitBlock.domain}${card.test ? ' (test split)' : ''}`);
1070
+ let contaminationAside = '';
1071
+ if (card.contamination.risk === CONTAMINATION_UNCHECKED) {
1072
+ contaminationAside = ` (${req.contaminationNote}. Not graded NONE: that would claim the text is unpublished. `
1073
+ + 'Re-run online to compare it, or state a grade: --contamination none|low|medium|high)';
1074
+ } else if (req.contaminationNote) {
1075
+ contaminationAside = ` (${req.contaminationNote} — the grade is your statement)`;
1076
+ } else if (req.contaminationCompared && !req.contaminationReasoning) {
1077
+ contaminationAside = ` (compared with ${req.contaminationCompared}: no public copy of this file — the grade is your statement)`;
1078
+ }
1079
+ console.log(` Contamination: ${card.contamination.risk}${contaminationAside}`);
558
1080
  console.log(` Written to: ${filePath}`);
1081
+ if (sidecar) {
1082
+ console.log(` Data file: ${dataFile} (${dataInfo.rows} rows, sha256 ${dataInfo.sha256.slice(0, 12)}…)`);
1083
+ console.log(` Sidecar: ${sidecar.path}`
1084
+ + (sidecar.transmission === 'local-only' ? ' (carries the local-only mark the tools read)' : ''));
1085
+ }
559
1086
  console.log('');
1087
+ const runTarget = dataFile ? shellQuote(path.relative(cwd, dataFile) || dataFile) : '<your file>';
1088
+ const lang = languageFlags(req.pair, cwd);
1089
+ const unnamedNote = lang.unnamed.length > 0
1090
+ ? ` (${lang.unnamed.join(', ')} has no language card: replace the code after --${lang.unnamed.includes(req.pair.target) ? 'target' : 'source'}-lang with the language's name — it is what the model is told — or give it a "name" in champollion.config.json)`
1091
+ : null;
1092
+ // A set a model may be trained against: the nmt-forge steps come before
1093
+ // the baseline, which is a scoring read (see forgeFirstLines).
1094
+ const forgeFirst = mayBeTrainedAgainst(req)
1095
+ ? forgeFirstLines({ pair: req.pair, dataFile, sha256: dataInfo ? dataInfo.sha256 : null, cardId: id, cwd, lang })
1096
+ : null;
560
1097
 
561
1098
  if (req.tier.key === 'sealed') {
562
1099
  console.log(` Sealed artifact: ${sealedArtifactPath}`);
@@ -574,16 +1111,56 @@ async function run(args, cwd) {
574
1111
  console.log(' A method must first clear the paired PUBLIC qualifier before any sealed run');
575
1112
  console.log(' can even be proposed — and that run still requires custodian approval.');
576
1113
  } else if (req.tier.key === 'local-only') {
577
- console.log(' This card stays on your machine. Nothing was registered or uploaded,');
578
- console.log(' and your corpus text was never read. Delete the file to remove it.');
1114
+ console.log(' This card stays on your machine. Nothing was registered or uploaded.');
1115
+ if (dataFile) {
1116
+ console.log(' Your text was read only to count and checksum it. Only a model on this');
1117
+ console.log(' machine can be tested against it; every remote provider is refused.');
1118
+ console.log('');
1119
+ if (forgeFirst && !forgeFirst.done) {
1120
+ for (const line of forgeFirst.lines) console.log(line);
1121
+ console.log('');
1122
+ console.log(' Then measure a local model on it (the baseline) — or now, if no model will');
1123
+ console.log(' be trained against this set:');
1124
+ } else {
1125
+ if (forgeFirst) { for (const line of forgeFirst.lines) console.log(line); console.log(''); }
1126
+ console.log(' Next: measure a local model on it:');
1127
+ }
1128
+ const model = localModelFor(cwd);
1129
+ console.log(` mt-eval run --corpus ${runTarget} --provider local --model ${model ? shellQuote(model) : '<model>'} ${lang.flags}`);
1130
+ if (!model) console.log(' (<model>: the name your local server serves, e.g. llama3.1)');
1131
+ if (unnamedNote) console.log(unnamedNote);
1132
+ } else {
1133
+ console.log(' Your corpus text was never read. To protect the file itself, re-run with');
1134
+ console.log(' --data <file>: that marks it local-only for mt-eval.');
1135
+ }
579
1136
  } else if (req.tier.key === 'private') {
580
- console.log(' Metadata only is registered — your corpus TEXT was never read, uploaded,');
581
- console.log(' or hosted, and never will be. The set is catalogued but quarantined from');
582
- console.log(' the public queue/leaderboard (custodian-controlled, WMT-style held-out).');
1137
+ console.log(' Metadata only — your corpus TEXT was never uploaded or hosted, and never');
1138
+ console.log(' will be. A private set is quarantined from the public queue/leaderboard.');
1139
+ if (dataFile) console.log(' Your text was read on this machine only to count and checksum it.');
583
1140
  console.log('');
584
- console.log(' Next: rebuild the registry, then evaluate privately:');
585
- for (const line of REGISTRY_REBUILD_HINT) console.log(line);
586
- console.log(` mt-eval run --corpus ${id} --private # publish scores without exposing text`);
1141
+ if (installMode) {
1142
+ console.log(' The card is kept here with you. To have it listed in the index (metadata');
1143
+ console.log(' only, review-gated), submit it: champollion network submit --type dataset');
1144
+ } else {
1145
+ console.log(' Rebuild the registry so the card is catalogued:');
1146
+ for (const line of REGISTRY_REBUILD_HINT) console.log(line);
1147
+ }
1148
+ console.log('');
1149
+ if (forgeFirst && !forgeFirst.done) {
1150
+ for (const line of forgeFirst.lines) console.log(line);
1151
+ console.log('');
1152
+ console.log(' Then evaluate on it (the baseline) — or now, if no model will be trained');
1153
+ console.log(' against this set — and publish the scores without the text if you choose:');
1154
+ } else {
1155
+ if (forgeFirst) { for (const line of forgeFirst.lines) console.log(line); console.log(''); }
1156
+ console.log(' Evaluate on it, then publish the scores without the text if you choose:');
1157
+ }
1158
+ console.log(` mt-eval run --corpus ${runTarget} --provider <provider> --model <model> ${lang.flags} -o results`);
1159
+ if (unnamedNote) console.log(unnamedNote);
1160
+ console.log(' mt-eval publish results/<run-id>_report.json --scores-only --dry-run');
1161
+ } else if (installMode) {
1162
+ console.log(' The card is kept here with you. A public corpus is listed through review:');
1163
+ console.log(' champollion network submit --type dataset');
587
1164
  } else {
588
1165
  console.log(' A metadata card + fetch-from-source pointer is registered — your corpus');
589
1166
  console.log(' TEXT is never hosted by Champollion; it is fetched from source on demand.');
@@ -595,84 +1172,14 @@ async function run(args, cwd) {
595
1172
  return 0;
596
1173
  }
597
1174
 
1175
+ /**
1176
+ * `--help` for this command. The text lives in lib/command-help.js — the one
1177
+ * `champollion network register-corpus --help` prints. This module kept a
1178
+ * second copy, and the two drifted: Round 9 corrected what the file reads in
1179
+ * one, while the other still said "never reads" (Round 10, school persona).
1180
+ */
598
1181
  function showHelp() {
599
- console.log(`
600
- champollion register-corpus — Register a new evaluation corpus
601
-
602
- USAGE
603
- champollion register-corpus [options]
604
-
605
- DESCRIPTION
606
- Register a corpus with YOU in control of its license and how far it
607
- travels. Three exposure tiers, defaulting to the most private. We never
608
- read, upload, or host your corpus text in ANY tier — only metadata (and,
609
- for public sets, a pointer to where the data is fetched from).
610
-
611
- Interactive when run in a terminal; fully scriptable with flags (or --yes).
612
-
613
- EXPOSURE TIERS (--tier, default: local-only)
614
- local-only Never registered, never uploaded; card + text stay on your machine.
615
- private Register METADATA ONLY — a WMT-style sovereign held-out set;
616
- text never uploaded or hosted; you keep custody.
617
- public Publish a metadata card + fetch-from-source pointer; text never
618
- hosted by us. Requires a redistribution-cleared license (gated).
619
- sealed Encrypt the corpus ON YOUR DEVICE to the custodian group's
620
- key; the ciphertext stays with you and we receive only a
621
- content-free card. Catalogued + quarantined; paired with a
622
- public qualifier a method must clear before any sealed run.
623
-
624
- LICENSES (--license)
625
- cc-by-4.0, cc-by-sa-4.0, cc0-1.0, cc-by-nc-4.0, proprietary, other
626
- (or pass any SPDX id; it is classified through the license gate)
627
-
628
- OPTIONS
629
- --tier <tier> Exposure tier (default: local-only). Alias: --exposure
630
- --license <id> License key, SPDX id, or list number (see --list)
631
- --name <text> Corpus name (required)
632
- --pair <pair> Language pair: "eng>crk" (quote it — the shell eats a
633
- bare >) or the dash form eng-crk (or --source-lang/--target-lang)
634
- --publisher <text> Publisher / your name or org
635
- --description <text> One-line description
636
- --size <n> Number of sentence pairs (required)
637
- --domain <text> Domain: news, conversational, educational, … (required)
638
- --contamination <lvl> NONE | LOW | MEDIUM | HIGH (default: NONE private / LOW public)
639
- --repo-url <url> Public tier: fetch-from-source archive/repo URL
640
- --source-url <url> Public tier: canonical project/dataset URL
641
- --builder <id> Public tier: builder adapter id (rebuilds from source)
642
- --seal-input <path> Sealed tier: local corpus file to encrypt on-device
643
- --threshold-pubkey <k> Sealed tier: custodian group threshold public key
644
- (file path, or inline PEM / base64 DER SPKI, X25519)
645
- --custodian-group <id> Sealed tier: custodian group id that holds the key
646
- --seal-out <path> Sealed tier: where to write the ciphertext artifact
647
- --qualifier-id <id> Sealed tier: paired public qualifier card id (vYYYY)
648
- --qualifier-threshold <n> Sealed tier: score a method must clear on the qualifier
649
- --key-scheme <s> Sealed tier: custody scheme label (default: TSS-3-of-5)
650
- --id <id> Override the generated card id (eval-…)
651
- --out <dir> local-only: where to write the card (default: cwd)
652
- --list Print the license + tier catalog (add --json for JSON)
653
- --json Machine-readable output (for agents)
654
- --yes Non-interactive; take values from flags
655
-
656
- EXAMPLES
657
- champollion register-corpus # interactive wizard
658
- champollion register-corpus --list # see licenses + tiers
659
- # Keep a corpus entirely local (nothing leaves your machine):
660
- champollion register-corpus --yes --name "My set" --pair "eng>crk" \\
661
- --license cc-by-4.0 --tier local-only --size 200 --domain news
662
- # Register a sovereign held-out test set (metadata only, text never uploaded):
663
- champollion register-corpus --yes --name "Community holdout" --pair "eng>crk" \\
664
- --license cc-by-nc-4.0 --tier private --size 500 --domain educational
665
- # Publish a fetch-from-source pointer (gated to redistributable licenses):
666
- champollion register-corpus --yes --name "Tatoeba eng-crk" --pair "eng-crk" \\
667
- --license cc-by-4.0 --tier public --size 1000 --domain conversational \\
668
- --repo-url https://example.org/data.tar --builder tatoeba-challenge
669
- # Seal a community-controlled secret test set (encrypted ON YOUR DEVICE):
670
- champollion register-corpus --yes --name "Sealed eng-crk" --pair "eng>crk" \\
671
- --license proprietary --tier sealed --size 500 --domain educational \\
672
- --seal-input ./secret-corpus.json --threshold-pubkey ./group.pub \\
673
- --custodian-group nehiyawewin-trust --seal-out ./sealed.json \\
674
- --qualifier-id eval-eng-crk-nehiyaw-qualifier-v2026
675
- `);
1182
+ showCommandHelp('register-corpus');
676
1183
  }
677
1184
 
678
1185
  /**