champollion 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +133 -0
  2. package/README.md +387 -0
  3. package/bin/cli.js +278 -0
  4. package/index.js +135 -0
  5. package/lib/api-key.js +127 -0
  6. package/lib/autofix.js +432 -0
  7. package/lib/bridge/method_bridge.py +430 -0
  8. package/lib/card-source-resolution.mjs +284 -0
  9. package/lib/cards/cache.js +169 -0
  10. package/lib/cards/env.js +82 -0
  11. package/lib/cards/fetch-card-child.js +38 -0
  12. package/lib/cards/reader.js +435 -0
  13. package/lib/cards/refresh.js +111 -0
  14. package/lib/cards/remote.js +387 -0
  15. package/lib/cldf-export.mjs +540 -0
  16. package/lib/cldf-terms.mjs +62 -0
  17. package/lib/command-help.js +790 -0
  18. package/lib/commands/audit.js +49 -0
  19. package/lib/commands/card.js +454 -0
  20. package/lib/commands/doctor.js +559 -0
  21. package/lib/commands/fonts.js +489 -0
  22. package/lib/commands/help.js +91 -0
  23. package/lib/commands/init.js +1259 -0
  24. package/lib/commands/integrity.js +148 -0
  25. package/lib/commands/leaderboard.js +478 -0
  26. package/lib/commands/lint.js +30 -0
  27. package/lib/commands/models.js +177 -0
  28. package/lib/commands/plugin.js +103 -0
  29. package/lib/commands/provenance.js +45 -0
  30. package/lib/commands/recommend.js +75 -0
  31. package/lib/commands/register-corpus.js +678 -0
  32. package/lib/commands/repair-script.js +42 -0
  33. package/lib/commands/seal-corpus.js +355 -0
  34. package/lib/commands/seo.js +72 -0
  35. package/lib/commands/serve.js +147 -0
  36. package/lib/commands/status.js +265 -0
  37. package/lib/commands/submit.js +332 -0
  38. package/lib/commands/sync.js +89 -0
  39. package/lib/commands/tm.js +573 -0
  40. package/lib/commands/verify.js +39 -0
  41. package/lib/commands/watch.js +20 -0
  42. package/lib/commands/wrap.js +138 -0
  43. package/lib/commands/xliff.js +327 -0
  44. package/lib/commercial-eligibility.js +235 -0
  45. package/lib/concurrent.js +87 -0
  46. package/lib/config.js +523 -0
  47. package/lib/contamination-lane.js +76 -0
  48. package/lib/content-sync.js +731 -0
  49. package/lib/content.js +733 -0
  50. package/lib/corpus-registration.mjs +608 -0
  51. package/lib/cost-report.js +346 -0
  52. package/lib/diff.js +155 -0
  53. package/lib/docusaurus-sync.js +1256 -0
  54. package/lib/flatten.js +55 -0
  55. package/lib/format.js +954 -0
  56. package/lib/hash.js +159 -0
  57. package/lib/icu.js +473 -0
  58. package/lib/integrity.js +689 -0
  59. package/lib/license-gate.mjs +478 -0
  60. package/lib/license-identify.mjs +229 -0
  61. package/lib/lint.js +629 -0
  62. package/lib/method-manifest.js +60 -0
  63. package/lib/methods/anthropic.js +140 -0
  64. package/lib/methods/apertium.js +163 -0
  65. package/lib/methods/api.js +316 -0
  66. package/lib/methods/base.js +184 -0
  67. package/lib/methods/content-separator.js +45 -0
  68. package/lib/methods/deepl.js +426 -0
  69. package/lib/methods/direct-llm.js +586 -0
  70. package/lib/methods/external.js +332 -0
  71. package/lib/methods/fetch-with-retry.js +124 -0
  72. package/lib/methods/gemini.js +147 -0
  73. package/lib/methods/google-translate.js +402 -0
  74. package/lib/methods/http-utils.js +122 -0
  75. package/lib/methods/libretranslate.js +314 -0
  76. package/lib/methods/llm-coached.js +670 -0
  77. package/lib/methods/llm.js +592 -0
  78. package/lib/methods/local.js +76 -0
  79. package/lib/methods/microsoft-translator.js +331 -0
  80. package/lib/methods/openai.js +131 -0
  81. package/lib/methods/openrouter-client.js +327 -0
  82. package/lib/methods/openrouter-pricing.js +156 -0
  83. package/lib/methods/provider-env.js +115 -0
  84. package/lib/methods/provider-pricing.js +310 -0
  85. package/lib/methods/tilde.js +150 -0
  86. package/lib/methods/translated.js +229 -0
  87. package/lib/methods/translation-error.js +80 -0
  88. package/lib/models.js +258 -0
  89. package/lib/no-translate.js +233 -0
  90. package/lib/output.js +238 -0
  91. package/lib/pairs.js +547 -0
  92. package/lib/plugins.js +447 -0
  93. package/lib/provenance.js +323 -0
  94. package/lib/recommend.js +648 -0
  95. package/lib/registers.js +1185 -0
  96. package/lib/repair-script.js +266 -0
  97. package/lib/scripts.js +994 -0
  98. package/lib/seal.mjs +464 -0
  99. package/lib/sealed-qualifier.mjs +211 -0
  100. package/lib/security.js +59 -0
  101. package/lib/segment.js +369 -0
  102. package/lib/seo.js +275 -0
  103. package/lib/serve.js +854 -0
  104. package/lib/string-classify.js +85 -0
  105. package/lib/submit.mjs +344 -0
  106. package/lib/sync.js +969 -0
  107. package/lib/tags/bcp47.js +202 -0
  108. package/lib/tags/resolve.js +314 -0
  109. package/lib/terminology.js +111 -0
  110. package/lib/tm-seed.js +294 -0
  111. package/lib/tm.js +515 -0
  112. package/lib/translate-pair.js +197 -0
  113. package/lib/translate.js +203 -0
  114. package/lib/types.js +230 -0
  115. package/lib/validate.js +510 -0
  116. package/lib/verify.js +451 -0
  117. package/lib/watch.js +145 -0
  118. package/lib/xliff.js +184 -0
  119. package/package.json +93 -0
  120. package/shared/ATTRIBUTION.md +145 -0
  121. package/shared/CORPORA-CARDS.md +288 -0
  122. package/shared/DATA-SOVEREIGNTY.md +500 -0
  123. package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
  124. package/shared/card-lint-baseline.json +3189 -0
  125. package/shared/cards-fallback.json +1 -0
  126. package/shared/catalogue/card-config.json +6091 -0
  127. package/shared/catalogue/external-results.json +3888 -0
  128. package/shared/catalogue/gender-guidance.json +1038 -0
  129. package/shared/catalogue/method-coverage.json +1751 -0
  130. package/shared/catalogue/metric-coverage.json +170 -0
  131. package/shared/catalogue/metric-reliability.json +1 -0
  132. package/shared/catalogue/register-presets.json +3180 -0
  133. package/shared/catalogue/vitality-scales.json +55 -0
  134. package/shared/cldr-index.json +1115 -0
  135. package/shared/code-bridge.json +253 -0
  136. package/shared/corpora-cards-v1-reference.md +281 -0
  137. package/shared/curated-dictionary-flags.json +35 -0
  138. package/shared/curated-endonyms.json +35 -0
  139. package/shared/curated-fsts.json +51 -0
  140. package/shared/curated-orthography-conventions.json +26 -0
  141. package/shared/curated-sil-resources.json +374 -0
  142. package/shared/curated-tools.json +41 -0
  143. package/shared/docent/corpus.json +11333 -0
  144. package/shared/docent/faq.en.json +564 -0
  145. package/shared/docent/register-blocks.json +60 -0
  146. package/shared/docent/system-prompt.md +144 -0
  147. package/shared/domain-taxonomy.json +35 -0
  148. package/shared/explainers/glossary.json +2975 -0
  149. package/shared/explainers/tc-features.json +20112 -0
  150. package/shared/explainers/term-watchlist.json +147 -0
  151. package/shared/human-services.json +59 -0
  152. package/shared/license-corrections.json +261 -0
  153. package/shared/license-evidence.json +13452 -0
  154. package/shared/licenses.json +6781 -0
  155. package/shared/method-registry.json +236 -0
  156. package/shared/metric-registry.json +620 -0
  157. package/shared/model-aliases.json +7 -0
  158. package/shared/schemas/champollion-plugin.schema.json +206 -0
  159. package/shared/schemas/corpora-card.schema.json +957 -0
  160. package/shared/schemas/domain-taxonomy.schema.json +64 -0
  161. package/shared/schemas/external-results.schema.json +314 -0
  162. package/shared/schemas/human-services.schema.json +90 -0
  163. package/shared/schemas/language-card.schema.json +1308 -0
  164. package/shared/schemas/licenses.schema.json +155 -0
  165. package/shared/schemas/method-card.schema.json +412 -0
  166. package/shared/schemas/method-registry.schema.json +85 -0
  167. package/shared/schemas/metric-registry.schema.json +96 -0
  168. package/shared/schemas/metric-reliability.schema.json +178 -0
  169. package/shared/schemas/model-aliases.schema.json +27 -0
  170. package/shared/schemas/source-snapshot.schema.json +96 -0
@@ -0,0 +1,678 @@
1
+ /**
2
+ * Command: register-corpus
3
+ *
4
+ * Register a new evaluation corpus, with the author in control of BOTH its
5
+ * license and how far it travels. Three exposure tiers, defaulting to the most
6
+ * private:
7
+ *
8
+ * 1. local-only — never registered, never uploaded; the card + your text stay
9
+ * entirely on your machine.
10
+ * 2. private — a WMT-style sovereign held-out / secret test set: register
11
+ * METADATA ONLY; your text is never uploaded or hosted; you keep custody.
12
+ * 3. public — publish a metadata card + a fetch-from-source pointer; your
13
+ * text is never hosted by us. Gated by cli/lib/license-gate.mjs so NC /
14
+ * no-redistribute / unconfirmed licenses cannot enter the public lane.
15
+ *
16
+ * We NEVER read, upload, or host corpus content in ANY tier — only metadata and
17
+ * (for public) a pointer to where the data is fetched from. This command writes
18
+ * a metadata card; it never touches your sentences.
19
+ *
20
+ * Interactive wizard when stdin is a TTY; fully scriptable via flags otherwise
21
+ * (or with --yes). See `champollion register-corpus --help`.
22
+ *
23
+ * Exit codes: 0 = registered/saved (or cancelled); 1 = invalid request / gate block.
24
+ */
25
+
26
+ import fs from 'node:fs';
27
+ import path from 'node:path';
28
+ import readline from 'node:readline';
29
+ import { fileURLToPath } from 'node:url';
30
+ import { output } from '../output.js';
31
+ import {
32
+ LICENSE_OPTIONS,
33
+ EXPOSURE_TIERS,
34
+ DEFAULT_TIER,
35
+ resolveLicense,
36
+ resolveTier,
37
+ gatePublicRegistration,
38
+ buildCorpusCard,
39
+ deriveCardId,
40
+ slugify,
41
+ resolveDestination,
42
+ validateRegistration,
43
+ } from '../corpus-registration.mjs';
44
+ import {
45
+ sealPlaintext,
46
+ buildSealedArtifact,
47
+ buildSealedCardBlock,
48
+ resolveThresholdPublicKey,
49
+ buildAad,
50
+ } from '../seal.mjs';
51
+ import { DEFAULT_QUALIFIER_THRESHOLD } from '../sealed-qualifier.mjs';
52
+
53
+ // Tracked corpora-cards SSOT, resolved relative to this module.
54
+ const DEFAULT_CARDS_DIR = fileURLToPath(new URL('../../shared/corpora-cards/', import.meta.url));
55
+
56
+ const CONTAMINATION_LEVELS = ['NONE', 'LOW', 'MEDIUM', 'HIGH'];
57
+
58
+ /**
59
+ * Resolve a path to its REAL location, following symlinks. The destination
60
+ * directory may not exist yet, so we realpath the deepest existing ancestor
61
+ * and re-append the remaining (non-existent, therefore non-symlink) segments.
62
+ * Unlike path.resolve, this defeats a symlinked ancestor that points elsewhere.
63
+ *
64
+ * @param {string} p - Path to resolve (absolute or relative to cwd)
65
+ * @returns {string} Real, symlink-free absolute path
66
+ */
67
+ function realPathDeep(p) {
68
+ let cur = path.resolve(p);
69
+ const tail = [];
70
+ while (!fs.existsSync(cur)) {
71
+ tail.unshift(path.basename(cur));
72
+ const parent = path.dirname(cur);
73
+ if (parent === cur) return path.resolve(p); // reached root, nothing exists
74
+ cur = parent;
75
+ }
76
+ let real;
77
+ try { real = fs.realpathSync(cur); } catch { return path.resolve(p); }
78
+ return tail.length ? path.join(real, ...tail) : real;
79
+ }
80
+
81
+ /**
82
+ * True if `child` is `parent` itself or nested anywhere under it.
83
+ * Both arguments must already be real (symlink-resolved) paths.
84
+ */
85
+ function isWithin(child, parent) {
86
+ const rel = path.relative(parent, child);
87
+ return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
88
+ }
89
+
90
+ function isInteractive() {
91
+ return process.stdin.isTTY === true;
92
+ }
93
+
94
+ function ask(rl, question, defaultValue) {
95
+ return new Promise((resolve) => {
96
+ const suffix = defaultValue ? ` (${defaultValue})` : '';
97
+ rl.question(` ${question}${suffix}: `, (answer) => {
98
+ resolve(answer.trim() || defaultValue || '');
99
+ });
100
+ });
101
+ }
102
+
103
+ /** Parse "src>tgt" / "src-tgt" / "src,tgt" into {source, target}, else null. */
104
+ function parsePair(value) {
105
+ if (!value) return null;
106
+ const m = String(value).trim().split(/[>\-,\s]+/).filter(Boolean);
107
+ if (m.length >= 2) return { source: m[0].toLowerCase(), target: m[1].toLowerCase() };
108
+ return null;
109
+ }
110
+
111
+ // ── Catalog printing (plain language for humans, machine-readable for agents) ──
112
+
113
+ function printCatalogs(asJson) {
114
+ if (asJson) {
115
+ console.log(JSON.stringify({
116
+ licenses: LICENSE_OPTIONS.map((o) => ({ key: o.key, spdx: o.spdx, commercial: o.commercial, redistribution: o.redistribution, publicEligible: o.publicEligible, explanation: o.explanation })),
117
+ tiers: EXPOSURE_TIERS.map((t) => ({ key: t.key, registers: t.registers, uploadsContent: t.uploadsContent, uploadsCiphertext: !!t.uploadsCiphertext, explanation: t.explanation })),
118
+ default_tier: DEFAULT_TIER,
119
+ }, null, 2));
120
+ return;
121
+ }
122
+ console.log('');
123
+ console.log(' LICENSES — pick the terms others may use your corpus under:');
124
+ console.log('');
125
+ LICENSE_OPTIONS.forEach((o, i) => {
126
+ console.log(` ${i + 1}. ${o.label}`);
127
+ console.log(` ${o.explanation}`);
128
+ console.log(` commercial: ${o.commercial} · redistribution: ${o.redistribution} · public-lane: ${o.publicEligible ? 'eligible' : 'blocked'}`);
129
+ });
130
+ console.log('');
131
+ console.log(' EXPOSURE TIERS — how far your corpus travels (default: most private):');
132
+ console.log('');
133
+ EXPOSURE_TIERS.forEach((t, i) => {
134
+ const def = t.key === DEFAULT_TIER ? ' ★ default' : '';
135
+ console.log(` ${i + 1}. ${t.label}${def}`);
136
+ console.log(` ${t.explanation}`);
137
+ });
138
+ console.log('');
139
+ console.log(' Champollion never reads, uploads, or hosts your corpus text — in ANY tier.');
140
+ console.log('');
141
+ }
142
+
143
+ // ── Interactive wizard ──────────────────────────────────────────────────────
144
+
145
+ async function runInteractive(rl, args) {
146
+ console.log('');
147
+ console.log(' champollion — Register a Corpus');
148
+ console.log(' ════════════════════════════════════════════════');
149
+ console.log('');
150
+ console.log(' You choose the license and how far this corpus travels.');
151
+ console.log(' We never read, upload, or host your text — only metadata,');
152
+ console.log(' and (for public sets) a pointer to where the data is fetched from.');
153
+
154
+ // Step 1 — basics
155
+ console.log('');
156
+ console.log(' Step 1/4 — Basics');
157
+ console.log(' ────────────────────────────────────────────────');
158
+ const name = await ask(rl, 'Corpus name', args.name || '');
159
+ const pairStr = await ask(rl, 'Language pair (e.g. eng-crk, or "eng>crk")', args.pair || '');
160
+ const pair = parsePair(pairStr);
161
+ const publisher = await ask(rl, 'Publisher / your name or org', args.publisher || '');
162
+ const description = await ask(rl, 'One-line description', args.description || `${name} evaluation corpus`);
163
+
164
+ // Step 2 — license
165
+ console.log('');
166
+ console.log(' Step 2/4 — License');
167
+ console.log(' ────────────────────────────────────────────────');
168
+ console.log('');
169
+ LICENSE_OPTIONS.forEach((o, i) => {
170
+ console.log(` ${i + 1}. ${o.label} — ${o.explanation}`);
171
+ });
172
+ console.log('');
173
+ const licChoice = await ask(rl, 'Choose a license', '1');
174
+ const licenseOption = resolveLicense(licChoice) || LICENSE_OPTIONS[0];
175
+ console.log(` → ${licenseOption.label} (commercial: ${licenseOption.commercial}, redistribution: ${licenseOption.redistribution})`);
176
+
177
+ // Step 3 — exposure tier (default most private)
178
+ console.log('');
179
+ console.log(' Step 3/4 — Exposure');
180
+ console.log(' ────────────────────────────────────────────────');
181
+ console.log('');
182
+ EXPOSURE_TIERS.forEach((t, i) => {
183
+ const def = t.key === DEFAULT_TIER ? ' ★ safe default' : '';
184
+ console.log(` ${i + 1}. ${t.label}${def}`);
185
+ console.log(` ${t.explanation}`);
186
+ });
187
+ console.log('');
188
+ const tierChoice = await ask(rl, 'Choose exposure', '1');
189
+ let tier = resolveTier(tierChoice);
190
+
191
+ // Gate the public tier through the license gate, with a graceful fallback.
192
+ if (tier.key === 'public') {
193
+ const gate = gatePublicRegistration(licenseOption);
194
+ if (!gate.allowed) {
195
+ console.log('');
196
+ console.log(` ⚠ ${gate.reason}`);
197
+ const fallback = await ask(rl, 'Register privately instead (metadata only, text never uploaded)?', 'yes');
198
+ if (fallback.toLowerCase().startsWith('y')) {
199
+ tier = resolveTier('private');
200
+ console.log(' → Switched to private (sovereign held-out) registration.');
201
+ } else {
202
+ return { cancelled: true, reason: gate.reason };
203
+ }
204
+ } else {
205
+ console.log(` → ${gate.reason}`);
206
+ }
207
+ }
208
+
209
+ // Public tier needs a fetch-from-source pointer (we host nothing).
210
+ let repoUrl = args['repo-url'] || null;
211
+ let builder = args.builder || null;
212
+ let sourceUrl = args['source-url'] || null;
213
+ if (tier.key === 'public') {
214
+ console.log('');
215
+ console.log(' Public sets are fetched from their source — give us a pointer, not the data:');
216
+ sourceUrl = await ask(rl, 'Canonical URL (project/dataset page)', sourceUrl || '');
217
+ repoUrl = await ask(rl, 'Fetch-from-source archive/repo URL', repoUrl || '');
218
+ builder = await ask(rl, 'Builder adapter id (rebuilds from source, e.g. tatoeba-challenge)', builder || '');
219
+ }
220
+
221
+ // Sealed tier — encrypt on this device, hand us ciphertext + a content-free card.
222
+ let custodianGroupId = args['custodian-group'] || null;
223
+ let thresholdPubkey = args['threshold-pubkey'] || null;
224
+ let sealInput = args['seal-input'] || null;
225
+ let sealOut = args['seal-out'] || null;
226
+ let qualifierId = args['qualifier-id'] || null;
227
+ let qualifierThreshold = args['qualifier-threshold'] != null ? Number(args['qualifier-threshold']) : null;
228
+ const keyScheme = args['key-scheme'] || null;
229
+ if (tier.key === 'sealed') {
230
+ console.log('');
231
+ console.log(' Sealed sets are encrypted ON THIS DEVICE before anything leaves.');
232
+ console.log(' We receive ciphertext + a content-free card only — we cannot read it.');
233
+ console.log('');
234
+ sealInput = await ask(rl, 'Path to the corpus file to seal (your plaintext, stays local)', sealInput || '');
235
+ thresholdPubkey = await ask(rl, 'Custodian group threshold public key (path or PEM/base64)', thresholdPubkey || '');
236
+ custodianGroupId = await ask(rl, 'Custodian group id (the community group that holds the key)', custodianGroupId || '');
237
+ sealOut = await ask(rl, 'Where to write the encrypted artifact', sealOut || '');
238
+ console.log('');
239
+ console.log(' A sealed set must be paired with a PUBLIC qualifier (a disjoint CC-BY/CC0');
240
+ console.log(' twin) that a method must clear before any sealed run can be proposed.');
241
+ qualifierId = await ask(rl, 'Paired public qualifier card id (e.g. eval-eng-crk-…-qualifier-v2026)', qualifierId || '');
242
+ const qt = await ask(rl, 'Qualifier clearance threshold', String(qualifierThreshold ?? DEFAULT_QUALIFIER_THRESHOLD));
243
+ qualifierThreshold = Number(qt);
244
+ }
245
+
246
+ // Step 4 — consumer-reports metadata
247
+ console.log('');
248
+ console.log(' Step 4/4 — Dataset details (the "consumer report")');
249
+ console.log(' ────────────────────────────────────────────────');
250
+ const sizeStr = await ask(rl, 'Size — number of sentence pairs', args.size || '');
251
+ const domain = await ask(rl, 'Domain (news, conversational, educational, …)', args.domain || 'mixed');
252
+ const defaultRisk = tier.key === 'public' ? 'LOW' : 'NONE';
253
+ console.log('');
254
+ console.log(' Contamination risk — is this text in known LLM training sets?');
255
+ console.log(' NONE = private/unpublished · LOW = niche/recent · MEDIUM = public · HIGH = in major training sets');
256
+ const riskIn = await ask(rl, 'Contamination risk', defaultRisk);
257
+ const contaminationRisk = CONTAMINATION_LEVELS.includes(riskIn.toUpperCase()) ? riskIn.toUpperCase() : defaultRisk;
258
+
259
+ return {
260
+ name, pair, publisher, description,
261
+ licenseOption, tier,
262
+ repoUrl, builder, sourceUrl,
263
+ size: Number(sizeStr), domain, contaminationRisk,
264
+ custodianGroupId, thresholdPubkey, sealInput, sealOut,
265
+ qualifierId, qualifierThreshold, keyScheme,
266
+ };
267
+ }
268
+
269
+ // ── Non-interactive (flags) ─────────────────────────────────────────────────
270
+
271
+ function fromFlags(args) {
272
+ const pair = parsePair(args.pair) ||
273
+ (args['source-lang'] && args['target-lang']
274
+ ? { source: String(args['source-lang']).toLowerCase(), target: String(args['target-lang']).toLowerCase() }
275
+ : null);
276
+ const licenseOption = resolveLicense(args.license);
277
+ const tier = resolveTier(args.tier || args.exposure);
278
+ const riskIn = args.contamination ? String(args.contamination).toUpperCase() : null;
279
+ return {
280
+ name: args.name || '',
281
+ pair,
282
+ publisher: args.publisher || (args.name ? '' : ''),
283
+ description: args.description || (args.name ? `${args.name} evaluation corpus` : ''),
284
+ licenseOption,
285
+ tier,
286
+ repoUrl: args['repo-url'] || null,
287
+ builder: args.builder || null,
288
+ sourceUrl: args['source-url'] || null,
289
+ size: args.size != null ? Number(args.size) : NaN,
290
+ domain: args.domain || '',
291
+ contaminationRisk: riskIn && CONTAMINATION_LEVELS.includes(riskIn) ? riskIn : null,
292
+ // sealed tier
293
+ custodianGroupId: args['custodian-group'] || null,
294
+ thresholdPubkey: args['threshold-pubkey'] || null,
295
+ sealInput: args['seal-input'] || null,
296
+ sealOut: args['seal-out'] || null,
297
+ qualifierId: args['qualifier-id'] || null,
298
+ qualifierThreshold: args['qualifier-threshold'] != null ? Number(args['qualifier-threshold']) : null,
299
+ keyScheme: args['key-scheme'] || null,
300
+ };
301
+ }
302
+
303
+ // ── Sealed tier: client-side encryption ──────────────────────────────────────
304
+
305
+ /**
306
+ * Seal a corpus on the author's device: read the local plaintext, encrypt it to
307
+ * the custodian group's threshold PUBLIC key, write a ciphertext-ONLY artifact,
308
+ * and return the content-free `sealed` card block. The plaintext never leaves
309
+ * this machine in readable form; we hold only ciphertext + metadata, and we
310
+ * cannot decrypt it (no single party can — see docs/governance/OCAP_MULTISIG_PLAN.md).
311
+ *
312
+ * @returns {{cardBlock:object, artifactPath:string}|{error:string}}
313
+ */
314
+ function sealCorpusForRegistration({ req, id, addedAt, cwd }) {
315
+ // 1. Read the author's plaintext corpus (stays on this device).
316
+ let plaintext;
317
+ try {
318
+ plaintext = fs.readFileSync(path.resolve(cwd, req.sealInput));
319
+ } catch (e) {
320
+ return { error: `cannot read the corpus file (--seal-input ${req.sealInput}): ${e.message}` };
321
+ }
322
+ if (!plaintext.length) return { error: `the corpus file (--seal-input ${req.sealInput}) is empty — nothing to seal.` };
323
+
324
+ // 2. Resolve the custodian group's threshold PUBLIC key (a path or an inline
325
+ // PEM/base64 value). WAVE-2 SEAM: this becomes the aggregated 3-of-5 FROST
326
+ // group key from the custodian key ceremony — same call site, same format.
327
+ let thresholdKeyInput = req.thresholdPubkey;
328
+ let keyAsPath = null;
329
+ try { keyAsPath = path.resolve(cwd, String(req.thresholdPubkey)); } catch { keyAsPath = null; }
330
+ if (keyAsPath && fs.existsSync(keyAsPath) && fs.statSync(keyAsPath).isFile()) {
331
+ try { thresholdKeyInput = fs.readFileSync(keyAsPath, 'utf-8').trim(); }
332
+ catch (e) { return { error: `cannot read the threshold public key file: ${e.message}` }; }
333
+ }
334
+ let thresholdPublicKey;
335
+ try { thresholdPublicKey = resolveThresholdPublicKey(thresholdKeyInput); }
336
+ catch (e) { return { error: e.message }; }
337
+
338
+ // 3. Encrypt CLIENT-SIDE. The AAD binds the ciphertext to this card + group.
339
+ const aad = buildAad({ cardId: id, custodianGroupId: req.custodianGroupId });
340
+ let sealed;
341
+ try { sealed = sealPlaintext({ plaintext, thresholdPublicKey, aad }); }
342
+ catch (e) { return { error: e.message }; }
343
+ // Drop the plaintext reference as soon as it is sealed.
344
+ plaintext = null;
345
+
346
+ // 4. Write the ciphertext-ONLY artifact (the off-git encrypted store stand-in).
347
+ const artifactPath = req.sealOut
348
+ ? path.resolve(cwd, req.sealOut)
349
+ : path.resolve(cwd, `${id}.sealed.json`);
350
+ // The ciphertext artifact must NEVER land in the tracked corpora-cards SSOT.
351
+ const ssotReal = realPathDeep(DEFAULT_CARDS_DIR);
352
+ if (isWithin(realPathDeep(artifactPath), ssotReal)) {
353
+ return { error: 'the sealed (ciphertext) artifact must not be written into the tracked corpora-cards directory — choose a different --seal-out path.' };
354
+ }
355
+ const artifact = buildSealedArtifact({
356
+ sealed, cardId: id, custodianGroupId: req.custodianGroupId, createdAt: addedAt,
357
+ });
358
+ try {
359
+ fs.mkdirSync(path.dirname(artifactPath), { recursive: true });
360
+ fs.writeFileSync(artifactPath, JSON.stringify(artifact, null, 2) + '\n', 'utf-8');
361
+ } catch (e) {
362
+ return { error: `cannot write the sealed artifact to ${artifactPath}: ${e.message}` };
363
+ }
364
+
365
+ // 5. The content-free `sealed` block for the card (cipher / group / digest /
366
+ // AAD + key + paired-qualifier references).
367
+ const qt = req.qualifierThreshold != null && !Number.isNaN(req.qualifierThreshold)
368
+ ? req.qualifierThreshold
369
+ : DEFAULT_QUALIFIER_THRESHOLD;
370
+ const cardBlock = buildSealedCardBlock({
371
+ sealed,
372
+ custodianGroupId: req.custodianGroupId,
373
+ keyScheme: req.keyScheme || 'TSS-3-of-5',
374
+ qualifierId: req.qualifierId,
375
+ qualifierThreshold: qt,
376
+ artifactRef: path.basename(artifactPath),
377
+ });
378
+
379
+ return { cardBlock, artifactPath };
380
+ }
381
+
382
+ /**
383
+ * TODO (Wave 2 — web wizard): the browser registration wizard at
384
+ * champollion.dev will offer the same "Seal it" choice, doing the IDENTICAL
385
+ * client-side encryption in-browser (WebCrypto: X25519 ECDH → HKDF → AES-GCM)
386
+ * so the plaintext never leaves the user's browser either. It will POST only the
387
+ * ciphertext artifact + the content-free card produced here. The CLI path in
388
+ * this file is the Wave-1 deliverable and the reference implementation; the web
389
+ * wizard must produce a byte-compatible artifact (same SEAL_CIPHER/SEAL_VERSION
390
+ * envelope in lib/seal.mjs). Not implemented yet — the CLI path below is the
391
+ * only seal path today.
392
+ */
393
+
394
+ // ── Main ────────────────────────────────────────────────────────────────────
395
+
396
+ async function run(args, cwd) {
397
+ if (args.help) {
398
+ showHelp();
399
+ return 0;
400
+ }
401
+ if (args.list) {
402
+ printCatalogs(!!args.json);
403
+ return 0;
404
+ }
405
+
406
+ // Collect the registration request.
407
+ let req;
408
+ if (!args.yes && isInteractive()) {
409
+ const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
410
+ try {
411
+ req = await runInteractive(rl, args);
412
+ } finally {
413
+ rl.close();
414
+ }
415
+ if (req && req.cancelled) {
416
+ console.log('');
417
+ output.warn('Registration cancelled — nothing was written.');
418
+ return 0;
419
+ }
420
+ } else {
421
+ req = fromFlags(args);
422
+ }
423
+
424
+ // Validate (collects every problem, fails loud).
425
+ const v = validateRegistration({
426
+ tier: req.tier,
427
+ licenseOption: req.licenseOption,
428
+ pair: req.pair || {},
429
+ name: req.name,
430
+ repoUrl: req.repoUrl,
431
+ builder: req.builder,
432
+ size: req.size,
433
+ domain: req.domain,
434
+ custodianGroupId: req.custodianGroupId,
435
+ thresholdPublicKey: req.thresholdPubkey,
436
+ sealInput: req.sealInput,
437
+ qualifierId: req.qualifierId,
438
+ });
439
+ if (!v.ok) {
440
+ console.error('[ERR] Cannot register this corpus:');
441
+ for (const e of v.errors) console.error(` • ${e}`);
442
+ console.error('');
443
+ console.error(' Run "champollion register-corpus --list" to see licenses + tiers,');
444
+ console.error(' or "champollion register-corpus --help" for all flags.');
445
+ return 1;
446
+ }
447
+
448
+ // Build the (content-free) card.
449
+ const version = args.version || '1';
450
+ const slug = slugify(args.id || req.publisher || req.name || 'custom');
451
+ const id = args.id && /^(ref|eval)-/.test(args.id)
452
+ ? args.id
453
+ : deriveCardId({ source: req.pair.source, target: req.pair.target, slug, version });
454
+
455
+ const addedAt = new Date().toISOString().slice(0, 10);
456
+
457
+ // ── SEALED TIER: encrypt on THIS device before anything leaves ──────────────
458
+ // We compute the card id first (the AAD binds the ciphertext to it), read the
459
+ // author's plaintext locally, encrypt it to the custodian group's threshold
460
+ // public key, write a ciphertext-ONLY artifact, and keep only a content-free
461
+ // sealed block for the card. The plaintext is never uploaded, hosted, or kept.
462
+ let sealedBlock = null;
463
+ let sealedArtifactPath = null;
464
+ if (req.tier.key === 'sealed') {
465
+ const sealed = sealCorpusForRegistration({ req, id, addedAt, cwd });
466
+ if (sealed.error) {
467
+ console.error(`[ERR] Could not seal this corpus: ${sealed.error}`);
468
+ console.error(' Nothing was written. Your corpus text never left your machine.');
469
+ return 1;
470
+ }
471
+ sealedBlock = sealed.cardBlock;
472
+ sealedArtifactPath = sealed.artifactPath;
473
+ }
474
+
475
+ const card = buildCorpusCard({
476
+ id,
477
+ name: req.name,
478
+ version: /^\d/.test(version) ? `${version}.0.0`.split('.').slice(0, 3).join('.') : '0.1.0',
479
+ description: req.description || `${req.name} evaluation corpus`,
480
+ pair: req.pair,
481
+ publisher: req.publisher || 'Unattributed (author-registered)',
482
+ sourceUrl: req.sourceUrl,
483
+ repoUrl: req.repoUrl,
484
+ builder: req.builder,
485
+ sha256: args.sha256 || null,
486
+ licenseUrl: args['license-url'] || null,
487
+ licenseOption: req.licenseOption,
488
+ tier: req.tier,
489
+ contaminationRisk: req.contaminationRisk,
490
+ size: req.size,
491
+ domain: req.domain,
492
+ doNotTrain: args['do-not-train'] !== false,
493
+ addedAt,
494
+ sealed: sealedBlock,
495
+ });
496
+
497
+ // Resolve destination — local-only NEVER lands in the tracked SSOT.
498
+ const corporaCardsDir = args['cards-dir'] ? path.resolve(cwd, args['cards-dir']) : DEFAULT_CARDS_DIR;
499
+ const localDir = args.out ? path.resolve(cwd, args.out) : cwd;
500
+ const dest = resolveDestination({ tier: req.tier, id, corporaCardsDir, localDir });
501
+
502
+ // Guard: a local-only card must never land in the tracked corpora-cards SSOT.
503
+ // path.resolve does NOT follow symlinks and only the exact dir was checked —
504
+ // a symlinked --out (or one nested under the SSOT) slipped past. Resolve real
505
+ // paths and reject anything that IS the SSOT or sits under it. The never-host
506
+ // doctrine is absolute, so this guard must be too.
507
+ const ssotReal = realPathDeep(DEFAULT_CARDS_DIR);
508
+ const targetReal = realPathDeep(dest.dir);
509
+ if (req.tier.key === 'local-only' && isWithin(targetReal, ssotReal)) {
510
+ console.error('[ERR] local-only corpora must not be written into the tracked corpora-cards directory.');
511
+ console.error(' (Symlinks are resolved — a link that points into the SSOT is rejected too.)');
512
+ console.error(' Choose a different --out path, or use --tier private to register metadata.');
513
+ return 1;
514
+ }
515
+
516
+ const filePath = path.join(dest.dir, dest.filename);
517
+ if (fs.existsSync(filePath)) {
518
+ console.error(`[ERR] A card already exists at ${filePath}`);
519
+ console.error(' Pick a different --id, or remove the existing card first.');
520
+ return 1;
521
+ }
522
+ fs.mkdirSync(dest.dir, { recursive: true });
523
+ fs.writeFileSync(filePath, JSON.stringify(card, null, 2) + '\n', 'utf-8');
524
+
525
+ // ── Report ──
526
+ if (args.json) {
527
+ console.log(JSON.stringify({
528
+ ok: true,
529
+ id,
530
+ exposureTier: req.tier.key,
531
+ registered: dest.registered,
532
+ tracked: dest.tracked,
533
+ uploadedContent: false,
534
+ uploadedPlaintext: false,
535
+ ...(req.tier.key === 'sealed' ? {
536
+ sealed: true,
537
+ sealedArtifactPath,
538
+ ciphertextDigest: sealedBlock.ciphertextDigest,
539
+ cipher: sealedBlock.cipher,
540
+ custodianGroupId: sealedBlock.custodianGroupId,
541
+ qualifierId: sealedBlock.qualifierId,
542
+ qualifierThreshold: sealedBlock.qualifierThreshold,
543
+ } : {}),
544
+ license: card.license,
545
+ path: filePath,
546
+ }, null, 2));
547
+ return 0;
548
+ }
549
+
550
+ console.log('');
551
+ output.ok(`${req.tier.key === 'local-only' ? 'Saved' : 'Registered'} corpus card: ${id}`);
552
+ console.log('');
553
+ console.log(` Exposure: ${req.tier.label}`);
554
+ console.log(` License: ${card.license.spdx} (commercial: ${card.license.commercial}, redistribution: ${card.license.redistribution})`);
555
+ console.log(` Pair: ${req.pair.source}→${req.pair.target}`);
556
+ console.log(` Size/domain: ${req.size} ${card.dev.sizeUnit} · ${card.dev.domain}`);
557
+ console.log(` Contamination: ${card.contamination.risk}`);
558
+ console.log(` Written to: ${filePath}`);
559
+ console.log('');
560
+
561
+ if (req.tier.key === 'sealed') {
562
+ console.log(` Sealed artifact: ${sealedArtifactPath}`);
563
+ console.log(` Cipher: ${sealedBlock.cipher}`);
564
+ console.log(` Ciphertext digest: ${sealedBlock.ciphertextDigest}`);
565
+ console.log(` Custodian group: ${sealedBlock.custodianGroupId}`);
566
+ console.log(` Qualifier: ${sealedBlock.qualifierId} (clear ≥ ${sealedBlock.qualifierThreshold} first)`);
567
+ console.log('');
568
+ console.log(' ✓ Your sentences were SCRAMBLED LOCALLY on this machine before anything');
569
+ console.log(' was written. They are NEVER sent or stored in readable form. The only');
570
+ console.log(' artifact produced is ciphertext — Champollion cannot decrypt it, and no');
571
+ console.log(' single party can: it takes M-of-N custodian approval (the threshold key).');
572
+ console.log('');
573
+ console.log(' The card is content-free and quarantined from the public queue/leaderboard.');
574
+ console.log(' A method must first clear the paired PUBLIC qualifier before any sealed run');
575
+ console.log(' can even be proposed — and that run still requires custodian approval.');
576
+ } else if (req.tier.key === 'local-only') {
577
+ console.log(' This card stays on your machine. Nothing was registered or uploaded,');
578
+ console.log(' and your corpus text was never read. Delete the file to remove it.');
579
+ } else if (req.tier.key === 'private') {
580
+ console.log(' Metadata only is registered — your corpus TEXT was never read, uploaded,');
581
+ console.log(' or hosted, and never will be. The set is catalogued but quarantined from');
582
+ console.log(' the public queue/leaderboard (custodian-controlled, WMT-style held-out).');
583
+ console.log('');
584
+ console.log(' Next: rebuild the registry, then evaluate privately:');
585
+ console.log(' npm run generate:registries # from the repo root (arena)');
586
+ console.log(` mt-eval run --corpus ${id} --private # publish scores without exposing text`);
587
+ } else {
588
+ console.log(' A metadata card + fetch-from-source pointer is registered — your corpus');
589
+ console.log(' TEXT is never hosted by Champollion; it is fetched from source on demand.');
590
+ console.log('');
591
+ console.log(' Next: rebuild the registry so the pair enters the public queue:');
592
+ console.log(' npm run generate:registries # from the repo root (arena)');
593
+ }
594
+ console.log('');
595
+ return 0;
596
+ }
597
+
598
+ function showHelp() {
599
+ console.log(`
600
+ champollion register-corpus — Register a new evaluation corpus
601
+
602
+ USAGE
603
+ champollion register-corpus [options]
604
+
605
+ DESCRIPTION
606
+ Register a corpus with YOU in control of its license and how far it
607
+ travels. Three exposure tiers, defaulting to the most private. We never
608
+ read, upload, or host your corpus text in ANY tier — only metadata (and,
609
+ for public sets, a pointer to where the data is fetched from).
610
+
611
+ Interactive when run in a terminal; fully scriptable with flags (or --yes).
612
+
613
+ EXPOSURE TIERS (--tier, default: local-only)
614
+ local-only Never registered, never uploaded; card + text stay on your machine.
615
+ private Register METADATA ONLY — a WMT-style sovereign held-out set;
616
+ text never uploaded or hosted; you keep custody.
617
+ public Publish a metadata card + fetch-from-source pointer; text never
618
+ hosted by us. Requires a redistribution-cleared license (gated).
619
+ sealed Encrypt the corpus ON YOUR DEVICE under the custodian group's
620
+ threshold key; we receive ciphertext + a content-free card only
621
+ and cannot decrypt it. Catalogued + quarantined; paired with a
622
+ public qualifier a method must clear before any sealed run.
623
+
624
+ LICENSES (--license)
625
+ cc-by-4.0, cc-by-sa-4.0, cc0-1.0, cc-by-nc-4.0, proprietary, other
626
+ (or pass any SPDX id; it is classified through the license gate)
627
+
628
+ OPTIONS
629
+ --tier <tier> Exposure tier (default: local-only). Alias: --exposure
630
+ --license <id> License key, SPDX id, or list number (see --list)
631
+ --name <text> Corpus name (required)
632
+ --pair <pair> Language pair: "eng>crk" (quote it — the shell eats a
633
+ bare >) or the dash form eng-crk (or --source-lang/--target-lang)
634
+ --publisher <text> Publisher / your name or org
635
+ --description <text> One-line description
636
+ --size <n> Number of sentence pairs (required)
637
+ --domain <text> Domain: news, conversational, educational, … (required)
638
+ --contamination <lvl> NONE | LOW | MEDIUM | HIGH (default: NONE private / LOW public)
639
+ --repo-url <url> Public tier: fetch-from-source archive/repo URL
640
+ --source-url <url> Public tier: canonical project/dataset URL
641
+ --builder <id> Public tier: builder adapter id (rebuilds from source)
642
+ --seal-input <path> Sealed tier: local corpus file to encrypt on-device
643
+ --threshold-pubkey <k> Sealed tier: custodian group threshold public key
644
+ (file path, or inline PEM / base64 DER SPKI, X25519)
645
+ --custodian-group <id> Sealed tier: custodian group id that holds the key
646
+ --seal-out <path> Sealed tier: where to write the ciphertext artifact
647
+ --qualifier-id <id> Sealed tier: paired public qualifier card id (vYYYY)
648
+ --qualifier-threshold <n> Sealed tier: score a method must clear on the qualifier
649
+ --key-scheme <s> Sealed tier: custody scheme label (default: TSS-3-of-5)
650
+ --id <id> Override the generated card id (eval-…)
651
+ --out <dir> local-only: where to write the card (default: cwd)
652
+ --list Print the license + tier catalog (add --json for JSON)
653
+ --json Machine-readable output (for agents)
654
+ --yes Non-interactive; take values from flags
655
+
656
+ EXAMPLES
657
+ champollion register-corpus # interactive wizard
658
+ champollion register-corpus --list # see licenses + tiers
659
+ # Keep a corpus entirely local (nothing leaves your machine):
660
+ champollion register-corpus --yes --name "My set" --pair "eng>crk" \\
661
+ --license cc-by-4.0 --tier local-only --size 200 --domain news
662
+ # Register a sovereign held-out test set (metadata only, text never uploaded):
663
+ champollion register-corpus --yes --name "Community holdout" --pair "eng>crk" \\
664
+ --license cc-by-nc-4.0 --tier private --size 500 --domain educational
665
+ # Publish a fetch-from-source pointer (gated to redistributable licenses):
666
+ champollion register-corpus --yes --name "Tatoeba eng-crk" --pair "eng-crk" \\
667
+ --license cc-by-4.0 --tier public --size 1000 --domain conversational \\
668
+ --repo-url https://example.org/data.tar --builder tatoeba-challenge
669
+ # Seal a community-controlled secret test set (encrypted ON YOUR DEVICE):
670
+ champollion register-corpus --yes --name "Sealed eng-crk" --pair "eng>crk" \\
671
+ --license proprietary --tier sealed --size 500 --domain educational \\
672
+ --seal-input ./secret-corpus.json --threshold-pubkey ./group.pub \\
673
+ --custodian-group nehiyawewin-trust --seal-out ./sealed.json \\
674
+ --qualifier-id eval-eng-crk-nehiyaw-qualifier-v2026
675
+ `);
676
+ }
677
+
678
+ export { run };