champollion 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +133 -0
- package/README.md +387 -0
- package/bin/cli.js +278 -0
- package/index.js +135 -0
- package/lib/api-key.js +127 -0
- package/lib/autofix.js +432 -0
- package/lib/bridge/method_bridge.py +430 -0
- package/lib/card-source-resolution.mjs +284 -0
- package/lib/cards/cache.js +169 -0
- package/lib/cards/env.js +82 -0
- package/lib/cards/fetch-card-child.js +38 -0
- package/lib/cards/reader.js +435 -0
- package/lib/cards/refresh.js +111 -0
- package/lib/cards/remote.js +387 -0
- package/lib/cldf-export.mjs +540 -0
- package/lib/cldf-terms.mjs +62 -0
- package/lib/command-help.js +790 -0
- package/lib/commands/audit.js +49 -0
- package/lib/commands/card.js +454 -0
- package/lib/commands/doctor.js +559 -0
- package/lib/commands/fonts.js +489 -0
- package/lib/commands/help.js +91 -0
- package/lib/commands/init.js +1259 -0
- package/lib/commands/integrity.js +148 -0
- package/lib/commands/leaderboard.js +478 -0
- package/lib/commands/lint.js +30 -0
- package/lib/commands/models.js +177 -0
- package/lib/commands/plugin.js +103 -0
- package/lib/commands/provenance.js +45 -0
- package/lib/commands/recommend.js +75 -0
- package/lib/commands/register-corpus.js +678 -0
- package/lib/commands/repair-script.js +42 -0
- package/lib/commands/seal-corpus.js +355 -0
- package/lib/commands/seo.js +72 -0
- package/lib/commands/serve.js +147 -0
- package/lib/commands/status.js +265 -0
- package/lib/commands/submit.js +332 -0
- package/lib/commands/sync.js +89 -0
- package/lib/commands/tm.js +573 -0
- package/lib/commands/verify.js +39 -0
- package/lib/commands/watch.js +20 -0
- package/lib/commands/wrap.js +138 -0
- package/lib/commands/xliff.js +327 -0
- package/lib/commercial-eligibility.js +235 -0
- package/lib/concurrent.js +87 -0
- package/lib/config.js +523 -0
- package/lib/contamination-lane.js +76 -0
- package/lib/content-sync.js +731 -0
- package/lib/content.js +733 -0
- package/lib/corpus-registration.mjs +608 -0
- package/lib/cost-report.js +346 -0
- package/lib/diff.js +155 -0
- package/lib/docusaurus-sync.js +1256 -0
- package/lib/flatten.js +55 -0
- package/lib/format.js +954 -0
- package/lib/hash.js +159 -0
- package/lib/icu.js +473 -0
- package/lib/integrity.js +689 -0
- package/lib/license-gate.mjs +478 -0
- package/lib/license-identify.mjs +229 -0
- package/lib/lint.js +629 -0
- package/lib/method-manifest.js +60 -0
- package/lib/methods/anthropic.js +140 -0
- package/lib/methods/apertium.js +163 -0
- package/lib/methods/api.js +316 -0
- package/lib/methods/base.js +184 -0
- package/lib/methods/content-separator.js +45 -0
- package/lib/methods/deepl.js +426 -0
- package/lib/methods/direct-llm.js +586 -0
- package/lib/methods/external.js +332 -0
- package/lib/methods/fetch-with-retry.js +124 -0
- package/lib/methods/gemini.js +147 -0
- package/lib/methods/google-translate.js +402 -0
- package/lib/methods/http-utils.js +122 -0
- package/lib/methods/libretranslate.js +314 -0
- package/lib/methods/llm-coached.js +670 -0
- package/lib/methods/llm.js +592 -0
- package/lib/methods/local.js +76 -0
- package/lib/methods/microsoft-translator.js +331 -0
- package/lib/methods/openai.js +131 -0
- package/lib/methods/openrouter-client.js +327 -0
- package/lib/methods/openrouter-pricing.js +156 -0
- package/lib/methods/provider-env.js +115 -0
- package/lib/methods/provider-pricing.js +310 -0
- package/lib/methods/tilde.js +150 -0
- package/lib/methods/translated.js +229 -0
- package/lib/methods/translation-error.js +80 -0
- package/lib/models.js +258 -0
- package/lib/no-translate.js +233 -0
- package/lib/output.js +238 -0
- package/lib/pairs.js +547 -0
- package/lib/plugins.js +447 -0
- package/lib/provenance.js +323 -0
- package/lib/recommend.js +648 -0
- package/lib/registers.js +1185 -0
- package/lib/repair-script.js +266 -0
- package/lib/scripts.js +994 -0
- package/lib/seal.mjs +464 -0
- package/lib/sealed-qualifier.mjs +211 -0
- package/lib/security.js +59 -0
- package/lib/segment.js +369 -0
- package/lib/seo.js +275 -0
- package/lib/serve.js +854 -0
- package/lib/string-classify.js +85 -0
- package/lib/submit.mjs +344 -0
- package/lib/sync.js +969 -0
- package/lib/tags/bcp47.js +202 -0
- package/lib/tags/resolve.js +314 -0
- package/lib/terminology.js +111 -0
- package/lib/tm-seed.js +294 -0
- package/lib/tm.js +515 -0
- package/lib/translate-pair.js +197 -0
- package/lib/translate.js +203 -0
- package/lib/types.js +230 -0
- package/lib/validate.js +510 -0
- package/lib/verify.js +451 -0
- package/lib/watch.js +145 -0
- package/lib/xliff.js +184 -0
- package/package.json +93 -0
- package/shared/ATTRIBUTION.md +145 -0
- package/shared/CORPORA-CARDS.md +288 -0
- package/shared/DATA-SOVEREIGNTY.md +500 -0
- package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
- package/shared/card-lint-baseline.json +3189 -0
- package/shared/cards-fallback.json +1 -0
- package/shared/catalogue/card-config.json +6091 -0
- package/shared/catalogue/external-results.json +3888 -0
- package/shared/catalogue/gender-guidance.json +1038 -0
- package/shared/catalogue/method-coverage.json +1751 -0
- package/shared/catalogue/metric-coverage.json +170 -0
- package/shared/catalogue/metric-reliability.json +1 -0
- package/shared/catalogue/register-presets.json +3180 -0
- package/shared/catalogue/vitality-scales.json +55 -0
- package/shared/cldr-index.json +1115 -0
- package/shared/code-bridge.json +253 -0
- package/shared/corpora-cards-v1-reference.md +281 -0
- package/shared/curated-dictionary-flags.json +35 -0
- package/shared/curated-endonyms.json +35 -0
- package/shared/curated-fsts.json +51 -0
- package/shared/curated-orthography-conventions.json +26 -0
- package/shared/curated-sil-resources.json +374 -0
- package/shared/curated-tools.json +41 -0
- package/shared/docent/corpus.json +11333 -0
- package/shared/docent/faq.en.json +564 -0
- package/shared/docent/register-blocks.json +60 -0
- package/shared/docent/system-prompt.md +144 -0
- package/shared/domain-taxonomy.json +35 -0
- package/shared/explainers/glossary.json +2975 -0
- package/shared/explainers/tc-features.json +20112 -0
- package/shared/explainers/term-watchlist.json +147 -0
- package/shared/human-services.json +59 -0
- package/shared/license-corrections.json +261 -0
- package/shared/license-evidence.json +13452 -0
- package/shared/licenses.json +6781 -0
- package/shared/method-registry.json +236 -0
- package/shared/metric-registry.json +620 -0
- package/shared/model-aliases.json +7 -0
- package/shared/schemas/champollion-plugin.schema.json +206 -0
- package/shared/schemas/corpora-card.schema.json +957 -0
- package/shared/schemas/domain-taxonomy.schema.json +64 -0
- package/shared/schemas/external-results.schema.json +314 -0
- package/shared/schemas/human-services.schema.json +90 -0
- package/shared/schemas/language-card.schema.json +1308 -0
- package/shared/schemas/licenses.schema.json +155 -0
- package/shared/schemas/method-card.schema.json +412 -0
- package/shared/schemas/method-registry.schema.json +85 -0
- package/shared/schemas/metric-registry.schema.json +96 -0
- package/shared/schemas/metric-reliability.schema.json +178 -0
- package/shared/schemas/model-aliases.schema.json +27 -0
- package/shared/schemas/source-snapshot.schema.json +96 -0
|
@@ -0,0 +1,678 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Command: register-corpus
|
|
3
|
+
*
|
|
4
|
+
* Register a new evaluation corpus, with the author in control of BOTH its
|
|
5
|
+
* license and how far it travels. Three exposure tiers, defaulting to the most
|
|
6
|
+
* private:
|
|
7
|
+
*
|
|
8
|
+
* 1. local-only — never registered, never uploaded; the card + your text stay
|
|
9
|
+
* entirely on your machine.
|
|
10
|
+
* 2. private — a WMT-style sovereign held-out / secret test set: register
|
|
11
|
+
* METADATA ONLY; your text is never uploaded or hosted; you keep custody.
|
|
12
|
+
* 3. public — publish a metadata card + a fetch-from-source pointer; your
|
|
13
|
+
* text is never hosted by us. Gated by cli/lib/license-gate.mjs so NC /
|
|
14
|
+
* no-redistribute / unconfirmed licenses cannot enter the public lane.
|
|
15
|
+
*
|
|
16
|
+
* We NEVER read, upload, or host corpus content in ANY tier — only metadata and
|
|
17
|
+
* (for public) a pointer to where the data is fetched from. This command writes
|
|
18
|
+
* a metadata card; it never touches your sentences.
|
|
19
|
+
*
|
|
20
|
+
* Interactive wizard when stdin is a TTY; fully scriptable via flags otherwise
|
|
21
|
+
* (or with --yes). See `champollion register-corpus --help`.
|
|
22
|
+
*
|
|
23
|
+
* Exit codes: 0 = registered/saved (or cancelled); 1 = invalid request / gate block.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import fs from 'node:fs';
|
|
27
|
+
import path from 'node:path';
|
|
28
|
+
import readline from 'node:readline';
|
|
29
|
+
import { fileURLToPath } from 'node:url';
|
|
30
|
+
import { output } from '../output.js';
|
|
31
|
+
import {
|
|
32
|
+
LICENSE_OPTIONS,
|
|
33
|
+
EXPOSURE_TIERS,
|
|
34
|
+
DEFAULT_TIER,
|
|
35
|
+
resolveLicense,
|
|
36
|
+
resolveTier,
|
|
37
|
+
gatePublicRegistration,
|
|
38
|
+
buildCorpusCard,
|
|
39
|
+
deriveCardId,
|
|
40
|
+
slugify,
|
|
41
|
+
resolveDestination,
|
|
42
|
+
validateRegistration,
|
|
43
|
+
} from '../corpus-registration.mjs';
|
|
44
|
+
import {
|
|
45
|
+
sealPlaintext,
|
|
46
|
+
buildSealedArtifact,
|
|
47
|
+
buildSealedCardBlock,
|
|
48
|
+
resolveThresholdPublicKey,
|
|
49
|
+
buildAad,
|
|
50
|
+
} from '../seal.mjs';
|
|
51
|
+
import { DEFAULT_QUALIFIER_THRESHOLD } from '../sealed-qualifier.mjs';
|
|
52
|
+
|
|
53
|
+
// Tracked corpora-cards SSOT, resolved relative to this module.
|
|
54
|
+
const DEFAULT_CARDS_DIR = fileURLToPath(new URL('../../shared/corpora-cards/', import.meta.url));
|
|
55
|
+
|
|
56
|
+
const CONTAMINATION_LEVELS = ['NONE', 'LOW', 'MEDIUM', 'HIGH'];
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Resolve a path to its REAL location, following symlinks. The destination
|
|
60
|
+
* directory may not exist yet, so we realpath the deepest existing ancestor
|
|
61
|
+
* and re-append the remaining (non-existent, therefore non-symlink) segments.
|
|
62
|
+
* Unlike path.resolve, this defeats a symlinked ancestor that points elsewhere.
|
|
63
|
+
*
|
|
64
|
+
* @param {string} p - Path to resolve (absolute or relative to cwd)
|
|
65
|
+
* @returns {string} Real, symlink-free absolute path
|
|
66
|
+
*/
|
|
67
|
+
function realPathDeep(p) {
|
|
68
|
+
let cur = path.resolve(p);
|
|
69
|
+
const tail = [];
|
|
70
|
+
while (!fs.existsSync(cur)) {
|
|
71
|
+
tail.unshift(path.basename(cur));
|
|
72
|
+
const parent = path.dirname(cur);
|
|
73
|
+
if (parent === cur) return path.resolve(p); // reached root, nothing exists
|
|
74
|
+
cur = parent;
|
|
75
|
+
}
|
|
76
|
+
let real;
|
|
77
|
+
try { real = fs.realpathSync(cur); } catch { return path.resolve(p); }
|
|
78
|
+
return tail.length ? path.join(real, ...tail) : real;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* True if `child` is `parent` itself or nested anywhere under it.
|
|
83
|
+
* Both arguments must already be real (symlink-resolved) paths.
|
|
84
|
+
*/
|
|
85
|
+
function isWithin(child, parent) {
|
|
86
|
+
const rel = path.relative(parent, child);
|
|
87
|
+
return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function isInteractive() {
|
|
91
|
+
return process.stdin.isTTY === true;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function ask(rl, question, defaultValue) {
|
|
95
|
+
return new Promise((resolve) => {
|
|
96
|
+
const suffix = defaultValue ? ` (${defaultValue})` : '';
|
|
97
|
+
rl.question(` ${question}${suffix}: `, (answer) => {
|
|
98
|
+
resolve(answer.trim() || defaultValue || '');
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Parse "src>tgt" / "src-tgt" / "src,tgt" into {source, target}, else null. */
|
|
104
|
+
function parsePair(value) {
|
|
105
|
+
if (!value) return null;
|
|
106
|
+
const m = String(value).trim().split(/[>\-,\s]+/).filter(Boolean);
|
|
107
|
+
if (m.length >= 2) return { source: m[0].toLowerCase(), target: m[1].toLowerCase() };
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// ── Catalog printing (plain language for humans, machine-readable for agents) ──
|
|
112
|
+
|
|
113
|
+
function printCatalogs(asJson) {
|
|
114
|
+
if (asJson) {
|
|
115
|
+
console.log(JSON.stringify({
|
|
116
|
+
licenses: LICENSE_OPTIONS.map((o) => ({ key: o.key, spdx: o.spdx, commercial: o.commercial, redistribution: o.redistribution, publicEligible: o.publicEligible, explanation: o.explanation })),
|
|
117
|
+
tiers: EXPOSURE_TIERS.map((t) => ({ key: t.key, registers: t.registers, uploadsContent: t.uploadsContent, uploadsCiphertext: !!t.uploadsCiphertext, explanation: t.explanation })),
|
|
118
|
+
default_tier: DEFAULT_TIER,
|
|
119
|
+
}, null, 2));
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
console.log('');
|
|
123
|
+
console.log(' LICENSES — pick the terms others may use your corpus under:');
|
|
124
|
+
console.log('');
|
|
125
|
+
LICENSE_OPTIONS.forEach((o, i) => {
|
|
126
|
+
console.log(` ${i + 1}. ${o.label}`);
|
|
127
|
+
console.log(` ${o.explanation}`);
|
|
128
|
+
console.log(` commercial: ${o.commercial} · redistribution: ${o.redistribution} · public-lane: ${o.publicEligible ? 'eligible' : 'blocked'}`);
|
|
129
|
+
});
|
|
130
|
+
console.log('');
|
|
131
|
+
console.log(' EXPOSURE TIERS — how far your corpus travels (default: most private):');
|
|
132
|
+
console.log('');
|
|
133
|
+
EXPOSURE_TIERS.forEach((t, i) => {
|
|
134
|
+
const def = t.key === DEFAULT_TIER ? ' ★ default' : '';
|
|
135
|
+
console.log(` ${i + 1}. ${t.label}${def}`);
|
|
136
|
+
console.log(` ${t.explanation}`);
|
|
137
|
+
});
|
|
138
|
+
console.log('');
|
|
139
|
+
console.log(' Champollion never reads, uploads, or hosts your corpus text — in ANY tier.');
|
|
140
|
+
console.log('');
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ── Interactive wizard ──────────────────────────────────────────────────────
|
|
144
|
+
|
|
145
|
+
async function runInteractive(rl, args) {
|
|
146
|
+
console.log('');
|
|
147
|
+
console.log(' champollion — Register a Corpus');
|
|
148
|
+
console.log(' ════════════════════════════════════════════════');
|
|
149
|
+
console.log('');
|
|
150
|
+
console.log(' You choose the license and how far this corpus travels.');
|
|
151
|
+
console.log(' We never read, upload, or host your text — only metadata,');
|
|
152
|
+
console.log(' and (for public sets) a pointer to where the data is fetched from.');
|
|
153
|
+
|
|
154
|
+
// Step 1 — basics
|
|
155
|
+
console.log('');
|
|
156
|
+
console.log(' Step 1/4 — Basics');
|
|
157
|
+
console.log(' ────────────────────────────────────────────────');
|
|
158
|
+
const name = await ask(rl, 'Corpus name', args.name || '');
|
|
159
|
+
const pairStr = await ask(rl, 'Language pair (e.g. eng-crk, or "eng>crk")', args.pair || '');
|
|
160
|
+
const pair = parsePair(pairStr);
|
|
161
|
+
const publisher = await ask(rl, 'Publisher / your name or org', args.publisher || '');
|
|
162
|
+
const description = await ask(rl, 'One-line description', args.description || `${name} evaluation corpus`);
|
|
163
|
+
|
|
164
|
+
// Step 2 — license
|
|
165
|
+
console.log('');
|
|
166
|
+
console.log(' Step 2/4 — License');
|
|
167
|
+
console.log(' ────────────────────────────────────────────────');
|
|
168
|
+
console.log('');
|
|
169
|
+
LICENSE_OPTIONS.forEach((o, i) => {
|
|
170
|
+
console.log(` ${i + 1}. ${o.label} — ${o.explanation}`);
|
|
171
|
+
});
|
|
172
|
+
console.log('');
|
|
173
|
+
const licChoice = await ask(rl, 'Choose a license', '1');
|
|
174
|
+
const licenseOption = resolveLicense(licChoice) || LICENSE_OPTIONS[0];
|
|
175
|
+
console.log(` → ${licenseOption.label} (commercial: ${licenseOption.commercial}, redistribution: ${licenseOption.redistribution})`);
|
|
176
|
+
|
|
177
|
+
// Step 3 — exposure tier (default most private)
|
|
178
|
+
console.log('');
|
|
179
|
+
console.log(' Step 3/4 — Exposure');
|
|
180
|
+
console.log(' ────────────────────────────────────────────────');
|
|
181
|
+
console.log('');
|
|
182
|
+
EXPOSURE_TIERS.forEach((t, i) => {
|
|
183
|
+
const def = t.key === DEFAULT_TIER ? ' ★ safe default' : '';
|
|
184
|
+
console.log(` ${i + 1}. ${t.label}${def}`);
|
|
185
|
+
console.log(` ${t.explanation}`);
|
|
186
|
+
});
|
|
187
|
+
console.log('');
|
|
188
|
+
const tierChoice = await ask(rl, 'Choose exposure', '1');
|
|
189
|
+
let tier = resolveTier(tierChoice);
|
|
190
|
+
|
|
191
|
+
// Gate the public tier through the license gate, with a graceful fallback.
|
|
192
|
+
if (tier.key === 'public') {
|
|
193
|
+
const gate = gatePublicRegistration(licenseOption);
|
|
194
|
+
if (!gate.allowed) {
|
|
195
|
+
console.log('');
|
|
196
|
+
console.log(` ⚠ ${gate.reason}`);
|
|
197
|
+
const fallback = await ask(rl, 'Register privately instead (metadata only, text never uploaded)?', 'yes');
|
|
198
|
+
if (fallback.toLowerCase().startsWith('y')) {
|
|
199
|
+
tier = resolveTier('private');
|
|
200
|
+
console.log(' → Switched to private (sovereign held-out) registration.');
|
|
201
|
+
} else {
|
|
202
|
+
return { cancelled: true, reason: gate.reason };
|
|
203
|
+
}
|
|
204
|
+
} else {
|
|
205
|
+
console.log(` → ${gate.reason}`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Public tier needs a fetch-from-source pointer (we host nothing).
|
|
210
|
+
let repoUrl = args['repo-url'] || null;
|
|
211
|
+
let builder = args.builder || null;
|
|
212
|
+
let sourceUrl = args['source-url'] || null;
|
|
213
|
+
if (tier.key === 'public') {
|
|
214
|
+
console.log('');
|
|
215
|
+
console.log(' Public sets are fetched from their source — give us a pointer, not the data:');
|
|
216
|
+
sourceUrl = await ask(rl, 'Canonical URL (project/dataset page)', sourceUrl || '');
|
|
217
|
+
repoUrl = await ask(rl, 'Fetch-from-source archive/repo URL', repoUrl || '');
|
|
218
|
+
builder = await ask(rl, 'Builder adapter id (rebuilds from source, e.g. tatoeba-challenge)', builder || '');
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Sealed tier — encrypt on this device, hand us ciphertext + a content-free card.
|
|
222
|
+
let custodianGroupId = args['custodian-group'] || null;
|
|
223
|
+
let thresholdPubkey = args['threshold-pubkey'] || null;
|
|
224
|
+
let sealInput = args['seal-input'] || null;
|
|
225
|
+
let sealOut = args['seal-out'] || null;
|
|
226
|
+
let qualifierId = args['qualifier-id'] || null;
|
|
227
|
+
let qualifierThreshold = args['qualifier-threshold'] != null ? Number(args['qualifier-threshold']) : null;
|
|
228
|
+
const keyScheme = args['key-scheme'] || null;
|
|
229
|
+
if (tier.key === 'sealed') {
|
|
230
|
+
console.log('');
|
|
231
|
+
console.log(' Sealed sets are encrypted ON THIS DEVICE before anything leaves.');
|
|
232
|
+
console.log(' We receive ciphertext + a content-free card only — we cannot read it.');
|
|
233
|
+
console.log('');
|
|
234
|
+
sealInput = await ask(rl, 'Path to the corpus file to seal (your plaintext, stays local)', sealInput || '');
|
|
235
|
+
thresholdPubkey = await ask(rl, 'Custodian group threshold public key (path or PEM/base64)', thresholdPubkey || '');
|
|
236
|
+
custodianGroupId = await ask(rl, 'Custodian group id (the community group that holds the key)', custodianGroupId || '');
|
|
237
|
+
sealOut = await ask(rl, 'Where to write the encrypted artifact', sealOut || '');
|
|
238
|
+
console.log('');
|
|
239
|
+
console.log(' A sealed set must be paired with a PUBLIC qualifier (a disjoint CC-BY/CC0');
|
|
240
|
+
console.log(' twin) that a method must clear before any sealed run can be proposed.');
|
|
241
|
+
qualifierId = await ask(rl, 'Paired public qualifier card id (e.g. eval-eng-crk-…-qualifier-v2026)', qualifierId || '');
|
|
242
|
+
const qt = await ask(rl, 'Qualifier clearance threshold', String(qualifierThreshold ?? DEFAULT_QUALIFIER_THRESHOLD));
|
|
243
|
+
qualifierThreshold = Number(qt);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Step 4 — consumer-reports metadata
|
|
247
|
+
console.log('');
|
|
248
|
+
console.log(' Step 4/4 — Dataset details (the "consumer report")');
|
|
249
|
+
console.log(' ────────────────────────────────────────────────');
|
|
250
|
+
const sizeStr = await ask(rl, 'Size — number of sentence pairs', args.size || '');
|
|
251
|
+
const domain = await ask(rl, 'Domain (news, conversational, educational, …)', args.domain || 'mixed');
|
|
252
|
+
const defaultRisk = tier.key === 'public' ? 'LOW' : 'NONE';
|
|
253
|
+
console.log('');
|
|
254
|
+
console.log(' Contamination risk — is this text in known LLM training sets?');
|
|
255
|
+
console.log(' NONE = private/unpublished · LOW = niche/recent · MEDIUM = public · HIGH = in major training sets');
|
|
256
|
+
const riskIn = await ask(rl, 'Contamination risk', defaultRisk);
|
|
257
|
+
const contaminationRisk = CONTAMINATION_LEVELS.includes(riskIn.toUpperCase()) ? riskIn.toUpperCase() : defaultRisk;
|
|
258
|
+
|
|
259
|
+
return {
|
|
260
|
+
name, pair, publisher, description,
|
|
261
|
+
licenseOption, tier,
|
|
262
|
+
repoUrl, builder, sourceUrl,
|
|
263
|
+
size: Number(sizeStr), domain, contaminationRisk,
|
|
264
|
+
custodianGroupId, thresholdPubkey, sealInput, sealOut,
|
|
265
|
+
qualifierId, qualifierThreshold, keyScheme,
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// ── Non-interactive (flags) ─────────────────────────────────────────────────
|
|
270
|
+
|
|
271
|
+
function fromFlags(args) {
|
|
272
|
+
const pair = parsePair(args.pair) ||
|
|
273
|
+
(args['source-lang'] && args['target-lang']
|
|
274
|
+
? { source: String(args['source-lang']).toLowerCase(), target: String(args['target-lang']).toLowerCase() }
|
|
275
|
+
: null);
|
|
276
|
+
const licenseOption = resolveLicense(args.license);
|
|
277
|
+
const tier = resolveTier(args.tier || args.exposure);
|
|
278
|
+
const riskIn = args.contamination ? String(args.contamination).toUpperCase() : null;
|
|
279
|
+
return {
|
|
280
|
+
name: args.name || '',
|
|
281
|
+
pair,
|
|
282
|
+
publisher: args.publisher || (args.name ? '' : ''),
|
|
283
|
+
description: args.description || (args.name ? `${args.name} evaluation corpus` : ''),
|
|
284
|
+
licenseOption,
|
|
285
|
+
tier,
|
|
286
|
+
repoUrl: args['repo-url'] || null,
|
|
287
|
+
builder: args.builder || null,
|
|
288
|
+
sourceUrl: args['source-url'] || null,
|
|
289
|
+
size: args.size != null ? Number(args.size) : NaN,
|
|
290
|
+
domain: args.domain || '',
|
|
291
|
+
contaminationRisk: riskIn && CONTAMINATION_LEVELS.includes(riskIn) ? riskIn : null,
|
|
292
|
+
// sealed tier
|
|
293
|
+
custodianGroupId: args['custodian-group'] || null,
|
|
294
|
+
thresholdPubkey: args['threshold-pubkey'] || null,
|
|
295
|
+
sealInput: args['seal-input'] || null,
|
|
296
|
+
sealOut: args['seal-out'] || null,
|
|
297
|
+
qualifierId: args['qualifier-id'] || null,
|
|
298
|
+
qualifierThreshold: args['qualifier-threshold'] != null ? Number(args['qualifier-threshold']) : null,
|
|
299
|
+
keyScheme: args['key-scheme'] || null,
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// ── Sealed tier: client-side encryption ──────────────────────────────────────
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Seal a corpus on the author's device: read the local plaintext, encrypt it to
|
|
307
|
+
* the custodian group's threshold PUBLIC key, write a ciphertext-ONLY artifact,
|
|
308
|
+
* and return the content-free `sealed` card block. The plaintext never leaves
|
|
309
|
+
* this machine in readable form; we hold only ciphertext + metadata, and we
|
|
310
|
+
* cannot decrypt it (no single party can — see docs/governance/OCAP_MULTISIG_PLAN.md).
|
|
311
|
+
*
|
|
312
|
+
* @returns {{cardBlock:object, artifactPath:string}|{error:string}}
|
|
313
|
+
*/
|
|
314
|
+
function sealCorpusForRegistration({ req, id, addedAt, cwd }) {
|
|
315
|
+
// 1. Read the author's plaintext corpus (stays on this device).
|
|
316
|
+
let plaintext;
|
|
317
|
+
try {
|
|
318
|
+
plaintext = fs.readFileSync(path.resolve(cwd, req.sealInput));
|
|
319
|
+
} catch (e) {
|
|
320
|
+
return { error: `cannot read the corpus file (--seal-input ${req.sealInput}): ${e.message}` };
|
|
321
|
+
}
|
|
322
|
+
if (!plaintext.length) return { error: `the corpus file (--seal-input ${req.sealInput}) is empty — nothing to seal.` };
|
|
323
|
+
|
|
324
|
+
// 2. Resolve the custodian group's threshold PUBLIC key (a path or an inline
|
|
325
|
+
// PEM/base64 value). WAVE-2 SEAM: this becomes the aggregated 3-of-5 FROST
|
|
326
|
+
// group key from the custodian key ceremony — same call site, same format.
|
|
327
|
+
let thresholdKeyInput = req.thresholdPubkey;
|
|
328
|
+
let keyAsPath = null;
|
|
329
|
+
try { keyAsPath = path.resolve(cwd, String(req.thresholdPubkey)); } catch { keyAsPath = null; }
|
|
330
|
+
if (keyAsPath && fs.existsSync(keyAsPath) && fs.statSync(keyAsPath).isFile()) {
|
|
331
|
+
try { thresholdKeyInput = fs.readFileSync(keyAsPath, 'utf-8').trim(); }
|
|
332
|
+
catch (e) { return { error: `cannot read the threshold public key file: ${e.message}` }; }
|
|
333
|
+
}
|
|
334
|
+
let thresholdPublicKey;
|
|
335
|
+
try { thresholdPublicKey = resolveThresholdPublicKey(thresholdKeyInput); }
|
|
336
|
+
catch (e) { return { error: e.message }; }
|
|
337
|
+
|
|
338
|
+
// 3. Encrypt CLIENT-SIDE. The AAD binds the ciphertext to this card + group.
|
|
339
|
+
const aad = buildAad({ cardId: id, custodianGroupId: req.custodianGroupId });
|
|
340
|
+
let sealed;
|
|
341
|
+
try { sealed = sealPlaintext({ plaintext, thresholdPublicKey, aad }); }
|
|
342
|
+
catch (e) { return { error: e.message }; }
|
|
343
|
+
// Drop the plaintext reference as soon as it is sealed.
|
|
344
|
+
plaintext = null;
|
|
345
|
+
|
|
346
|
+
// 4. Write the ciphertext-ONLY artifact (the off-git encrypted store stand-in).
|
|
347
|
+
const artifactPath = req.sealOut
|
|
348
|
+
? path.resolve(cwd, req.sealOut)
|
|
349
|
+
: path.resolve(cwd, `${id}.sealed.json`);
|
|
350
|
+
// The ciphertext artifact must NEVER land in the tracked corpora-cards SSOT.
|
|
351
|
+
const ssotReal = realPathDeep(DEFAULT_CARDS_DIR);
|
|
352
|
+
if (isWithin(realPathDeep(artifactPath), ssotReal)) {
|
|
353
|
+
return { error: 'the sealed (ciphertext) artifact must not be written into the tracked corpora-cards directory — choose a different --seal-out path.' };
|
|
354
|
+
}
|
|
355
|
+
const artifact = buildSealedArtifact({
|
|
356
|
+
sealed, cardId: id, custodianGroupId: req.custodianGroupId, createdAt: addedAt,
|
|
357
|
+
});
|
|
358
|
+
try {
|
|
359
|
+
fs.mkdirSync(path.dirname(artifactPath), { recursive: true });
|
|
360
|
+
fs.writeFileSync(artifactPath, JSON.stringify(artifact, null, 2) + '\n', 'utf-8');
|
|
361
|
+
} catch (e) {
|
|
362
|
+
return { error: `cannot write the sealed artifact to ${artifactPath}: ${e.message}` };
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// 5. The content-free `sealed` block for the card (cipher / group / digest /
|
|
366
|
+
// AAD + key + paired-qualifier references).
|
|
367
|
+
const qt = req.qualifierThreshold != null && !Number.isNaN(req.qualifierThreshold)
|
|
368
|
+
? req.qualifierThreshold
|
|
369
|
+
: DEFAULT_QUALIFIER_THRESHOLD;
|
|
370
|
+
const cardBlock = buildSealedCardBlock({
|
|
371
|
+
sealed,
|
|
372
|
+
custodianGroupId: req.custodianGroupId,
|
|
373
|
+
keyScheme: req.keyScheme || 'TSS-3-of-5',
|
|
374
|
+
qualifierId: req.qualifierId,
|
|
375
|
+
qualifierThreshold: qt,
|
|
376
|
+
artifactRef: path.basename(artifactPath),
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
return { cardBlock, artifactPath };
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* TODO (Wave 2 — web wizard): the browser registration wizard at
|
|
384
|
+
* champollion.dev will offer the same "Seal it" choice, doing the IDENTICAL
|
|
385
|
+
* client-side encryption in-browser (WebCrypto: X25519 ECDH → HKDF → AES-GCM)
|
|
386
|
+
* so the plaintext never leaves the user's browser either. It will POST only the
|
|
387
|
+
* ciphertext artifact + the content-free card produced here. The CLI path in
|
|
388
|
+
* this file is the Wave-1 deliverable and the reference implementation; the web
|
|
389
|
+
* wizard must produce a byte-compatible artifact (same SEAL_CIPHER/SEAL_VERSION
|
|
390
|
+
* envelope in lib/seal.mjs). Not implemented yet — the CLI path below is the
|
|
391
|
+
* only seal path today.
|
|
392
|
+
*/
|
|
393
|
+
|
|
394
|
+
// ── Main ────────────────────────────────────────────────────────────────────
|
|
395
|
+
|
|
396
|
+
async function run(args, cwd) {
|
|
397
|
+
if (args.help) {
|
|
398
|
+
showHelp();
|
|
399
|
+
return 0;
|
|
400
|
+
}
|
|
401
|
+
if (args.list) {
|
|
402
|
+
printCatalogs(!!args.json);
|
|
403
|
+
return 0;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// Collect the registration request.
|
|
407
|
+
let req;
|
|
408
|
+
if (!args.yes && isInteractive()) {
|
|
409
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
410
|
+
try {
|
|
411
|
+
req = await runInteractive(rl, args);
|
|
412
|
+
} finally {
|
|
413
|
+
rl.close();
|
|
414
|
+
}
|
|
415
|
+
if (req && req.cancelled) {
|
|
416
|
+
console.log('');
|
|
417
|
+
output.warn('Registration cancelled — nothing was written.');
|
|
418
|
+
return 0;
|
|
419
|
+
}
|
|
420
|
+
} else {
|
|
421
|
+
req = fromFlags(args);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// Validate (collects every problem, fails loud).
|
|
425
|
+
const v = validateRegistration({
|
|
426
|
+
tier: req.tier,
|
|
427
|
+
licenseOption: req.licenseOption,
|
|
428
|
+
pair: req.pair || {},
|
|
429
|
+
name: req.name,
|
|
430
|
+
repoUrl: req.repoUrl,
|
|
431
|
+
builder: req.builder,
|
|
432
|
+
size: req.size,
|
|
433
|
+
domain: req.domain,
|
|
434
|
+
custodianGroupId: req.custodianGroupId,
|
|
435
|
+
thresholdPublicKey: req.thresholdPubkey,
|
|
436
|
+
sealInput: req.sealInput,
|
|
437
|
+
qualifierId: req.qualifierId,
|
|
438
|
+
});
|
|
439
|
+
if (!v.ok) {
|
|
440
|
+
console.error('[ERR] Cannot register this corpus:');
|
|
441
|
+
for (const e of v.errors) console.error(` • ${e}`);
|
|
442
|
+
console.error('');
|
|
443
|
+
console.error(' Run "champollion register-corpus --list" to see licenses + tiers,');
|
|
444
|
+
console.error(' or "champollion register-corpus --help" for all flags.');
|
|
445
|
+
return 1;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Build the (content-free) card.
|
|
449
|
+
const version = args.version || '1';
|
|
450
|
+
const slug = slugify(args.id || req.publisher || req.name || 'custom');
|
|
451
|
+
const id = args.id && /^(ref|eval)-/.test(args.id)
|
|
452
|
+
? args.id
|
|
453
|
+
: deriveCardId({ source: req.pair.source, target: req.pair.target, slug, version });
|
|
454
|
+
|
|
455
|
+
const addedAt = new Date().toISOString().slice(0, 10);
|
|
456
|
+
|
|
457
|
+
// ── SEALED TIER: encrypt on THIS device before anything leaves ──────────────
|
|
458
|
+
// We compute the card id first (the AAD binds the ciphertext to it), read the
|
|
459
|
+
// author's plaintext locally, encrypt it to the custodian group's threshold
|
|
460
|
+
// public key, write a ciphertext-ONLY artifact, and keep only a content-free
|
|
461
|
+
// sealed block for the card. The plaintext is never uploaded, hosted, or kept.
|
|
462
|
+
let sealedBlock = null;
|
|
463
|
+
let sealedArtifactPath = null;
|
|
464
|
+
if (req.tier.key === 'sealed') {
|
|
465
|
+
const sealed = sealCorpusForRegistration({ req, id, addedAt, cwd });
|
|
466
|
+
if (sealed.error) {
|
|
467
|
+
console.error(`[ERR] Could not seal this corpus: ${sealed.error}`);
|
|
468
|
+
console.error(' Nothing was written. Your corpus text never left your machine.');
|
|
469
|
+
return 1;
|
|
470
|
+
}
|
|
471
|
+
sealedBlock = sealed.cardBlock;
|
|
472
|
+
sealedArtifactPath = sealed.artifactPath;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
const card = buildCorpusCard({
|
|
476
|
+
id,
|
|
477
|
+
name: req.name,
|
|
478
|
+
version: /^\d/.test(version) ? `${version}.0.0`.split('.').slice(0, 3).join('.') : '0.1.0',
|
|
479
|
+
description: req.description || `${req.name} evaluation corpus`,
|
|
480
|
+
pair: req.pair,
|
|
481
|
+
publisher: req.publisher || 'Unattributed (author-registered)',
|
|
482
|
+
sourceUrl: req.sourceUrl,
|
|
483
|
+
repoUrl: req.repoUrl,
|
|
484
|
+
builder: req.builder,
|
|
485
|
+
sha256: args.sha256 || null,
|
|
486
|
+
licenseUrl: args['license-url'] || null,
|
|
487
|
+
licenseOption: req.licenseOption,
|
|
488
|
+
tier: req.tier,
|
|
489
|
+
contaminationRisk: req.contaminationRisk,
|
|
490
|
+
size: req.size,
|
|
491
|
+
domain: req.domain,
|
|
492
|
+
doNotTrain: args['do-not-train'] !== false,
|
|
493
|
+
addedAt,
|
|
494
|
+
sealed: sealedBlock,
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
// Resolve destination — local-only NEVER lands in the tracked SSOT.
|
|
498
|
+
const corporaCardsDir = args['cards-dir'] ? path.resolve(cwd, args['cards-dir']) : DEFAULT_CARDS_DIR;
|
|
499
|
+
const localDir = args.out ? path.resolve(cwd, args.out) : cwd;
|
|
500
|
+
const dest = resolveDestination({ tier: req.tier, id, corporaCardsDir, localDir });
|
|
501
|
+
|
|
502
|
+
// Guard: a local-only card must never land in the tracked corpora-cards SSOT.
|
|
503
|
+
// path.resolve does NOT follow symlinks and only the exact dir was checked —
|
|
504
|
+
// a symlinked --out (or one nested under the SSOT) slipped past. Resolve real
|
|
505
|
+
// paths and reject anything that IS the SSOT or sits under it. The never-host
|
|
506
|
+
// doctrine is absolute, so this guard must be too.
|
|
507
|
+
const ssotReal = realPathDeep(DEFAULT_CARDS_DIR);
|
|
508
|
+
const targetReal = realPathDeep(dest.dir);
|
|
509
|
+
if (req.tier.key === 'local-only' && isWithin(targetReal, ssotReal)) {
|
|
510
|
+
console.error('[ERR] local-only corpora must not be written into the tracked corpora-cards directory.');
|
|
511
|
+
console.error(' (Symlinks are resolved — a link that points into the SSOT is rejected too.)');
|
|
512
|
+
console.error(' Choose a different --out path, or use --tier private to register metadata.');
|
|
513
|
+
return 1;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const filePath = path.join(dest.dir, dest.filename);
|
|
517
|
+
if (fs.existsSync(filePath)) {
|
|
518
|
+
console.error(`[ERR] A card already exists at ${filePath}`);
|
|
519
|
+
console.error(' Pick a different --id, or remove the existing card first.');
|
|
520
|
+
return 1;
|
|
521
|
+
}
|
|
522
|
+
fs.mkdirSync(dest.dir, { recursive: true });
|
|
523
|
+
fs.writeFileSync(filePath, JSON.stringify(card, null, 2) + '\n', 'utf-8');
|
|
524
|
+
|
|
525
|
+
// ── Report ──
|
|
526
|
+
if (args.json) {
|
|
527
|
+
console.log(JSON.stringify({
|
|
528
|
+
ok: true,
|
|
529
|
+
id,
|
|
530
|
+
exposureTier: req.tier.key,
|
|
531
|
+
registered: dest.registered,
|
|
532
|
+
tracked: dest.tracked,
|
|
533
|
+
uploadedContent: false,
|
|
534
|
+
uploadedPlaintext: false,
|
|
535
|
+
...(req.tier.key === 'sealed' ? {
|
|
536
|
+
sealed: true,
|
|
537
|
+
sealedArtifactPath,
|
|
538
|
+
ciphertextDigest: sealedBlock.ciphertextDigest,
|
|
539
|
+
cipher: sealedBlock.cipher,
|
|
540
|
+
custodianGroupId: sealedBlock.custodianGroupId,
|
|
541
|
+
qualifierId: sealedBlock.qualifierId,
|
|
542
|
+
qualifierThreshold: sealedBlock.qualifierThreshold,
|
|
543
|
+
} : {}),
|
|
544
|
+
license: card.license,
|
|
545
|
+
path: filePath,
|
|
546
|
+
}, null, 2));
|
|
547
|
+
return 0;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
console.log('');
|
|
551
|
+
output.ok(`${req.tier.key === 'local-only' ? 'Saved' : 'Registered'} corpus card: ${id}`);
|
|
552
|
+
console.log('');
|
|
553
|
+
console.log(` Exposure: ${req.tier.label}`);
|
|
554
|
+
console.log(` License: ${card.license.spdx} (commercial: ${card.license.commercial}, redistribution: ${card.license.redistribution})`);
|
|
555
|
+
console.log(` Pair: ${req.pair.source}→${req.pair.target}`);
|
|
556
|
+
console.log(` Size/domain: ${req.size} ${card.dev.sizeUnit} · ${card.dev.domain}`);
|
|
557
|
+
console.log(` Contamination: ${card.contamination.risk}`);
|
|
558
|
+
console.log(` Written to: ${filePath}`);
|
|
559
|
+
console.log('');
|
|
560
|
+
|
|
561
|
+
if (req.tier.key === 'sealed') {
|
|
562
|
+
console.log(` Sealed artifact: ${sealedArtifactPath}`);
|
|
563
|
+
console.log(` Cipher: ${sealedBlock.cipher}`);
|
|
564
|
+
console.log(` Ciphertext digest: ${sealedBlock.ciphertextDigest}`);
|
|
565
|
+
console.log(` Custodian group: ${sealedBlock.custodianGroupId}`);
|
|
566
|
+
console.log(` Qualifier: ${sealedBlock.qualifierId} (clear ≥ ${sealedBlock.qualifierThreshold} first)`);
|
|
567
|
+
console.log('');
|
|
568
|
+
console.log(' ✓ Your sentences were SCRAMBLED LOCALLY on this machine before anything');
|
|
569
|
+
console.log(' was written. They are NEVER sent or stored in readable form. The only');
|
|
570
|
+
console.log(' artifact produced is ciphertext — Champollion cannot decrypt it, and no');
|
|
571
|
+
console.log(' single party can: it takes M-of-N custodian approval (the threshold key).');
|
|
572
|
+
console.log('');
|
|
573
|
+
console.log(' The card is content-free and quarantined from the public queue/leaderboard.');
|
|
574
|
+
console.log(' A method must first clear the paired PUBLIC qualifier before any sealed run');
|
|
575
|
+
console.log(' can even be proposed — and that run still requires custodian approval.');
|
|
576
|
+
} else if (req.tier.key === 'local-only') {
|
|
577
|
+
console.log(' This card stays on your machine. Nothing was registered or uploaded,');
|
|
578
|
+
console.log(' and your corpus text was never read. Delete the file to remove it.');
|
|
579
|
+
} else if (req.tier.key === 'private') {
|
|
580
|
+
console.log(' Metadata only is registered — your corpus TEXT was never read, uploaded,');
|
|
581
|
+
console.log(' or hosted, and never will be. The set is catalogued but quarantined from');
|
|
582
|
+
console.log(' the public queue/leaderboard (custodian-controlled, WMT-style held-out).');
|
|
583
|
+
console.log('');
|
|
584
|
+
console.log(' Next: rebuild the registry, then evaluate privately:');
|
|
585
|
+
console.log(' npm run generate:registries # from the repo root (arena)');
|
|
586
|
+
console.log(` mt-eval run --corpus ${id} --private # publish scores without exposing text`);
|
|
587
|
+
} else {
|
|
588
|
+
console.log(' A metadata card + fetch-from-source pointer is registered — your corpus');
|
|
589
|
+
console.log(' TEXT is never hosted by Champollion; it is fetched from source on demand.');
|
|
590
|
+
console.log('');
|
|
591
|
+
console.log(' Next: rebuild the registry so the pair enters the public queue:');
|
|
592
|
+
console.log(' npm run generate:registries # from the repo root (arena)');
|
|
593
|
+
}
|
|
594
|
+
console.log('');
|
|
595
|
+
return 0;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
function showHelp() {
|
|
599
|
+
console.log(`
|
|
600
|
+
champollion register-corpus — Register a new evaluation corpus
|
|
601
|
+
|
|
602
|
+
USAGE
|
|
603
|
+
champollion register-corpus [options]
|
|
604
|
+
|
|
605
|
+
DESCRIPTION
|
|
606
|
+
Register a corpus with YOU in control of its license and how far it
|
|
607
|
+
travels. Three exposure tiers, defaulting to the most private. We never
|
|
608
|
+
read, upload, or host your corpus text in ANY tier — only metadata (and,
|
|
609
|
+
for public sets, a pointer to where the data is fetched from).
|
|
610
|
+
|
|
611
|
+
Interactive when run in a terminal; fully scriptable with flags (or --yes).
|
|
612
|
+
|
|
613
|
+
EXPOSURE TIERS (--tier, default: local-only)
|
|
614
|
+
local-only Never registered, never uploaded; card + text stay on your machine.
|
|
615
|
+
private Register METADATA ONLY — a WMT-style sovereign held-out set;
|
|
616
|
+
text never uploaded or hosted; you keep custody.
|
|
617
|
+
public Publish a metadata card + fetch-from-source pointer; text never
|
|
618
|
+
hosted by us. Requires a redistribution-cleared license (gated).
|
|
619
|
+
sealed Encrypt the corpus ON YOUR DEVICE under the custodian group's
|
|
620
|
+
threshold key; we receive ciphertext + a content-free card only
|
|
621
|
+
and cannot decrypt it. Catalogued + quarantined; paired with a
|
|
622
|
+
public qualifier a method must clear before any sealed run.
|
|
623
|
+
|
|
624
|
+
LICENSES (--license)
|
|
625
|
+
cc-by-4.0, cc-by-sa-4.0, cc0-1.0, cc-by-nc-4.0, proprietary, other
|
|
626
|
+
(or pass any SPDX id; it is classified through the license gate)
|
|
627
|
+
|
|
628
|
+
OPTIONS
|
|
629
|
+
--tier <tier> Exposure tier (default: local-only). Alias: --exposure
|
|
630
|
+
--license <id> License key, SPDX id, or list number (see --list)
|
|
631
|
+
--name <text> Corpus name (required)
|
|
632
|
+
--pair <pair> Language pair: "eng>crk" (quote it — the shell eats a
|
|
633
|
+
bare >) or the dash form eng-crk (or --source-lang/--target-lang)
|
|
634
|
+
--publisher <text> Publisher / your name or org
|
|
635
|
+
--description <text> One-line description
|
|
636
|
+
--size <n> Number of sentence pairs (required)
|
|
637
|
+
--domain <text> Domain: news, conversational, educational, … (required)
|
|
638
|
+
--contamination <lvl> NONE | LOW | MEDIUM | HIGH (default: NONE private / LOW public)
|
|
639
|
+
--repo-url <url> Public tier: fetch-from-source archive/repo URL
|
|
640
|
+
--source-url <url> Public tier: canonical project/dataset URL
|
|
641
|
+
--builder <id> Public tier: builder adapter id (rebuilds from source)
|
|
642
|
+
--seal-input <path> Sealed tier: local corpus file to encrypt on-device
|
|
643
|
+
--threshold-pubkey <k> Sealed tier: custodian group threshold public key
|
|
644
|
+
(file path, or inline PEM / base64 DER SPKI, X25519)
|
|
645
|
+
--custodian-group <id> Sealed tier: custodian group id that holds the key
|
|
646
|
+
--seal-out <path> Sealed tier: where to write the ciphertext artifact
|
|
647
|
+
--qualifier-id <id> Sealed tier: paired public qualifier card id (vYYYY)
|
|
648
|
+
--qualifier-threshold <n> Sealed tier: score a method must clear on the qualifier
|
|
649
|
+
--key-scheme <s> Sealed tier: custody scheme label (default: TSS-3-of-5)
|
|
650
|
+
--id <id> Override the generated card id (eval-…)
|
|
651
|
+
--out <dir> local-only: where to write the card (default: cwd)
|
|
652
|
+
--list Print the license + tier catalog (add --json for JSON)
|
|
653
|
+
--json Machine-readable output (for agents)
|
|
654
|
+
--yes Non-interactive; take values from flags
|
|
655
|
+
|
|
656
|
+
EXAMPLES
|
|
657
|
+
champollion register-corpus # interactive wizard
|
|
658
|
+
champollion register-corpus --list # see licenses + tiers
|
|
659
|
+
# Keep a corpus entirely local (nothing leaves your machine):
|
|
660
|
+
champollion register-corpus --yes --name "My set" --pair "eng>crk" \\
|
|
661
|
+
--license cc-by-4.0 --tier local-only --size 200 --domain news
|
|
662
|
+
# Register a sovereign held-out test set (metadata only, text never uploaded):
|
|
663
|
+
champollion register-corpus --yes --name "Community holdout" --pair "eng>crk" \\
|
|
664
|
+
--license cc-by-nc-4.0 --tier private --size 500 --domain educational
|
|
665
|
+
# Publish a fetch-from-source pointer (gated to redistributable licenses):
|
|
666
|
+
champollion register-corpus --yes --name "Tatoeba eng-crk" --pair "eng-crk" \\
|
|
667
|
+
--license cc-by-4.0 --tier public --size 1000 --domain conversational \\
|
|
668
|
+
--repo-url https://example.org/data.tar --builder tatoeba-challenge
|
|
669
|
+
# Seal a community-controlled secret test set (encrypted ON YOUR DEVICE):
|
|
670
|
+
champollion register-corpus --yes --name "Sealed eng-crk" --pair "eng>crk" \\
|
|
671
|
+
--license proprietary --tier sealed --size 500 --domain educational \\
|
|
672
|
+
--seal-input ./secret-corpus.json --threshold-pubkey ./group.pub \\
|
|
673
|
+
--custodian-group nehiyawewin-trust --seal-out ./sealed.json \\
|
|
674
|
+
--qualifier-id eval-eng-crk-nehiyaw-qualifier-v2026
|
|
675
|
+
`);
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
export { run };
|