champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
|
@@ -13,16 +13,19 @@
|
|
|
13
13
|
* text is never hosted by us. Gated by cli/lib/license-gate.mjs so NC /
|
|
14
14
|
* no-redistribute / unconfirmed licenses cannot enter the public lane.
|
|
15
15
|
*
|
|
16
|
-
* We NEVER
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* We NEVER upload or host corpus content in ANY tier — only metadata and (for
|
|
17
|
+
* public) a pointer to where the data is fetched from. This command writes a
|
|
18
|
+
* metadata card. A file it is given is read on this machine only: --data to
|
|
19
|
+
* count its entries and compute its sha256, --seal-input (sealed tier) to
|
|
20
|
+
* encrypt it. Nothing it reads is copied, uploaded or sent anywhere.
|
|
19
21
|
*
|
|
20
22
|
* Interactive wizard when stdin is a TTY; fully scriptable via flags otherwise
|
|
21
|
-
* (or with --yes). See `champollion register-corpus --help`.
|
|
23
|
+
* (or with --yes). See `champollion network register-corpus --help`.
|
|
22
24
|
*
|
|
23
25
|
* Exit codes: 0 = registered/saved (or cancelled); 1 = invalid request / gate block.
|
|
24
26
|
*/
|
|
25
27
|
|
|
28
|
+
import crypto from 'node:crypto';
|
|
26
29
|
import fs from 'node:fs';
|
|
27
30
|
import path from 'node:path';
|
|
28
31
|
import readline from 'node:readline';
|
|
@@ -37,6 +40,10 @@ import {
|
|
|
37
40
|
gatePublicRegistration,
|
|
38
41
|
buildCorpusCard,
|
|
39
42
|
deriveCardId,
|
|
43
|
+
deriveIdSlug,
|
|
44
|
+
normalizeRole,
|
|
45
|
+
CORPUS_ROLES,
|
|
46
|
+
CARD_ID_PATTERN,
|
|
40
47
|
slugify,
|
|
41
48
|
resolveDestination,
|
|
42
49
|
validateRegistration,
|
|
@@ -49,11 +56,280 @@ import {
|
|
|
49
56
|
buildAad,
|
|
50
57
|
} from '../seal.mjs';
|
|
51
58
|
import { DEFAULT_QUALIFIER_THRESHOLD } from '../sealed-qualifier.mjs';
|
|
59
|
+
import { getLanguageCard } from '../registers.js';
|
|
60
|
+
import { findPublicCorporaBySha } from '../public-catalogue.js';
|
|
61
|
+
import { parseLanguagePair, formatLanguagePair } from '../language-pair.js';
|
|
62
|
+
import { showCommandHelp } from '../command-help.js';
|
|
63
|
+
|
|
64
|
+
/** A word as a POSIX shell reads it back exactly (quoted only when it must be). */
|
|
65
|
+
function shellQuote(word) {
|
|
66
|
+
const w = String(word);
|
|
67
|
+
return /^[A-Za-z0-9_.:/@%+=,-]+$/.test(w) ? w : `'${w.replace(/'/g, `'\\''`)}'`;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The language flags of the `mt-eval run` hint, filled from the card's pair
|
|
72
|
+
* (Round 7, hospital persona: the hint printed <src>/<tgt> although the card
|
|
73
|
+
* it had just written holds the pair). mt-eval's --source-lang/--target-lang
|
|
74
|
+
* are the NAMES the model is told (from the language card, else the
|
|
75
|
+
* project's own "name" for the code in champollion.config.json — a
|
|
76
|
+
* private-use code such as qaa has no card); --source-code/--target-lang-code
|
|
77
|
+
* are the codes the run is recorded under (--target-lang-code is mt-eval's
|
|
78
|
+
* documented spelling — --target-code is its alias, Round 12).
|
|
79
|
+
*
|
|
80
|
+
* @returns {{ flags: string, unnamed: string[], cardless: string[], names: Object<string, string|null> }}
|
|
81
|
+
* unnamed: codes with no name anywhere (the hint says to put the language's
|
|
82
|
+
* name in their flag); cardless: codes with no language card (nmt-forge init
|
|
83
|
+
* needs --no-card for them); names: the name found for each code, or null
|
|
84
|
+
*/
|
|
85
|
+
function languageFlags(pair, cwd) {
|
|
86
|
+
let projectNames = {};
|
|
87
|
+
try {
|
|
88
|
+
const cfg = JSON.parse(fs.readFileSync(path.join(cwd, 'champollion.config.json'), 'utf-8'));
|
|
89
|
+
if (cfg && cfg.languages && typeof cfg.languages === 'object' && !Array.isArray(cfg.languages)) {
|
|
90
|
+
for (const [code, v] of Object.entries(cfg.languages)) if (v && typeof v.name === 'string') projectNames[code] = v.name;
|
|
91
|
+
}
|
|
92
|
+
} catch { projectNames = {}; }
|
|
93
|
+
const unnamed = [];
|
|
94
|
+
const cardless = [];
|
|
95
|
+
const names = {};
|
|
96
|
+
const nameOf = (code) => {
|
|
97
|
+
let card = null;
|
|
98
|
+
try { card = getLanguageCard(code); } catch { card = null; }
|
|
99
|
+
if (!card) cardless.push(code);
|
|
100
|
+
const name = (card && typeof card.name === 'string' && card.name) || projectNames[code] || null;
|
|
101
|
+
if (!name) unnamed.push(code);
|
|
102
|
+
names[code] = name;
|
|
103
|
+
return name || code;
|
|
104
|
+
};
|
|
105
|
+
const flags = `--source-lang ${shellQuote(nameOf(pair.source))} --target-lang ${shellQuote(nameOf(pair.target))} `
|
|
106
|
+
+ `--source-code ${shellQuote(pair.source)} --target-lang-code ${shellQuote(pair.target)}`;
|
|
107
|
+
return { flags, unnamed, cardless, names };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Would someone train a model against this set? Then its first score must come
|
|
112
|
+
* AFTER nmt-forge has registered it and the predictions are written down.
|
|
113
|
+
* A test set — stated with --role test, or a local-only / private set whose
|
|
114
|
+
* role was not stated (the tiers a community's own held-out set uses) — may be
|
|
115
|
+
* trained against; a dev or train set is not what forge's predictions are
|
|
116
|
+
* judged on.
|
|
117
|
+
*/
|
|
118
|
+
function mayBeTrainedAgainst(req) {
|
|
119
|
+
if (req.role === 'test') return true;
|
|
120
|
+
if (req.role) return false;
|
|
121
|
+
return req.tier.key === 'local-only' || req.tier.key === 'private';
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* The nmt-forge project already set up for this pair: this directory, or one
|
|
126
|
+
* folder below it, that holds a forge workspace (.forge/) and a config.json
|
|
127
|
+
* for the same source and target. Exactly one such folder, or null — with
|
|
128
|
+
* none (or several) the hint starts a new one with `nmt-forge init`.
|
|
129
|
+
*/
|
|
130
|
+
function findForgeProject(cwd, pair) {
|
|
131
|
+
const candidates = [cwd];
|
|
132
|
+
let entries = [];
|
|
133
|
+
try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { entries = []; }
|
|
134
|
+
for (const e of entries) {
|
|
135
|
+
if (e.isDirectory() && !e.name.startsWith('.') && e.name !== 'node_modules') candidates.push(path.join(cwd, e.name));
|
|
136
|
+
}
|
|
137
|
+
const matches = candidates.filter((dir) => {
|
|
138
|
+
if (!fs.existsSync(path.join(dir, '.forge'))) return false;
|
|
139
|
+
let lang;
|
|
140
|
+
try { lang = JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf-8'))?.language; } catch { return false; }
|
|
141
|
+
return !!lang && lang.source === pair.source && lang.target === pair.target;
|
|
142
|
+
});
|
|
143
|
+
return matches.length === 1 ? matches[0] : null;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* What an existing forge project already knows about this file: the name it
|
|
148
|
+
* is registered under (matched by sha256 — forge hashes the file's bytes, as
|
|
149
|
+
* describeDataFile does), the preregistrations bound to it, and the set names
|
|
150
|
+
* taken by other files.
|
|
151
|
+
*/
|
|
152
|
+
function forgeKnows(projectDir, sha256) {
|
|
153
|
+
let sets = {};
|
|
154
|
+
try {
|
|
155
|
+
sets = JSON.parse(fs.readFileSync(path.join(projectDir, '.forge', 'eval-registry.json'), 'utf-8'))?.sets || {};
|
|
156
|
+
} catch { sets = {}; } // no registry yet: nothing is registered
|
|
157
|
+
const name = sha256 ? (Object.entries(sets).find(([, e]) => e && e.sha256 === sha256) || [null])[0] : null;
|
|
158
|
+
const preregs = [];
|
|
159
|
+
const preregIds = new Set();
|
|
160
|
+
const dir = path.join(projectDir, '.forge', 'preregistrations');
|
|
161
|
+
let files = [];
|
|
162
|
+
try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { files = []; } // none written yet
|
|
163
|
+
for (const f of files) {
|
|
164
|
+
const id = f.replace(/\.json$/, '');
|
|
165
|
+
preregIds.add(id);
|
|
166
|
+
try {
|
|
167
|
+
const pr = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf-8'));
|
|
168
|
+
if (sha256 && pr?.eval_set?.sha256 === sha256) preregs.push(pr.id || id);
|
|
169
|
+
} catch { /* not a preregistration forge wrote: its id is still taken */ }
|
|
170
|
+
}
|
|
171
|
+
return { name, preregs, preregIds, taken: new Set(Object.keys(sets)) };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The nmt-forge steps that must come before ANY score of a test set a model
|
|
176
|
+
* may be trained against — the build-MT guide's "register, screen, predict"
|
|
177
|
+
* (Round 10, Cree-school persona: the "Next:" line went straight to
|
|
178
|
+
* `mt-eval run`, a scoring read, and forge then refuses the predictions).
|
|
179
|
+
* Paths are written from inside the forge project, where forge runs. An
|
|
180
|
+
* existing project is read first, so no printed step is one forge refuses:
|
|
181
|
+
* a file it already holds keeps its name, and `project-test` is not reused
|
|
182
|
+
* for a different file.
|
|
183
|
+
*
|
|
184
|
+
* @returns {{lines: string[], done: boolean}} done: the file is registered
|
|
185
|
+
* and has predictions bound to it — the baseline can run now
|
|
186
|
+
*/
|
|
187
|
+
function forgeFirstLines({ pair, dataFile, sha256, cardId, cwd, lang }) {
|
|
188
|
+
const existing = findForgeProject(cwd, pair);
|
|
189
|
+
const projectDir = existing || path.join(cwd, `${pair.target}-model`);
|
|
190
|
+
const rel = path.relative(cwd, projectDir);
|
|
191
|
+
const where = rel || '.';
|
|
192
|
+
const known = existing ? forgeKnows(projectDir, sha256) : { name: null, preregs: [], preregIds: new Set(), taken: new Set() };
|
|
193
|
+
if (known.name && known.preregs.length > 0) {
|
|
194
|
+
return {
|
|
195
|
+
done: true,
|
|
196
|
+
lines: [
|
|
197
|
+
` nmt-forge already holds this file (as ${known.name}, in ${where}) with predictions written`,
|
|
198
|
+
` down before any score (${known.preregs.join(', ')}). \`nmt-forge status\` there names the next step.`,
|
|
199
|
+
],
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
const setName = known.name || (known.taken.has('project-test') ? cardId : 'project-test');
|
|
203
|
+
// forge refuses a prereg id that exists ("the old one stands as history").
|
|
204
|
+
let preregId = 'all-data';
|
|
205
|
+
for (let n = 2; known.preregIds.has(preregId); n += 1) preregId = `all-data-${n}`;
|
|
206
|
+
// …and `prereg template --out` will not overwrite a predictions file.
|
|
207
|
+
let predictions = preregId === 'all-data' ? 'predictions.json' : `predictions-${preregId}.json`;
|
|
208
|
+
for (let n = 2; existing && fs.existsSync(path.join(projectDir, predictions)); n += 1) predictions = `predictions-${preregId}-${n}.json`;
|
|
209
|
+
const testFile = dataFile ? shellQuote(path.relative(projectDir, dataFile)) : '<your test file>';
|
|
210
|
+
const lines = [
|
|
211
|
+
` Training a model for ${formatLanguagePair(pair)}? Do this first, before anything scores this file.`,
|
|
212
|
+
' A benchmark is a scoring read, and nmt-forge refuses predictions written after one:',
|
|
213
|
+
];
|
|
214
|
+
if (!existing) {
|
|
215
|
+
const noCard = lang.cardless.includes(pair.target)
|
|
216
|
+
? ` --no-card --name ${lang.names[pair.target] ? shellQuote(lang.names[pair.target]) : "'<the language name>'"}`
|
|
217
|
+
: '';
|
|
218
|
+
lines.push(` nmt-forge init ${shellQuote(pair.target)} --dir ${shellQuote(rel)}${noCard} # once: the training project (any folder name)`);
|
|
219
|
+
}
|
|
220
|
+
if (rel) lines.push(` cd ${shellQuote(rel)}`);
|
|
221
|
+
if (known.name) {
|
|
222
|
+
lines.push(` # already registered with nmt-forge as ${known.name}`);
|
|
223
|
+
} else {
|
|
224
|
+
lines.push(` nmt-forge registry add ${shellQuote(setName)} ${testFile} --role test`);
|
|
225
|
+
}
|
|
226
|
+
lines.push(' nmt-forge leak-audit <training corpus> --clean-to corpus.clean.jsonl');
|
|
227
|
+
lines.push(` nmt-forge prereg template --out ${predictions} # then edit it: what you expect, and why`);
|
|
228
|
+
lines.push(` nmt-forge prereg new ${preregId} --eval-set ${shellQuote(setName)} --predictions ${predictions}`);
|
|
229
|
+
if (rel) lines.push(' cd ..');
|
|
230
|
+
lines.push(' (One prediction file per model you plan to train, each named after it — leak-audit\'s');
|
|
231
|
+
lines.push(' verdict may call for a second, twin-free model.)');
|
|
232
|
+
return { lines, done: false };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** The model for a local run: the project's own when it runs `local`, else a placeholder. */
|
|
236
|
+
function localModelFor(cwd) {
|
|
237
|
+
try {
|
|
238
|
+
const cfg = JSON.parse(fs.readFileSync(path.join(cwd, 'champollion.config.json'), 'utf-8'));
|
|
239
|
+
if (cfg?.defaultMethod === 'local' && typeof cfg.model === 'string' && cfg.model) return cfg.model;
|
|
240
|
+
} catch { /* no project config here */ }
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
52
243
|
|
|
53
244
|
// Tracked corpora-cards SSOT, resolved relative to this module.
|
|
54
245
|
const DEFAULT_CARDS_DIR = fileURLToPath(new URL('../../shared/corpora-cards/', import.meta.url));
|
|
55
246
|
|
|
56
247
|
const CONTAMINATION_LEVELS = ['NONE', 'LOW', 'MEDIUM', 'HIGH'];
|
|
248
|
+
// Written (never typed): the file could not be compared with the public
|
|
249
|
+
// corpora and no grade was stated — see the contamination block in run().
|
|
250
|
+
const CONTAMINATION_UNCHECKED = 'UNCHECKED';
|
|
251
|
+
|
|
252
|
+
// The steward sidecar the harness reads next to a corpus file
|
|
253
|
+
// (mt_eval_harness.corpus_loader.SIDECAR_SUFFIX — keep the two in step).
|
|
254
|
+
const SIDECAR_SUFFIX = '.champollion.json';
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Read the local data file a registration describes: its sha256 and how many
|
|
258
|
+
* sentence pairs it holds (TSV/text: non-empty lines that are not '# '
|
|
259
|
+
* comments; JSONL: non-empty lines; JSON: entries). The text itself is read
|
|
260
|
+
* only to hash and count it — it is never copied, uploaded or stored.
|
|
261
|
+
*/
|
|
262
|
+
function describeDataFile(file) {
|
|
263
|
+
const bytes = fs.readFileSync(file);
|
|
264
|
+
const sha256 = crypto.createHash('sha256').update(bytes).digest('hex');
|
|
265
|
+
const text = bytes.toString('utf-8').replace(/^\uFEFF/, '');
|
|
266
|
+
let rows;
|
|
267
|
+
if (file.endsWith('.json')) {
|
|
268
|
+
const data = JSON.parse(text);
|
|
269
|
+
rows = Array.isArray(data) ? data.length
|
|
270
|
+
: Array.isArray(data?.entries) ? data.entries.length : 0;
|
|
271
|
+
} else {
|
|
272
|
+
rows = text.split('\n')
|
|
273
|
+
.map((l) => l.replace(/\r$/, ''))
|
|
274
|
+
.filter((l) => l.trim() && !/^#(\s|$)/.test(l)).length;
|
|
275
|
+
}
|
|
276
|
+
return { sha256, rows };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* The public corpus a file is a byte-identical copy of, from the corpora
|
|
281
|
+
* cards this CLI can see (each pins its built corpus's sha256, and the
|
|
282
|
+
* upstream archive's) — or null. A private, local-only or sealed card claims
|
|
283
|
+
* text no model has seen; a file the cards show is public cannot claim that
|
|
284
|
+
* (Round 8, researcher persona: a copy of the public Tatoeba eng→sme set was
|
|
285
|
+
* graded "Contamination: NONE", caught only later by `contest prepare`).
|
|
286
|
+
*
|
|
287
|
+
* @param {string} sha256
|
|
288
|
+
* @param {string} cardsDir
|
|
289
|
+
* @returns {{ id: string, name: string, license: string|null, risk: string|null }|null}
|
|
290
|
+
*/
|
|
291
|
+
function findPublicCopy(sha256, cardsDir) {
|
|
292
|
+
let files;
|
|
293
|
+
try { files = fs.readdirSync(cardsDir).filter((f) => f.endsWith('.json')); } catch { return null; }
|
|
294
|
+
for (const f of files) {
|
|
295
|
+
let card;
|
|
296
|
+
try { card = JSON.parse(fs.readFileSync(path.join(cardsDir, f), 'utf-8')); } catch { continue; }
|
|
297
|
+
const src = card?.source || {};
|
|
298
|
+
const tier = card?.exposureTier || (src.repo_url || src.url ? 'public' : null);
|
|
299
|
+
if (tier && tier !== 'public') continue;
|
|
300
|
+
if (src.sha256 === sha256 || src.archive_sha256 === sha256) {
|
|
301
|
+
return { id: card.id || f.replace(/\.json$/, ''), name: card.name || card.id, license: card.license?.spdx || src.license || null, risk: card.contamination?.risk || null };
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Write (or update) the sidecar next to the data file so `mt-eval run
|
|
309
|
+
* --corpus <file>` knows what this registration said: its id, licence and
|
|
310
|
+
* checksum — and, for local-only, that no outside service may see it. The
|
|
311
|
+
* harness lets a sidecar only TIGHTEN what it does, and this never loosens an
|
|
312
|
+
* existing one: a file already marked local-only stays local-only.
|
|
313
|
+
*/
|
|
314
|
+
function readSidecar(file) {
|
|
315
|
+
const sidecar = file + SIDECAR_SUFFIX;
|
|
316
|
+
if (!fs.existsSync(sidecar)) return {};
|
|
317
|
+
try {
|
|
318
|
+
return JSON.parse(fs.readFileSync(sidecar, 'utf-8'));
|
|
319
|
+
} catch (e) {
|
|
320
|
+
throw new Error(`${sidecar} exists but is not valid JSON (${e.message}). `
|
|
321
|
+
+ 'Fix or remove it — it states how this corpus may be used.');
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function writeSidecar(file, fields) {
|
|
326
|
+
const sidecar = file + SIDECAR_SUFFIX;
|
|
327
|
+
const existing = readSidecar(file);
|
|
328
|
+
const merged = { ...existing, ...fields };
|
|
329
|
+
if (existing.transmission === 'local-only') merged.transmission = 'local-only';
|
|
330
|
+
fs.writeFileSync(sidecar, JSON.stringify(merged, null, 2) + '\n', 'utf-8');
|
|
331
|
+
return { path: sidecar, transmission: merged.transmission || null };
|
|
332
|
+
}
|
|
57
333
|
|
|
58
334
|
/**
|
|
59
335
|
* Resolve a path to its REAL location, following symlinks. The destination
|
|
@@ -91,21 +367,30 @@ function isInteractive() {
|
|
|
91
367
|
return process.stdin.isTTY === true;
|
|
92
368
|
}
|
|
93
369
|
|
|
94
|
-
function ask(rl, question, defaultValue) {
|
|
370
|
+
function ask(rl, question, defaultValue, shownDefault = defaultValue) {
|
|
95
371
|
return new Promise((resolve) => {
|
|
96
|
-
|
|
372
|
+
// shownDefault: what Enter means, shown in the prompt — the caller may
|
|
373
|
+
// want to tell Enter apart from a typed value (defaultValue '').
|
|
374
|
+
const suffix = shownDefault ? ` (${shownDefault})` : '';
|
|
97
375
|
rl.question(` ${question}${suffix}: `, (answer) => {
|
|
98
376
|
resolve(answer.trim() || defaultValue || '');
|
|
99
377
|
});
|
|
100
378
|
});
|
|
101
379
|
}
|
|
102
380
|
|
|
103
|
-
/**
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
381
|
+
/**
|
|
382
|
+
* Read --pair (any spelling lib/language-pair.js accepts: eng>crk, eng-crk,
|
|
383
|
+
* eng:crk, …). Codes are lower-cased: they name the card id, which is
|
|
384
|
+
* lower-case. The old reader split on every - and > and kept the first two
|
|
385
|
+
* pieces, so "eng>pt-BR" was registered as eng→pt without a word.
|
|
386
|
+
*
|
|
387
|
+
* @returns {{pair: {source: string, target: string}|null, pairError: string|null}}
|
|
388
|
+
*/
|
|
389
|
+
function readPair(value) {
|
|
390
|
+
if (value === undefined || value === null || value === '') return { pair: null, pairError: null };
|
|
391
|
+
const p = parseLanguagePair(value, { label: '--pair' });
|
|
392
|
+
if (!p.ok) return { pair: null, pairError: p.error };
|
|
393
|
+
return { pair: { source: p.source.toLowerCase(), target: p.target.toLowerCase() }, pairError: null };
|
|
109
394
|
}
|
|
110
395
|
|
|
111
396
|
// ── Catalog printing (plain language for humans, machine-readable for agents) ──
|
|
@@ -119,24 +404,29 @@ function printCatalogs(asJson) {
|
|
|
119
404
|
}, null, 2));
|
|
120
405
|
return;
|
|
121
406
|
}
|
|
407
|
+
// The id beside each label is what --license / --tier take (Round 8,
|
|
408
|
+
// school + hospital personas: the list showed numbered labels only).
|
|
122
409
|
console.log('');
|
|
123
|
-
console.log(' LICENSES — pick the terms others may use your corpus under:');
|
|
410
|
+
console.log(' LICENSES — pick the terms others may use your corpus under (pass the id to --license):');
|
|
124
411
|
console.log('');
|
|
125
412
|
LICENSE_OPTIONS.forEach((o, i) => {
|
|
126
|
-
console.log(` ${i + 1}. ${o.label}`);
|
|
413
|
+
console.log(` ${i + 1}. ${o.key.padEnd(24)} ${o.label}`);
|
|
127
414
|
console.log(` ${o.explanation}`);
|
|
128
415
|
console.log(` commercial: ${o.commercial} · redistribution: ${o.redistribution} · public-lane: ${o.publicEligible ? 'eligible' : 'blocked'}`);
|
|
129
416
|
});
|
|
130
417
|
console.log('');
|
|
131
|
-
console.log('
|
|
418
|
+
console.log(' --license also takes the list number, or an SPDX id (e.g. --license CC-BY-4.0).');
|
|
419
|
+
console.log('');
|
|
420
|
+
console.log(' EXPOSURE TIERS — how far your corpus travels (default: most private; pass the id to --tier):');
|
|
132
421
|
console.log('');
|
|
133
422
|
EXPOSURE_TIERS.forEach((t, i) => {
|
|
134
423
|
const def = t.key === DEFAULT_TIER ? ' ★ default' : '';
|
|
135
|
-
console.log(` ${i + 1}. ${t.label}${def}`);
|
|
424
|
+
console.log(` ${i + 1}. ${t.key.padEnd(11)} ${t.label}${def}`);
|
|
136
425
|
console.log(` ${t.explanation}`);
|
|
137
426
|
});
|
|
138
427
|
console.log('');
|
|
139
|
-
console.log(' Champollion never
|
|
428
|
+
console.log(' Champollion never uploads or hosts your corpus text — in ANY tier. With --data, the');
|
|
429
|
+
console.log(' file is read on this machine only to count and checksum it; none of it leaves.');
|
|
140
430
|
console.log('');
|
|
141
431
|
}
|
|
142
432
|
|
|
@@ -148,18 +438,32 @@ async function runInteractive(rl, args) {
|
|
|
148
438
|
console.log(' ════════════════════════════════════════════════');
|
|
149
439
|
console.log('');
|
|
150
440
|
console.log(' You choose the license and how far this corpus travels.');
|
|
151
|
-
console.log(' We never
|
|
152
|
-
console.log('
|
|
441
|
+
console.log(' We never upload or host your text — only metadata, and (for public');
|
|
442
|
+
console.log(' sets) a pointer to where the data is fetched from. A file you name');
|
|
443
|
+
console.log(' is read on this machine only: --data to count and checksum it, and');
|
|
444
|
+
console.log(' a sealed set\'s file to encrypt it.');
|
|
153
445
|
|
|
154
446
|
// Step 1 — basics
|
|
155
447
|
console.log('');
|
|
156
448
|
console.log(' Step 1/4 — Basics');
|
|
157
449
|
console.log(' ────────────────────────────────────────────────');
|
|
158
450
|
const name = await ask(rl, 'Corpus name', args.name || '');
|
|
159
|
-
|
|
160
|
-
|
|
451
|
+
// Asked again until it reads as a pair (or is left blank — validation then
|
|
452
|
+
// says a pair is required): an unreadable pair is said here, not at the end.
|
|
453
|
+
let pair = null;
|
|
454
|
+
let pairError = null;
|
|
455
|
+
for (let pairIn = args.pair || ''; ;) {
|
|
456
|
+
const pairStr = await ask(rl, 'Language pair, source>target (e.g. eng>crk — eng-crk works too)', pairIn);
|
|
457
|
+
({ pair, pairError } = readPair(pairStr));
|
|
458
|
+
if (!pairError) break;
|
|
459
|
+
console.log(` ⚠ ${pairError}`);
|
|
460
|
+
pairIn = '';
|
|
461
|
+
}
|
|
161
462
|
const publisher = await ask(rl, 'Publisher / your name or org', args.publisher || '');
|
|
162
463
|
const description = await ask(rl, 'One-line description', args.description || `${name} evaluation corpus`);
|
|
464
|
+
const role = normalizeRole(await ask(rl,
|
|
465
|
+
`What is it for — ${CORPUS_ROLES.join(', ')}? (blank: don't say; the id then names no role)`,
|
|
466
|
+
normalizeRole(args.role) || ''));
|
|
163
467
|
|
|
164
468
|
// Step 2 — license
|
|
165
469
|
console.log('');
|
|
@@ -253,14 +557,17 @@ async function runInteractive(rl, args) {
|
|
|
253
557
|
console.log('');
|
|
254
558
|
console.log(' Contamination risk — is this text in known LLM training sets?');
|
|
255
559
|
console.log(' NONE = private/unpublished · LOW = niche/recent · MEDIUM = public · HIGH = in major training sets');
|
|
256
|
-
|
|
257
|
-
|
|
560
|
+
// Typed, or the default accepted? A typed grade is a statement; the default
|
|
561
|
+
// is not, and gives way to UNCHECKED when the file cannot be compared.
|
|
562
|
+
const riskTyped = (await ask(rl, 'Contamination risk', '', defaultRisk)).toUpperCase();
|
|
563
|
+
const contaminationStated = CONTAMINATION_LEVELS.includes(riskTyped);
|
|
564
|
+
const contaminationRisk = contaminationStated ? riskTyped : defaultRisk;
|
|
258
565
|
|
|
259
566
|
return {
|
|
260
|
-
name, pair, publisher, description,
|
|
567
|
+
name, pair, publisher, description, role,
|
|
261
568
|
licenseOption, tier,
|
|
262
569
|
repoUrl, builder, sourceUrl,
|
|
263
|
-
size: Number(sizeStr), domain, contaminationRisk,
|
|
570
|
+
size: Number(sizeStr), domain, contaminationRisk, contaminationStated,
|
|
264
571
|
custodianGroupId, thresholdPubkey, sealInput, sealOut,
|
|
265
572
|
qualifierId, qualifierThreshold, keyScheme,
|
|
266
573
|
};
|
|
@@ -269,8 +576,10 @@ async function runInteractive(rl, args) {
|
|
|
269
576
|
// ── Non-interactive (flags) ─────────────────────────────────────────────────
|
|
270
577
|
|
|
271
578
|
function fromFlags(args) {
|
|
272
|
-
const
|
|
273
|
-
|
|
579
|
+
const read = args.pair !== undefined ? readPair(args.pair) : { pair: null, pairError: null };
|
|
580
|
+
const pairError = read.pairError;
|
|
581
|
+
const pair = read.pair ||
|
|
582
|
+
(!pairError && args['source-lang'] && args['target-lang']
|
|
274
583
|
? { source: String(args['source-lang']).toLowerCase(), target: String(args['target-lang']).toLowerCase() }
|
|
275
584
|
: null);
|
|
276
585
|
const licenseOption = resolveLicense(args.license);
|
|
@@ -279,8 +588,10 @@ function fromFlags(args) {
|
|
|
279
588
|
return {
|
|
280
589
|
name: args.name || '',
|
|
281
590
|
pair,
|
|
282
|
-
|
|
591
|
+
pairError,
|
|
592
|
+
publisher: args.publisher || '',
|
|
283
593
|
description: args.description || (args.name ? `${args.name} evaluation corpus` : ''),
|
|
594
|
+
role: args.role === true ? true : normalizeRole(args.role), // bare --role: validation says it needs a value
|
|
284
595
|
licenseOption,
|
|
285
596
|
tier,
|
|
286
597
|
repoUrl: args['repo-url'] || null,
|
|
@@ -289,6 +600,9 @@ function fromFlags(args) {
|
|
|
289
600
|
size: args.size != null ? Number(args.size) : NaN,
|
|
290
601
|
domain: args.domain || '',
|
|
291
602
|
contaminationRisk: riskIn && CONTAMINATION_LEVELS.includes(riskIn) ? riskIn : null,
|
|
603
|
+
// A grade you passed is your statement; it stands even when the file
|
|
604
|
+
// could not be compared with the public corpora.
|
|
605
|
+
contaminationStated: !!(riskIn && CONTAMINATION_LEVELS.includes(riskIn)),
|
|
292
606
|
// sealed tier
|
|
293
607
|
custodianGroupId: args['custodian-group'] || null,
|
|
294
608
|
thresholdPubkey: args['threshold-pubkey'] || null,
|
|
@@ -421,16 +735,111 @@ async function run(args, cwd) {
|
|
|
421
735
|
req = fromFlags(args);
|
|
422
736
|
}
|
|
423
737
|
|
|
738
|
+
// --data: the local file this registration describes. Hashed and counted
|
|
739
|
+
// here (never copied or uploaded) so the card carries its checksum and the
|
|
740
|
+
// harness can find what was registered, through the sidecar written below.
|
|
741
|
+
let dataFile = null;
|
|
742
|
+
let dataInfo = null;
|
|
743
|
+
if (args.data) {
|
|
744
|
+
dataFile = path.resolve(cwd, String(args.data));
|
|
745
|
+
if (!fs.existsSync(dataFile) || !fs.statSync(dataFile).isFile()) {
|
|
746
|
+
console.error(`[ERR] --data: no such file: ${dataFile}`);
|
|
747
|
+
return 1;
|
|
748
|
+
}
|
|
749
|
+
if (req.tier && (req.tier.key === 'public' || req.tier.key === 'sealed')) {
|
|
750
|
+
console.error(`[ERR] --data is for local-only and private corpora. A ${req.tier.key} corpus is `
|
|
751
|
+
+ (req.tier.key === 'public' ? 'fetched from its source (--repo-url / --builder).'
|
|
752
|
+
: 'encrypted from --seal-input.'));
|
|
753
|
+
return 1;
|
|
754
|
+
}
|
|
755
|
+
try {
|
|
756
|
+
dataInfo = describeDataFile(dataFile);
|
|
757
|
+
} catch (e) {
|
|
758
|
+
console.error(`[ERR] --data: could not read ${dataFile}: ${e.message}`);
|
|
759
|
+
return 1;
|
|
760
|
+
}
|
|
761
|
+
if (!Number.isFinite(req.size) || req.size <= 0) req.size = dataInfo.rows;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// A private / local-only / sealed registration of text that is public: never
|
|
765
|
+
// graded "Contamination: NONE". Compared with the corpora cards this CLI can
|
|
766
|
+
// see (a repo checkout), else — an npm install ships none — with the public
|
|
767
|
+
// corpus catalogue (lib/public-catalogue.js; only the public ids and
|
|
768
|
+
// checksums are downloaded, the file's own sha256 stays here). When neither
|
|
769
|
+
// can be read, the comparison was not made, and the grade says so
|
|
770
|
+
// (UNCHECKED) unless you state one with --contamination (Round 9,
|
|
771
|
+
// researcher persona: an npm install graded a byte-identical copy of a
|
|
772
|
+
// public benchmark NONE because it had nothing to compare with).
|
|
773
|
+
{
|
|
774
|
+
const cardsDir = args['cards-dir'] ? path.resolve(cwd, args['cards-dir']) : DEFAULT_CARDS_DIR;
|
|
775
|
+
const checked = [];
|
|
776
|
+
if (dataInfo) checked.push({ flag: '--data', file: dataFile, sha256: dataInfo.sha256 });
|
|
777
|
+
if (req.tier && req.tier.key === 'sealed' && req.sealInput) {
|
|
778
|
+
try {
|
|
779
|
+
const plain = path.resolve(cwd, String(req.sealInput));
|
|
780
|
+
checked.push({ flag: '--seal-input', file: plain, sha256: crypto.createHash('sha256').update(fs.readFileSync(plain)).digest('hex') });
|
|
781
|
+
} catch { /* the sealing step names an unreadable file */ }
|
|
782
|
+
}
|
|
783
|
+
if (checked.length > 0 && req.tier && req.tier.key !== 'public') {
|
|
784
|
+
let lookup;
|
|
785
|
+
if (fs.existsSync(cardsDir)) {
|
|
786
|
+
lookup = { ok: true, where: 'the corpora cards', find: (sha) => findPublicCopy(sha, cardsDir) };
|
|
787
|
+
} else {
|
|
788
|
+
const cat = await findPublicCorporaBySha(checked.map(c => c.sha256));
|
|
789
|
+
lookup = cat.ok
|
|
790
|
+
? { ok: true, where: `the public corpus catalogue (${cat.compared} checksums)`, find: (sha) => (cat.hits.has(sha) ? { ...cat.hits.get(sha), risk: null } : null) }
|
|
791
|
+
: { ok: false, why: `this install ships no corpora cards, and ${cat.why}` };
|
|
792
|
+
}
|
|
793
|
+
if (!lookup.ok) {
|
|
794
|
+
// Not compared. NONE claims text no model has seen — it is never the
|
|
795
|
+
// default for a comparison that was not made.
|
|
796
|
+
const shown = checked.map(c => path.relative(cwd, c.file) || c.file).join(', ');
|
|
797
|
+
req.contaminationNote = `not compared with the public corpora — ${lookup.why}`;
|
|
798
|
+
if (req.contaminationStated) {
|
|
799
|
+
req.contaminationReasoning = `Stated at registration (${req.contaminationRisk}); ${shown} was not compared with the public corpora (${lookup.why}).`;
|
|
800
|
+
} else {
|
|
801
|
+
req.contaminationRisk = CONTAMINATION_UNCHECKED;
|
|
802
|
+
req.contaminationReasoning = `Not graded: ${shown} could not be compared with the public corpora at registration (${lookup.why}). `
|
|
803
|
+
+ 'Re-register online to compare it, or state a grade with --contamination none|low|medium|high.';
|
|
804
|
+
}
|
|
805
|
+
} else {
|
|
806
|
+
req.contaminationCompared = lookup.where;
|
|
807
|
+
for (const c of checked) {
|
|
808
|
+
const hit = lookup.find(c.sha256);
|
|
809
|
+
if (!hit) continue;
|
|
810
|
+
const shown = path.relative(cwd, c.file) || c.file;
|
|
811
|
+
const why = `${shown} is a byte-identical copy of the public corpus ${hit.id} (${hit.name}${hit.license ? `, ${hit.license}` : ''}; same sha256 ${c.sha256.slice(0, 12)}…, found in ${lookup.where}).`;
|
|
812
|
+
if (req.tier.key === 'sealed') {
|
|
813
|
+
console.error(`[ERR] ${why}`);
|
|
814
|
+
console.error(' A sealed test set is text no model has seen; this text is public, so sealing it tests nothing.');
|
|
815
|
+
console.error(` Register it as public instead (--tier public, fetched from its source — ${hit.id} already is), or seal sentences that are genuinely private.`);
|
|
816
|
+
return 1;
|
|
817
|
+
}
|
|
818
|
+
if (!req.contaminationRisk || req.contaminationRisk === 'NONE') {
|
|
819
|
+
console.error(`[ERR] ${why}`);
|
|
820
|
+
console.error(` A ${req.tier.key} card would say "Contamination: NONE" (text no model has seen); this text is public${hit.risk ? ` — the public card rates it ${hit.risk}` : ''}.`);
|
|
821
|
+
console.error(` Register it as public (--tier public — ${hit.id} already describes it), use sentences that are genuinely private,`);
|
|
822
|
+
console.error(` or keep it ${req.tier.key} and state its exposure: --contamination ${hit.risk && hit.risk !== 'NONE' ? hit.risk : 'MEDIUM'}.`);
|
|
823
|
+
return 1;
|
|
824
|
+
}
|
|
825
|
+
req.contaminationReasoning = `${why} Registered ${req.tier.key} with the stated risk ${req.contaminationRisk}; the text itself is public.`;
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
424
831
|
// Validate (collects every problem, fails loud).
|
|
425
832
|
const v = validateRegistration({
|
|
426
833
|
tier: req.tier,
|
|
427
834
|
licenseOption: req.licenseOption,
|
|
428
835
|
pair: req.pair || {},
|
|
836
|
+
pairError: req.pairError || null,
|
|
429
837
|
name: req.name,
|
|
430
838
|
repoUrl: req.repoUrl,
|
|
431
839
|
builder: req.builder,
|
|
432
840
|
size: req.size,
|
|
433
841
|
domain: req.domain,
|
|
842
|
+
role: req.role,
|
|
434
843
|
custodianGroupId: req.custodianGroupId,
|
|
435
844
|
thresholdPublicKey: req.thresholdPubkey,
|
|
436
845
|
sealInput: req.sealInput,
|
|
@@ -440,17 +849,58 @@ async function run(args, cwd) {
|
|
|
440
849
|
console.error('[ERR] Cannot register this corpus:');
|
|
441
850
|
for (const e of v.errors) console.error(` • ${e}`);
|
|
442
851
|
console.error('');
|
|
443
|
-
console.error(' Run "champollion register-corpus --list" to see licenses + tiers,');
|
|
444
|
-
console.error(' or "champollion register-corpus --help" for all flags.');
|
|
852
|
+
console.error(' Run "champollion network register-corpus --list" to see licenses + tiers,');
|
|
853
|
+
console.error(' or "champollion network register-corpus --help" for all flags.');
|
|
445
854
|
return 1;
|
|
446
855
|
}
|
|
447
856
|
|
|
448
857
|
// Build the (content-free) card.
|
|
858
|
+
// The id: a full --id (eval-…/ref-…) is used exactly as given; otherwise it
|
|
859
|
+
// is derived from --name (publisher only as a fallback), with a role segment
|
|
860
|
+
// only when --role states one. See deriveCardId.
|
|
449
861
|
const version = args.version || '1';
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
862
|
+
let id;
|
|
863
|
+
if (args.id && /^(ref|eval)-/.test(args.id)) {
|
|
864
|
+
id = String(args.id);
|
|
865
|
+
if (!CARD_ID_PATTERN.test(id)) {
|
|
866
|
+
console.error(`[ERR] --id '${id}' is not a valid card id: after eval- or ref-, use only`);
|
|
867
|
+
console.error(' lower-case letters, digits and hyphens (it is also the card filename).');
|
|
868
|
+
return 1;
|
|
869
|
+
}
|
|
870
|
+
if (req.role && !new RegExp(`-${req.role}(-|$)`).test(id)) {
|
|
871
|
+
console.error(`[ERR] --role ${req.role} shapes a generated id, but --id is used exactly as given`);
|
|
872
|
+
console.error(` and '${id}' does not name that role. Put the role in --id, or drop --role.`);
|
|
873
|
+
return 1;
|
|
874
|
+
}
|
|
875
|
+
} else {
|
|
876
|
+
const slug = (args.id && slugify(args.id)) || deriveIdSlug({ name: req.name, publisher: req.publisher });
|
|
877
|
+
id = deriveCardId({ source: req.pair.source, target: req.pair.target, slug, role: req.role, version });
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// A data file that is already registered keeps its id. Its sidecar records
|
|
881
|
+
// the id it was registered under; deriving a fresh one on a re-run would
|
|
882
|
+
// silently relink the file to a second card. Ids are never re-derived.
|
|
883
|
+
// A file already marked local-only stays local-only (writeSidecar never
|
|
884
|
+
// loosens a mark), and its card says so too.
|
|
885
|
+
let priorTransmission = null;
|
|
886
|
+
if (dataFile) {
|
|
887
|
+
let priorSidecar;
|
|
888
|
+
try {
|
|
889
|
+
priorSidecar = readSidecar(dataFile);
|
|
890
|
+
} catch (e) {
|
|
891
|
+
console.error(`[ERR] ${e.message}`);
|
|
892
|
+
return 1;
|
|
893
|
+
}
|
|
894
|
+
if (priorSidecar.transmission === 'local-only') priorTransmission = 'local-only';
|
|
895
|
+
if (priorSidecar.id && priorSidecar.id !== id && !args.id) {
|
|
896
|
+
const shown = path.relative(cwd, dataFile) || dataFile;
|
|
897
|
+
console.error(`[ERR] ${shown} is already registered as ${priorSidecar.id}`);
|
|
898
|
+
console.error(` (recorded in ${path.basename(dataFile)}${SIDECAR_SUFFIX}). A registered id is never re-derived.`);
|
|
899
|
+
console.error(` To keep it, add: --id ${priorSidecar.id}`);
|
|
900
|
+
console.error(` To register the file under a new id: --id ${id}`);
|
|
901
|
+
return 1;
|
|
902
|
+
}
|
|
903
|
+
}
|
|
454
904
|
|
|
455
905
|
const addedAt = new Date().toISOString().slice(0, 10);
|
|
456
906
|
|
|
@@ -487,17 +937,28 @@ async function run(args, cwd) {
|
|
|
487
937
|
licenseOption: req.licenseOption,
|
|
488
938
|
tier: req.tier,
|
|
489
939
|
contaminationRisk: req.contaminationRisk,
|
|
940
|
+
...(req.contaminationReasoning && { contaminationReasoning: req.contaminationReasoning }),
|
|
941
|
+
role: req.role || null,
|
|
942
|
+
dataRead: !!dataInfo,
|
|
490
943
|
size: req.size,
|
|
491
944
|
domain: req.domain,
|
|
492
945
|
doNotTrain: args['do-not-train'] !== false,
|
|
946
|
+
transmission: priorTransmission,
|
|
493
947
|
addedAt,
|
|
494
948
|
sealed: sealedBlock,
|
|
495
949
|
});
|
|
496
950
|
|
|
497
951
|
// Resolve destination — local-only NEVER lands in the tracked SSOT.
|
|
498
952
|
const corporaCardsDir = args['cards-dir'] ? path.resolve(cwd, args['cards-dir']) : DEFAULT_CARDS_DIR;
|
|
499
|
-
const localDir = args.out ? path.resolve(cwd, args.out) : cwd;
|
|
500
|
-
|
|
953
|
+
const localDir = args.out ? path.resolve(cwd, args.out) : (dataFile ? path.dirname(dataFile) : cwd);
|
|
954
|
+
// An npm install ships no corpora-cards directory (it is the monorepo's
|
|
955
|
+
// tracked SSOT). Writing a "registered" card into node_modules put it where
|
|
956
|
+
// nothing reads it and the next reinstall deletes it. From an install, the
|
|
957
|
+
// card stays with you, and listing it is a review-gated submission.
|
|
958
|
+
const installMode = !args['cards-dir'] && !fs.existsSync(DEFAULT_CARDS_DIR);
|
|
959
|
+
const dest = installMode && req.tier.key !== 'local-only'
|
|
960
|
+
? { registered: false, tracked: false, dir: localDir, filename: `${id}.json` }
|
|
961
|
+
: resolveDestination({ tier: req.tier, id, corporaCardsDir, localDir });
|
|
501
962
|
|
|
502
963
|
// Guard: a local-only card must never land in the tracked corpora-cards SSOT.
|
|
503
964
|
// path.resolve does NOT follow symlinks and only the exact dir was checked —
|
|
@@ -522,11 +983,32 @@ async function run(args, cwd) {
|
|
|
522
983
|
fs.mkdirSync(dest.dir, { recursive: true });
|
|
523
984
|
fs.writeFileSync(filePath, JSON.stringify(card, null, 2) + '\n', 'utf-8');
|
|
524
985
|
|
|
986
|
+
let sidecar = null;
|
|
987
|
+
if (dataFile) {
|
|
988
|
+
try {
|
|
989
|
+
sidecar = writeSidecar(dataFile, {
|
|
990
|
+
id,
|
|
991
|
+
name: req.name,
|
|
992
|
+
license: card.license.spdx,
|
|
993
|
+
tier: req.tier.key,
|
|
994
|
+
sha256: dataInfo.sha256,
|
|
995
|
+
rows: dataInfo.rows,
|
|
996
|
+
card: path.relative(path.dirname(dataFile), filePath) || path.basename(filePath),
|
|
997
|
+
registeredAt: addedAt,
|
|
998
|
+
...(req.tier.key === 'local-only' ? { transmission: 'local-only' } : {}),
|
|
999
|
+
});
|
|
1000
|
+
} catch (e) {
|
|
1001
|
+
console.error(`[ERR] ${e.message}`);
|
|
1002
|
+
return 1;
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
|
|
525
1006
|
// ── Report ──
|
|
526
1007
|
if (args.json) {
|
|
527
1008
|
console.log(JSON.stringify({
|
|
528
1009
|
ok: true,
|
|
529
1010
|
id,
|
|
1011
|
+
role: req.role || null,
|
|
530
1012
|
exposureTier: req.tier.key,
|
|
531
1013
|
registered: dest.registered,
|
|
532
1014
|
tracked: dest.tracked,
|
|
@@ -542,7 +1024,24 @@ async function run(args, cwd) {
|
|
|
542
1024
|
qualifierThreshold: sealedBlock.qualifierThreshold,
|
|
543
1025
|
} : {}),
|
|
544
1026
|
license: card.license,
|
|
1027
|
+
contamination: {
|
|
1028
|
+
risk: card.contamination.risk,
|
|
1029
|
+
// Where the file was compared with the public corpora (null: it could not be).
|
|
1030
|
+
comparedWith: req.contaminationCompared || null,
|
|
1031
|
+
...(req.contaminationNote && { note: req.contaminationNote }),
|
|
1032
|
+
},
|
|
545
1033
|
path: filePath,
|
|
1034
|
+
...(sidecar ? {
|
|
1035
|
+
data: dataFile,
|
|
1036
|
+
dataSha256: dataInfo.sha256,
|
|
1037
|
+
rows: dataInfo.rows,
|
|
1038
|
+
sidecar: sidecar.path,
|
|
1039
|
+
transmission: sidecar.transmission,
|
|
1040
|
+
} : {}),
|
|
1041
|
+
...(installMode && req.tier.key !== 'local-only' ? {
|
|
1042
|
+
listed: false,
|
|
1043
|
+
toList: 'champollion network submit --type dataset',
|
|
1044
|
+
} : {}),
|
|
546
1045
|
}, null, 2));
|
|
547
1046
|
return 0;
|
|
548
1047
|
}
|
|
@@ -551,12 +1050,50 @@ async function run(args, cwd) {
|
|
|
551
1050
|
output.ok(`${req.tier.key === 'local-only' ? 'Saved' : 'Registered'} corpus card: ${id}`);
|
|
552
1051
|
console.log('');
|
|
553
1052
|
console.log(` Exposure: ${req.tier.label}`);
|
|
554
|
-
console.log(`
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
1053
|
+
console.log(` Role: ${req.role || 'not stated (add --role test|dev|train to say what it is for)'}`);
|
|
1054
|
+
// Each term once, as the card states it (Round 14, researcher persona:
|
|
1055
|
+
// the card and this summary disagreed). The licence's terms; training
|
|
1056
|
+
// (doNotTrain); the transmission mark — a separate field, not a licence term.
|
|
1057
|
+
const yesNo = (b) => (b ? 'yes' : 'no');
|
|
1058
|
+
const custom = card.license.notes ? ' (custom/unconfirmed — verify its terms)' : '';
|
|
1059
|
+
const redistributed = { permitted: 'yes', 'same-terms': 'yes, under the same terms', prohibited: 'no' }[card.usageRestrictions.redistribution];
|
|
1060
|
+
console.log(` License: ${card.license.spdx}${custom} — commercial use ${yesNo(card.license.commercial)}, redistribution ${redistributed}`);
|
|
1061
|
+
console.log(` Training: ${card.doNotTrain
|
|
1062
|
+
? `not permitted — doNotTrain: true (${card.usageRestrictions.training === 'prohibited-by-license' ? 'the licence refuses training' : 'set at registration: a test set is not trained on'})`
|
|
1063
|
+
: 'permitted — doNotTrain: false'}`);
|
|
1064
|
+
if (card.transmission === 'local-only') {
|
|
1065
|
+
console.log(' Transmission: local-only — only a model on this machine may see it (a mark, not a licence term)');
|
|
1066
|
+
}
|
|
1067
|
+
console.log(` Pair: ${formatLanguagePair(req.pair)}`);
|
|
1068
|
+
const splitBlock = card.test || card.dev;
|
|
1069
|
+
console.log(` Size/domain: ${req.size} ${splitBlock.sizeUnit} · ${splitBlock.domain}${card.test ? ' (test split)' : ''}`);
|
|
1070
|
+
let contaminationAside = '';
|
|
1071
|
+
if (card.contamination.risk === CONTAMINATION_UNCHECKED) {
|
|
1072
|
+
contaminationAside = ` (${req.contaminationNote}. Not graded NONE: that would claim the text is unpublished. `
|
|
1073
|
+
+ 'Re-run online to compare it, or state a grade: --contamination none|low|medium|high)';
|
|
1074
|
+
} else if (req.contaminationNote) {
|
|
1075
|
+
contaminationAside = ` (${req.contaminationNote} — the grade is your statement)`;
|
|
1076
|
+
} else if (req.contaminationCompared && !req.contaminationReasoning) {
|
|
1077
|
+
contaminationAside = ` (compared with ${req.contaminationCompared}: no public copy of this file — the grade is your statement)`;
|
|
1078
|
+
}
|
|
1079
|
+
console.log(` Contamination: ${card.contamination.risk}${contaminationAside}`);
|
|
558
1080
|
console.log(` Written to: ${filePath}`);
|
|
1081
|
+
if (sidecar) {
|
|
1082
|
+
console.log(` Data file: ${dataFile} (${dataInfo.rows} rows, sha256 ${dataInfo.sha256.slice(0, 12)}…)`);
|
|
1083
|
+
console.log(` Sidecar: ${sidecar.path}`
|
|
1084
|
+
+ (sidecar.transmission === 'local-only' ? ' (carries the local-only mark the tools read)' : ''));
|
|
1085
|
+
}
|
|
559
1086
|
console.log('');
|
|
1087
|
+
const runTarget = dataFile ? shellQuote(path.relative(cwd, dataFile) || dataFile) : '<your file>';
|
|
1088
|
+
const lang = languageFlags(req.pair, cwd);
|
|
1089
|
+
const unnamedNote = lang.unnamed.length > 0
|
|
1090
|
+
? ` (${lang.unnamed.join(', ')} has no language card: replace the code after --${lang.unnamed.includes(req.pair.target) ? 'target' : 'source'}-lang with the language's name — it is what the model is told — or give it a "name" in champollion.config.json)`
|
|
1091
|
+
: null;
|
|
1092
|
+
// A set a model may be trained against: the nmt-forge steps come before
|
|
1093
|
+
// the baseline, which is a scoring read (see forgeFirstLines).
|
|
1094
|
+
const forgeFirst = mayBeTrainedAgainst(req)
|
|
1095
|
+
? forgeFirstLines({ pair: req.pair, dataFile, sha256: dataInfo ? dataInfo.sha256 : null, cardId: id, cwd, lang })
|
|
1096
|
+
: null;
|
|
560
1097
|
|
|
561
1098
|
if (req.tier.key === 'sealed') {
|
|
562
1099
|
console.log(` Sealed artifact: ${sealedArtifactPath}`);
|
|
@@ -574,16 +1111,56 @@ async function run(args, cwd) {
|
|
|
574
1111
|
console.log(' A method must first clear the paired PUBLIC qualifier before any sealed run');
|
|
575
1112
|
console.log(' can even be proposed — and that run still requires custodian approval.');
|
|
576
1113
|
} else if (req.tier.key === 'local-only') {
|
|
577
|
-
console.log(' This card stays on your machine. Nothing was registered or uploaded
|
|
578
|
-
|
|
1114
|
+
console.log(' This card stays on your machine. Nothing was registered or uploaded.');
|
|
1115
|
+
if (dataFile) {
|
|
1116
|
+
console.log(' Your text was read only to count and checksum it. Only a model on this');
|
|
1117
|
+
console.log(' machine can be tested against it; every remote provider is refused.');
|
|
1118
|
+
console.log('');
|
|
1119
|
+
if (forgeFirst && !forgeFirst.done) {
|
|
1120
|
+
for (const line of forgeFirst.lines) console.log(line);
|
|
1121
|
+
console.log('');
|
|
1122
|
+
console.log(' Then measure a local model on it (the baseline) — or now, if no model will');
|
|
1123
|
+
console.log(' be trained against this set:');
|
|
1124
|
+
} else {
|
|
1125
|
+
if (forgeFirst) { for (const line of forgeFirst.lines) console.log(line); console.log(''); }
|
|
1126
|
+
console.log(' Next: measure a local model on it:');
|
|
1127
|
+
}
|
|
1128
|
+
const model = localModelFor(cwd);
|
|
1129
|
+
console.log(` mt-eval run --corpus ${runTarget} --provider local --model ${model ? shellQuote(model) : '<model>'} ${lang.flags}`);
|
|
1130
|
+
if (!model) console.log(' (<model>: the name your local server serves, e.g. llama3.1)');
|
|
1131
|
+
if (unnamedNote) console.log(unnamedNote);
|
|
1132
|
+
} else {
|
|
1133
|
+
console.log(' Your corpus text was never read. To protect the file itself, re-run with');
|
|
1134
|
+
console.log(' --data <file>: that marks it local-only for mt-eval.');
|
|
1135
|
+
}
|
|
579
1136
|
} else if (req.tier.key === 'private') {
|
|
580
|
-
console.log(' Metadata only
|
|
581
|
-
console.log('
|
|
582
|
-
console.log('
|
|
1137
|
+
console.log(' Metadata only — your corpus TEXT was never uploaded or hosted, and never');
|
|
1138
|
+
console.log(' will be. A private set is quarantined from the public queue/leaderboard.');
|
|
1139
|
+
if (dataFile) console.log(' Your text was read on this machine only to count and checksum it.');
|
|
583
1140
|
console.log('');
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
1141
|
+
if (installMode) {
|
|
1142
|
+
console.log(' The card is kept here with you. To have it listed in the index (metadata');
|
|
1143
|
+
console.log(' only, review-gated), submit it: champollion network submit --type dataset');
|
|
1144
|
+
} else {
|
|
1145
|
+
console.log(' Rebuild the registry so the card is catalogued:');
|
|
1146
|
+
for (const line of REGISTRY_REBUILD_HINT) console.log(line);
|
|
1147
|
+
}
|
|
1148
|
+
console.log('');
|
|
1149
|
+
if (forgeFirst && !forgeFirst.done) {
|
|
1150
|
+
for (const line of forgeFirst.lines) console.log(line);
|
|
1151
|
+
console.log('');
|
|
1152
|
+
console.log(' Then evaluate on it (the baseline) — or now, if no model will be trained');
|
|
1153
|
+
console.log(' against this set — and publish the scores without the text if you choose:');
|
|
1154
|
+
} else {
|
|
1155
|
+
if (forgeFirst) { for (const line of forgeFirst.lines) console.log(line); console.log(''); }
|
|
1156
|
+
console.log(' Evaluate on it, then publish the scores without the text if you choose:');
|
|
1157
|
+
}
|
|
1158
|
+
console.log(` mt-eval run --corpus ${runTarget} --provider <provider> --model <model> ${lang.flags} -o results`);
|
|
1159
|
+
if (unnamedNote) console.log(unnamedNote);
|
|
1160
|
+
console.log(' mt-eval publish results/<run-id>_report.json --scores-only --dry-run');
|
|
1161
|
+
} else if (installMode) {
|
|
1162
|
+
console.log(' The card is kept here with you. A public corpus is listed through review:');
|
|
1163
|
+
console.log(' champollion network submit --type dataset');
|
|
587
1164
|
} else {
|
|
588
1165
|
console.log(' A metadata card + fetch-from-source pointer is registered — your corpus');
|
|
589
1166
|
console.log(' TEXT is never hosted by Champollion; it is fetched from source on demand.');
|
|
@@ -595,84 +1172,14 @@ async function run(args, cwd) {
|
|
|
595
1172
|
return 0;
|
|
596
1173
|
}
|
|
597
1174
|
|
|
1175
|
+
/**
|
|
1176
|
+
* `--help` for this command. The text lives in lib/command-help.js — the one
|
|
1177
|
+
* `champollion network register-corpus --help` prints. This module kept a
|
|
1178
|
+
* second copy, and the two drifted: Round 9 corrected what the file reads in
|
|
1179
|
+
* one, while the other still said "never reads" (Round 10, school persona).
|
|
1180
|
+
*/
|
|
598
1181
|
function showHelp() {
|
|
599
|
-
|
|
600
|
-
champollion register-corpus — Register a new evaluation corpus
|
|
601
|
-
|
|
602
|
-
USAGE
|
|
603
|
-
champollion register-corpus [options]
|
|
604
|
-
|
|
605
|
-
DESCRIPTION
|
|
606
|
-
Register a corpus with YOU in control of its license and how far it
|
|
607
|
-
travels. Three exposure tiers, defaulting to the most private. We never
|
|
608
|
-
read, upload, or host your corpus text in ANY tier — only metadata (and,
|
|
609
|
-
for public sets, a pointer to where the data is fetched from).
|
|
610
|
-
|
|
611
|
-
Interactive when run in a terminal; fully scriptable with flags (or --yes).
|
|
612
|
-
|
|
613
|
-
EXPOSURE TIERS (--tier, default: local-only)
|
|
614
|
-
local-only Never registered, never uploaded; card + text stay on your machine.
|
|
615
|
-
private Register METADATA ONLY — a WMT-style sovereign held-out set;
|
|
616
|
-
text never uploaded or hosted; you keep custody.
|
|
617
|
-
public Publish a metadata card + fetch-from-source pointer; text never
|
|
618
|
-
hosted by us. Requires a redistribution-cleared license (gated).
|
|
619
|
-
sealed Encrypt the corpus ON YOUR DEVICE to the custodian group's
|
|
620
|
-
key; the ciphertext stays with you and we receive only a
|
|
621
|
-
content-free card. Catalogued + quarantined; paired with a
|
|
622
|
-
public qualifier a method must clear before any sealed run.
|
|
623
|
-
|
|
624
|
-
LICENSES (--license)
|
|
625
|
-
cc-by-4.0, cc-by-sa-4.0, cc0-1.0, cc-by-nc-4.0, proprietary, other
|
|
626
|
-
(or pass any SPDX id; it is classified through the license gate)
|
|
627
|
-
|
|
628
|
-
OPTIONS
|
|
629
|
-
--tier <tier> Exposure tier (default: local-only). Alias: --exposure
|
|
630
|
-
--license <id> License key, SPDX id, or list number (see --list)
|
|
631
|
-
--name <text> Corpus name (required)
|
|
632
|
-
--pair <pair> Language pair: "eng>crk" (quote it — the shell eats a
|
|
633
|
-
bare >) or the dash form eng-crk (or --source-lang/--target-lang)
|
|
634
|
-
--publisher <text> Publisher / your name or org
|
|
635
|
-
--description <text> One-line description
|
|
636
|
-
--size <n> Number of sentence pairs (required)
|
|
637
|
-
--domain <text> Domain: news, conversational, educational, … (required)
|
|
638
|
-
--contamination <lvl> NONE | LOW | MEDIUM | HIGH (default: NONE private / LOW public)
|
|
639
|
-
--repo-url <url> Public tier: fetch-from-source archive/repo URL
|
|
640
|
-
--source-url <url> Public tier: canonical project/dataset URL
|
|
641
|
-
--builder <id> Public tier: builder adapter id (rebuilds from source)
|
|
642
|
-
--seal-input <path> Sealed tier: local corpus file to encrypt on-device
|
|
643
|
-
--threshold-pubkey <k> Sealed tier: custodian group threshold public key
|
|
644
|
-
(file path, or inline PEM / base64 DER SPKI, X25519)
|
|
645
|
-
--custodian-group <id> Sealed tier: custodian group id that holds the key
|
|
646
|
-
--seal-out <path> Sealed tier: where to write the ciphertext artifact
|
|
647
|
-
--qualifier-id <id> Sealed tier: paired public qualifier card id (vYYYY)
|
|
648
|
-
--qualifier-threshold <n> Sealed tier: score a method must clear on the qualifier
|
|
649
|
-
--key-scheme <s> Sealed tier: custody scheme label (default: TSS-3-of-5)
|
|
650
|
-
--id <id> Override the generated card id (eval-…)
|
|
651
|
-
--out <dir> local-only: where to write the card (default: cwd)
|
|
652
|
-
--list Print the license + tier catalog (add --json for JSON)
|
|
653
|
-
--json Machine-readable output (for agents)
|
|
654
|
-
--yes Non-interactive; take values from flags
|
|
655
|
-
|
|
656
|
-
EXAMPLES
|
|
657
|
-
champollion register-corpus # interactive wizard
|
|
658
|
-
champollion register-corpus --list # see licenses + tiers
|
|
659
|
-
# Keep a corpus entirely local (nothing leaves your machine):
|
|
660
|
-
champollion register-corpus --yes --name "My set" --pair "eng>crk" \\
|
|
661
|
-
--license cc-by-4.0 --tier local-only --size 200 --domain news
|
|
662
|
-
# Register a sovereign held-out test set (metadata only, text never uploaded):
|
|
663
|
-
champollion register-corpus --yes --name "Community holdout" --pair "eng>crk" \\
|
|
664
|
-
--license cc-by-nc-4.0 --tier private --size 500 --domain educational
|
|
665
|
-
# Publish a fetch-from-source pointer (gated to redistributable licenses):
|
|
666
|
-
champollion register-corpus --yes --name "Tatoeba eng-crk" --pair "eng-crk" \\
|
|
667
|
-
--license cc-by-4.0 --tier public --size 1000 --domain conversational \\
|
|
668
|
-
--repo-url https://example.org/data.tar --builder tatoeba-challenge
|
|
669
|
-
# Seal a community-controlled secret test set (encrypted ON YOUR DEVICE):
|
|
670
|
-
champollion register-corpus --yes --name "Sealed eng-crk" --pair "eng>crk" \\
|
|
671
|
-
--license proprietary --tier sealed --size 500 --domain educational \\
|
|
672
|
-
--seal-input ./secret-corpus.json --threshold-pubkey ./group.pub \\
|
|
673
|
-
--custodian-group nehiyawewin-trust --seal-out ./sealed.json \\
|
|
674
|
-
--qualifier-id eval-eng-crk-nehiyaw-qualifier-v2026
|
|
675
|
-
`);
|
|
1182
|
+
showCommandHelp('register-corpus');
|
|
676
1183
|
}
|
|
677
1184
|
|
|
678
1185
|
/**
|