champollion 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +133 -0
- package/README.md +387 -0
- package/bin/cli.js +278 -0
- package/index.js +135 -0
- package/lib/api-key.js +127 -0
- package/lib/autofix.js +432 -0
- package/lib/bridge/method_bridge.py +430 -0
- package/lib/card-source-resolution.mjs +284 -0
- package/lib/cards/cache.js +169 -0
- package/lib/cards/env.js +82 -0
- package/lib/cards/fetch-card-child.js +38 -0
- package/lib/cards/reader.js +435 -0
- package/lib/cards/refresh.js +111 -0
- package/lib/cards/remote.js +387 -0
- package/lib/cldf-export.mjs +540 -0
- package/lib/cldf-terms.mjs +62 -0
- package/lib/command-help.js +790 -0
- package/lib/commands/audit.js +49 -0
- package/lib/commands/card.js +454 -0
- package/lib/commands/doctor.js +559 -0
- package/lib/commands/fonts.js +489 -0
- package/lib/commands/help.js +91 -0
- package/lib/commands/init.js +1259 -0
- package/lib/commands/integrity.js +148 -0
- package/lib/commands/leaderboard.js +478 -0
- package/lib/commands/lint.js +30 -0
- package/lib/commands/models.js +177 -0
- package/lib/commands/plugin.js +103 -0
- package/lib/commands/provenance.js +45 -0
- package/lib/commands/recommend.js +75 -0
- package/lib/commands/register-corpus.js +678 -0
- package/lib/commands/repair-script.js +42 -0
- package/lib/commands/seal-corpus.js +355 -0
- package/lib/commands/seo.js +72 -0
- package/lib/commands/serve.js +147 -0
- package/lib/commands/status.js +265 -0
- package/lib/commands/submit.js +332 -0
- package/lib/commands/sync.js +89 -0
- package/lib/commands/tm.js +573 -0
- package/lib/commands/verify.js +39 -0
- package/lib/commands/watch.js +20 -0
- package/lib/commands/wrap.js +138 -0
- package/lib/commands/xliff.js +327 -0
- package/lib/commercial-eligibility.js +235 -0
- package/lib/concurrent.js +87 -0
- package/lib/config.js +523 -0
- package/lib/contamination-lane.js +76 -0
- package/lib/content-sync.js +731 -0
- package/lib/content.js +733 -0
- package/lib/corpus-registration.mjs +608 -0
- package/lib/cost-report.js +346 -0
- package/lib/diff.js +155 -0
- package/lib/docusaurus-sync.js +1256 -0
- package/lib/flatten.js +55 -0
- package/lib/format.js +954 -0
- package/lib/hash.js +159 -0
- package/lib/icu.js +473 -0
- package/lib/integrity.js +689 -0
- package/lib/license-gate.mjs +478 -0
- package/lib/license-identify.mjs +229 -0
- package/lib/lint.js +629 -0
- package/lib/method-manifest.js +60 -0
- package/lib/methods/anthropic.js +140 -0
- package/lib/methods/apertium.js +163 -0
- package/lib/methods/api.js +316 -0
- package/lib/methods/base.js +184 -0
- package/lib/methods/content-separator.js +45 -0
- package/lib/methods/deepl.js +426 -0
- package/lib/methods/direct-llm.js +586 -0
- package/lib/methods/external.js +332 -0
- package/lib/methods/fetch-with-retry.js +124 -0
- package/lib/methods/gemini.js +147 -0
- package/lib/methods/google-translate.js +402 -0
- package/lib/methods/http-utils.js +122 -0
- package/lib/methods/libretranslate.js +314 -0
- package/lib/methods/llm-coached.js +670 -0
- package/lib/methods/llm.js +592 -0
- package/lib/methods/local.js +76 -0
- package/lib/methods/microsoft-translator.js +331 -0
- package/lib/methods/openai.js +131 -0
- package/lib/methods/openrouter-client.js +327 -0
- package/lib/methods/openrouter-pricing.js +156 -0
- package/lib/methods/provider-env.js +115 -0
- package/lib/methods/provider-pricing.js +310 -0
- package/lib/methods/tilde.js +150 -0
- package/lib/methods/translated.js +229 -0
- package/lib/methods/translation-error.js +80 -0
- package/lib/models.js +258 -0
- package/lib/no-translate.js +233 -0
- package/lib/output.js +238 -0
- package/lib/pairs.js +547 -0
- package/lib/plugins.js +447 -0
- package/lib/provenance.js +323 -0
- package/lib/recommend.js +648 -0
- package/lib/registers.js +1185 -0
- package/lib/repair-script.js +266 -0
- package/lib/scripts.js +994 -0
- package/lib/seal.mjs +464 -0
- package/lib/sealed-qualifier.mjs +211 -0
- package/lib/security.js +59 -0
- package/lib/segment.js +369 -0
- package/lib/seo.js +275 -0
- package/lib/serve.js +854 -0
- package/lib/string-classify.js +85 -0
- package/lib/submit.mjs +344 -0
- package/lib/sync.js +969 -0
- package/lib/tags/bcp47.js +202 -0
- package/lib/tags/resolve.js +314 -0
- package/lib/terminology.js +111 -0
- package/lib/tm-seed.js +294 -0
- package/lib/tm.js +515 -0
- package/lib/translate-pair.js +197 -0
- package/lib/translate.js +203 -0
- package/lib/types.js +230 -0
- package/lib/validate.js +510 -0
- package/lib/verify.js +451 -0
- package/lib/watch.js +145 -0
- package/lib/xliff.js +184 -0
- package/package.json +93 -0
- package/shared/ATTRIBUTION.md +145 -0
- package/shared/CORPORA-CARDS.md +288 -0
- package/shared/DATA-SOVEREIGNTY.md +500 -0
- package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
- package/shared/card-lint-baseline.json +3189 -0
- package/shared/cards-fallback.json +1 -0
- package/shared/catalogue/card-config.json +6091 -0
- package/shared/catalogue/external-results.json +3888 -0
- package/shared/catalogue/gender-guidance.json +1038 -0
- package/shared/catalogue/method-coverage.json +1751 -0
- package/shared/catalogue/metric-coverage.json +170 -0
- package/shared/catalogue/metric-reliability.json +1 -0
- package/shared/catalogue/register-presets.json +3180 -0
- package/shared/catalogue/vitality-scales.json +55 -0
- package/shared/cldr-index.json +1115 -0
- package/shared/code-bridge.json +253 -0
- package/shared/corpora-cards-v1-reference.md +281 -0
- package/shared/curated-dictionary-flags.json +35 -0
- package/shared/curated-endonyms.json +35 -0
- package/shared/curated-fsts.json +51 -0
- package/shared/curated-orthography-conventions.json +26 -0
- package/shared/curated-sil-resources.json +374 -0
- package/shared/curated-tools.json +41 -0
- package/shared/docent/corpus.json +11333 -0
- package/shared/docent/faq.en.json +564 -0
- package/shared/docent/register-blocks.json +60 -0
- package/shared/docent/system-prompt.md +144 -0
- package/shared/domain-taxonomy.json +35 -0
- package/shared/explainers/glossary.json +2975 -0
- package/shared/explainers/tc-features.json +20112 -0
- package/shared/explainers/term-watchlist.json +147 -0
- package/shared/human-services.json +59 -0
- package/shared/license-corrections.json +261 -0
- package/shared/license-evidence.json +13452 -0
- package/shared/licenses.json +6781 -0
- package/shared/method-registry.json +236 -0
- package/shared/metric-registry.json +620 -0
- package/shared/model-aliases.json +7 -0
- package/shared/schemas/champollion-plugin.schema.json +206 -0
- package/shared/schemas/corpora-card.schema.json +957 -0
- package/shared/schemas/domain-taxonomy.schema.json +64 -0
- package/shared/schemas/external-results.schema.json +314 -0
- package/shared/schemas/human-services.schema.json +90 -0
- package/shared/schemas/language-card.schema.json +1308 -0
- package/shared/schemas/licenses.schema.json +155 -0
- package/shared/schemas/method-card.schema.json +412 -0
- package/shared/schemas/method-registry.schema.json +85 -0
- package/shared/schemas/metric-registry.schema.json +96 -0
- package/shared/schemas/metric-reliability.schema.json +178 -0
- package/shared/schemas/model-aliases.schema.json +27 -0
- package/shared/schemas/source-snapshot.schema.json +96 -0
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* corpus-registration.mjs — the licensing + exposure logic behind
|
|
3
|
+
* `champollion register-corpus`.
|
|
4
|
+
*
|
|
5
|
+
* This module is the SSOT for two author-facing choices made when a new corpus
|
|
6
|
+
* is registered, and the rules that bind them:
|
|
7
|
+
*
|
|
8
|
+
* 1. LICENSE — a plain-language picklist (CC-BY / CC-BY-SA / CC0 / CC-BY-NC /
|
|
9
|
+
* proprietary / other-custom) that also yields the commercial + redistribution
|
|
10
|
+
* booleans written onto the card's `license` block.
|
|
11
|
+
* 2. EXPOSURE — one of three tiers, defaulting to the most private:
|
|
12
|
+
* • local-only — never registered, never uploaded; card + text stay on
|
|
13
|
+
* the author's machine (the card is written OUTSIDE the tracked
|
|
14
|
+
* corpora-cards directory).
|
|
15
|
+
* • private — a WMT-style sovereign held-out / secret test set:
|
|
16
|
+
* register METADATA ONLY (quarantined), text never uploaded or hosted.
|
|
17
|
+
* • public — publish a metadata card + a fetch-from-source pointer;
|
|
18
|
+
* text never hosted by us. Gated by license-gate.mjs.
|
|
19
|
+
*
|
|
20
|
+
* DOCTRINE (the data-boundaries doctrine): Champollion NEVER hosts or redistributes
|
|
21
|
+
* corpus CONTENT in ANY tier — only metadata cards and fetch-from-source
|
|
22
|
+
* pointers. Every function here is content-free: it produces metadata, never
|
|
23
|
+
* source/reference text. The public tier is gated through the existing
|
|
24
|
+
* license/sovereignty gate (./license-gate.mjs) so NC / no-redistribute /
|
|
25
|
+
* unconfirmed-license sets cannot enter public/commercial/ranked lanes.
|
|
26
|
+
*
|
|
27
|
+
* Pure (no I/O, no process state) so the command layer and the test-suite share
|
|
28
|
+
* exactly one implementation.
|
|
29
|
+
*
|
|
30
|
+
* @module corpus-registration
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { classifyLicense, TIERS } from './license-gate.mjs';
|
|
34
|
+
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
// LICENSE OPTIONS — the plain-language picklist. `commercial` / `redistribution`
|
|
37
|
+
// are the booleans written onto the card's `license` block (the license's plain
|
|
38
|
+
// terms for OUR lanes). `publicEligible` is the author-facing hint; the real
|
|
39
|
+
// gate is gatePublicRegistration(), which cross-checks license-gate.mjs and
|
|
40
|
+
// fails safe to blocked.
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
export const LICENSE_OPTIONS = [
|
|
43
|
+
{
|
|
44
|
+
key: 'cc-by-4.0',
|
|
45
|
+
label: 'CC-BY-4.0',
|
|
46
|
+
spdx: 'CC-BY-4.0',
|
|
47
|
+
commercial: true,
|
|
48
|
+
redistribution: true,
|
|
49
|
+
publicEligible: true,
|
|
50
|
+
explanation: 'Anyone may use and share it, even commercially, as long as they credit you.',
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
key: 'cc-by-sa-4.0',
|
|
54
|
+
label: 'CC-BY-SA-4.0',
|
|
55
|
+
spdx: 'CC-BY-SA-4.0',
|
|
56
|
+
commercial: true,
|
|
57
|
+
redistribution: true,
|
|
58
|
+
publicEligible: true,
|
|
59
|
+
explanation: 'Like CC-BY, but anything built on it must be shared under the same license (share-alike).',
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
key: 'cc0-1.0',
|
|
63
|
+
label: 'CC0 / public domain',
|
|
64
|
+
spdx: 'CC0-1.0',
|
|
65
|
+
commercial: true,
|
|
66
|
+
redistribution: true,
|
|
67
|
+
publicEligible: true,
|
|
68
|
+
explanation: 'Public-domain dedication — no rights reserved. Anyone may do anything; no credit required.',
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
key: 'cc-by-nc-4.0',
|
|
72
|
+
label: 'CC-BY-NC-4.0',
|
|
73
|
+
spdx: 'CC-BY-NC-4.0',
|
|
74
|
+
commercial: false,
|
|
75
|
+
redistribution: false,
|
|
76
|
+
publicEligible: false,
|
|
77
|
+
explanation: 'Free for non-commercial use only. Cannot enter the public ranked or commercial lanes — register privately or local-only.',
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
key: 'cc-by-nc-sa-4.0',
|
|
81
|
+
label: 'CC-BY-NC-SA-4.0',
|
|
82
|
+
spdx: 'CC-BY-NC-SA-4.0',
|
|
83
|
+
commercial: false,
|
|
84
|
+
redistribution: true,
|
|
85
|
+
publicEligible: false,
|
|
86
|
+
explanation: 'Non-commercial, and anything built on it must be shared the same way. May be shared, but never in a commercial lane.',
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
key: 'cc-by-nd-4.0',
|
|
90
|
+
label: 'CC-BY-ND-4.0',
|
|
91
|
+
spdx: 'CC-BY-ND-4.0',
|
|
92
|
+
commercial: true,
|
|
93
|
+
redistribution: true,
|
|
94
|
+
publicEligible: false,
|
|
95
|
+
explanation: 'May be shared verbatim, even commercially, but no modified versions may be distributed. Derivative eval sets are not permitted.',
|
|
96
|
+
},
|
|
97
|
+
// ---- The steward-terms options -----------------------------------------
|
|
98
|
+
// THE MISSING CELL (added 2026-08-13). Every option above couples commercial
|
|
99
|
+
// use to redistribution: a steward who wants to keep control of their corpus
|
|
100
|
+
// could only reach for NC, which excludes the corpus from the commercial
|
|
101
|
+
// lane forever. That forced a false choice between sovereignty and ever
|
|
102
|
+
// being measured for a paying user.
|
|
103
|
+
//
|
|
104
|
+
// These two separate the rights that are genuinely separate. Ownership is
|
|
105
|
+
// never in the picklist because it is never negotiated: the corpus belongs
|
|
106
|
+
// to its steward in every option here.
|
|
107
|
+
{
|
|
108
|
+
key: 'community-eval-grant',
|
|
109
|
+
label: 'Steward terms — evaluation only (commercial evaluation permitted)',
|
|
110
|
+
spdx: 'LicenseRef-Champollion-Eval-Grant',
|
|
111
|
+
commercial: true,
|
|
112
|
+
redistribution: false,
|
|
113
|
+
aiTraining: false,
|
|
114
|
+
publicEligible: false,
|
|
115
|
+
explanation: 'You keep ownership. The corpus is never redistributed, never published, and never used to TRAIN anything — it is used only to score systems, including when someone pays for that scoring. Revocable.',
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
key: 'community-eval-grant-nc',
|
|
119
|
+
label: 'Steward terms — evaluation only, non-commercial',
|
|
120
|
+
spdx: 'LicenseRef-Champollion-Eval-Grant-NC',
|
|
121
|
+
commercial: false,
|
|
122
|
+
redistribution: false,
|
|
123
|
+
aiTraining: false,
|
|
124
|
+
publicEligible: false,
|
|
125
|
+
explanation: 'As above, but scoring for paying users is NOT permitted — research and community lanes only. The most conservative grant that still allows measurement.',
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
key: 'proprietary',
|
|
129
|
+
label: 'Proprietary / all rights reserved',
|
|
130
|
+
spdx: 'LicenseRef-Proprietary',
|
|
131
|
+
commercial: false,
|
|
132
|
+
redistribution: false,
|
|
133
|
+
publicEligible: false,
|
|
134
|
+
explanation: 'All rights reserved. You keep full control; it cannot be publicly redistributed — register privately or local-only.',
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
key: 'other',
|
|
138
|
+
label: 'Other / custom (unconfirmed)',
|
|
139
|
+
spdx: 'LicenseRef-Custom',
|
|
140
|
+
commercial: false,
|
|
141
|
+
redistribution: false,
|
|
142
|
+
publicEligible: false,
|
|
143
|
+
explanation: 'A custom or unconfirmed license. Treated as not-redistributable until confirmed — private or local-only only.',
|
|
144
|
+
},
|
|
145
|
+
];
|
|
146
|
+
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
// EXPOSURE TIERS — chosen explicitly, defaulting to the most private. Note
|
|
149
|
+
// `uploadsContent` is false for EVERY tier: we NEVER host or receive plaintext
|
|
150
|
+
// corpus content anywhere. The `sealed` tier may produce an encrypted
|
|
151
|
+
// (ciphertext-only) artifact — flagged by `uploadsCiphertext`, distinct from
|
|
152
|
+
// content: ciphertext is not readable content, and we cannot decrypt it.
|
|
153
|
+
// `tracked` = whether the metadata card lands in the tracked corpora-cards SSOT.
|
|
154
|
+
// ---------------------------------------------------------------------------
|
|
155
|
+
export const EXPOSURE_TIERS = [
|
|
156
|
+
{
|
|
157
|
+
key: 'local-only',
|
|
158
|
+
label: 'Private / local-only',
|
|
159
|
+
registers: false,
|
|
160
|
+
uploadsContent: false,
|
|
161
|
+
uploadsCiphertext: false,
|
|
162
|
+
tracked: false,
|
|
163
|
+
explanation:
|
|
164
|
+
'Never registered, never uploaded. The card and your text stay entirely on your machine. The safe default.',
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
key: 'private',
|
|
168
|
+
label: 'Register privately (sovereign held-out)',
|
|
169
|
+
registers: true,
|
|
170
|
+
uploadsContent: false,
|
|
171
|
+
uploadsCiphertext: false,
|
|
172
|
+
tracked: true,
|
|
173
|
+
explanation:
|
|
174
|
+
'Registers METADATA ONLY — a WMT-style secret/held-out test set. Your text is NEVER uploaded or hosted; you keep custody. Results can be published without exposing the data.',
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
key: 'public',
|
|
178
|
+
label: 'Register publicly (fetch-from-source)',
|
|
179
|
+
registers: true,
|
|
180
|
+
uploadsContent: false,
|
|
181
|
+
uploadsCiphertext: false,
|
|
182
|
+
tracked: true,
|
|
183
|
+
explanation:
|
|
184
|
+
'Publishes a metadata card + a fetch-from-source pointer. Your text is NEVER hosted by Champollion — it is fetched from the upstream source on demand. Requires a redistribution-cleared license.',
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
key: 'sealed',
|
|
188
|
+
label: 'Seal it (community-controlled secret test)',
|
|
189
|
+
registers: true,
|
|
190
|
+
uploadsContent: false, // NEVER plaintext content
|
|
191
|
+
uploadsCiphertext: true, // an encrypted, content-free artifact MAY be produced
|
|
192
|
+
tracked: true,
|
|
193
|
+
explanation:
|
|
194
|
+
'Encrypts your corpus ON YOUR DEVICE under the custodian group’s threshold key BEFORE a single byte leaves. Champollion receives ciphertext + a content-free card only and CANNOT decrypt it (no single party can — it takes M-of-N custodians). Catalogued but quarantined; pair it with a public qualifier that methods must clear before any sealed run can be proposed.',
|
|
195
|
+
},
|
|
196
|
+
];
|
|
197
|
+
|
|
198
|
+
/** The most-private tier — the default whenever the author does not choose. */
|
|
199
|
+
export const DEFAULT_TIER = 'local-only';
|
|
200
|
+
|
|
201
|
+
// ---------------------------------------------------------------------------
|
|
202
|
+
// Resolvers — turn a flag value (key / label / spdx / 1-based number) into the
|
|
203
|
+
// catalogue entry, tolerant enough for both humans and agents.
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Resolve a license choice to a normalized option object.
|
|
208
|
+
*
|
|
209
|
+
* Accepts: a catalogue key ('cc-by-4.0'), a label, an SPDX id ('CC-BY-4.0'),
|
|
210
|
+
* a 1-based index ('1'), or ANY other SPDX string (treated as custom and
|
|
211
|
+
* classified through license-gate.mjs). Returns null only for empty input.
|
|
212
|
+
*
|
|
213
|
+
* @param {string} input
|
|
214
|
+
* @returns {object|null} { key, label, spdx, commercial, redistribution, publicEligible, explanation, custom? }
|
|
215
|
+
*/
|
|
216
|
+
export function resolveLicense(input) {
|
|
217
|
+
if (input === undefined || input === null) return null;
|
|
218
|
+
const raw = String(input).trim();
|
|
219
|
+
if (!raw) return null;
|
|
220
|
+
const norm = raw.toLowerCase();
|
|
221
|
+
|
|
222
|
+
// 1-based numeric index into the picklist
|
|
223
|
+
if (/^\d+$/.test(raw)) {
|
|
224
|
+
const idx = parseInt(raw, 10) - 1;
|
|
225
|
+
if (idx >= 0 && idx < LICENSE_OPTIONS.length) return LICENSE_OPTIONS[idx];
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// Exact match against key / label / spdx (case-insensitive)
|
|
229
|
+
const hit = LICENSE_OPTIONS.find(
|
|
230
|
+
(o) =>
|
|
231
|
+
o.key === norm ||
|
|
232
|
+
o.label.toLowerCase() === norm ||
|
|
233
|
+
o.spdx.toLowerCase() === norm,
|
|
234
|
+
);
|
|
235
|
+
if (hit) return hit;
|
|
236
|
+
|
|
237
|
+
// Anything else: a raw SPDX the author typed. Classify it through the gate so
|
|
238
|
+
// a real permissive/share-alike id still works, but unknowns fail safe.
|
|
239
|
+
const cls = classifyLicense({ license_spdx: raw });
|
|
240
|
+
const permissive = cls.tier === TIERS.PERMISSIVE;
|
|
241
|
+
const shareAlike = cls.tier === TIERS.SHAREALIKE;
|
|
242
|
+
return {
|
|
243
|
+
key: 'custom',
|
|
244
|
+
label: raw,
|
|
245
|
+
spdx: raw,
|
|
246
|
+
commercial: permissive || shareAlike,
|
|
247
|
+
redistribution: permissive || shareAlike,
|
|
248
|
+
publicEligible: permissive || shareAlike,
|
|
249
|
+
explanation: `Custom license "${raw}" classified as ${cls.tier} by the license gate.`,
|
|
250
|
+
custom: true,
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Resolve an exposure-tier choice to a tier object. Accepts a key, a label, a
|
|
256
|
+
* 1-based index, or the aliases 'local'/'secret'/'sovereign'. Defaults to the
|
|
257
|
+
* most private tier for empty/unknown input — fail-private.
|
|
258
|
+
*
|
|
259
|
+
* @param {string} input
|
|
260
|
+
* @returns {object} an EXPOSURE_TIERS entry (never null; defaults to local-only)
|
|
261
|
+
*/
|
|
262
|
+
export function resolveTier(input) {
|
|
263
|
+
const fallback = EXPOSURE_TIERS.find((t) => t.key === DEFAULT_TIER);
|
|
264
|
+
if (input === undefined || input === null) return fallback;
|
|
265
|
+
const raw = String(input).trim();
|
|
266
|
+
if (!raw) return fallback;
|
|
267
|
+
const norm = raw.toLowerCase();
|
|
268
|
+
|
|
269
|
+
if (/^\d+$/.test(raw)) {
|
|
270
|
+
const idx = parseInt(raw, 10) - 1;
|
|
271
|
+
if (idx >= 0 && idx < EXPOSURE_TIERS.length) return EXPOSURE_TIERS[idx];
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const aliases = { local: 'local-only', secret: 'private', sovereign: 'private', seal: 'sealed', encrypted: 'sealed' };
|
|
275
|
+
const key = aliases[norm] || norm;
|
|
276
|
+
return EXPOSURE_TIERS.find((t) => t.key === key || t.label.toLowerCase() === norm) || fallback;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* The card-level `license` block derived from a resolved license option.
|
|
281
|
+
* @param {object} option a resolveLicense() result
|
|
282
|
+
* `aiTraining` stays `null` — "not determined by the licence choice" — unless
|
|
283
|
+
* the option states it. Only the steward-terms options do, because they are
|
|
284
|
+
* the only ones where refusing training use is an explicit part of the grant
|
|
285
|
+
* rather than an inference from an SPDX id.
|
|
286
|
+
*
|
|
287
|
+
* @returns {{spdx:string, commercial:boolean, redistribution:boolean, aiTraining:(boolean|null), notes:(string|null)}}
|
|
288
|
+
*/
|
|
289
|
+
export function deriveLicenseBlock(option) {
|
|
290
|
+
return {
|
|
291
|
+
spdx: option.spdx,
|
|
292
|
+
commercial: option.commercial,
|
|
293
|
+
redistribution: option.redistribution,
|
|
294
|
+
aiTraining: option.aiTraining === undefined ? null : option.aiTraining,
|
|
295
|
+
notes: option.custom ? 'Custom/unconfirmed license — verify terms before any commercial or redistribution use.' : null,
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Express any combination of the three rights, for a steward whose terms do
|
|
301
|
+
* not match a preset.
|
|
302
|
+
*
|
|
303
|
+
* The picklist is presets, not a fence. A steward may permit commercial
|
|
304
|
+
* evaluation while refusing redistribution, or permit redistribution while
|
|
305
|
+
* refusing training, or any other combination — the rights are orthogonal and
|
|
306
|
+
* `license-gate.mjs` has always modelled them that way. What was hemmed in was
|
|
307
|
+
* the *picklist*, not the model.
|
|
308
|
+
*
|
|
309
|
+
* The public tier is deliberately NOT settable here: whether a corpus may be
|
|
310
|
+
* published is decided by gatePublicRegistration(), which re-classifies the
|
|
311
|
+
* SPDX through the licence gate and fails safe. A steward can grant rights;
|
|
312
|
+
* nobody can assert their way past the public gate.
|
|
313
|
+
*
|
|
314
|
+
* @param {object} option a resolveLicense() result to start from
|
|
315
|
+
* @param {object} rights
|
|
316
|
+
* @param {boolean} [rights.commercial]
|
|
317
|
+
* @param {boolean} [rights.redistribution]
|
|
318
|
+
* @param {boolean} [rights.aiTraining]
|
|
319
|
+
* @param {string} [rights.note] why the steward set these terms
|
|
320
|
+
* @returns {object} a new option; the original is not mutated
|
|
321
|
+
*/
|
|
322
|
+
export function withStewardTerms(option, rights = {}) {
|
|
323
|
+
if (!option) throw new Error('withStewardTerms requires a resolved license option.');
|
|
324
|
+
const next = { ...option, stewardTerms: true };
|
|
325
|
+
for (const k of ['commercial', 'redistribution', 'aiTraining']) {
|
|
326
|
+
if (typeof rights[k] === 'boolean') next[k] = rights[k];
|
|
327
|
+
}
|
|
328
|
+
// Granting a right never widens the public lane — that stays with the gate.
|
|
329
|
+
next.publicEligible = option.publicEligible === true && next.redistribution === true;
|
|
330
|
+
if (rights.note) next.explanation = rights.note;
|
|
331
|
+
return next;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// ---------------------------------------------------------------------------
|
|
335
|
+
// THE PUBLIC-REGISTRATION GATE — the one rule that protects the open lane.
|
|
336
|
+
// Authoritative: re-classifies the chosen SPDX through license-gate.mjs and
|
|
337
|
+
// only admits PERMISSIVE or SHARE-ALIKE licenses to the public tier. NC,
|
|
338
|
+
// no-derivatives, no-redistribute, and any unconfirmed/unknown license are
|
|
339
|
+
// blocked. Belt-and-suspenders: the picklist hint must ALSO agree (fail-safe).
|
|
340
|
+
// ---------------------------------------------------------------------------
|
|
341
|
+
|
|
342
|
+
/** Tiers a public fetch-from-source pointer is allowed to carry. */
|
|
343
|
+
const PUBLIC_OK_TIERS = new Set([TIERS.PERMISSIVE, TIERS.SHAREALIKE]);
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* May a corpus under this license enter the PUBLIC tier (a published
|
|
347
|
+
* fetch-from-source pointer on the open leaderboard)?
|
|
348
|
+
*
|
|
349
|
+
* @param {object} option a resolveLicense() result
|
|
350
|
+
* @returns {{allowed:boolean, tier:string, reason:string}}
|
|
351
|
+
*/
|
|
352
|
+
export function gatePublicRegistration(option) {
|
|
353
|
+
if (!option) {
|
|
354
|
+
return { allowed: false, tier: TIERS.UNKNOWN, reason: 'No license selected — a confirmed license is required for public registration.' };
|
|
355
|
+
}
|
|
356
|
+
const cls = classifyLicense({
|
|
357
|
+
license_spdx: option.spdx,
|
|
358
|
+
non_commercial_only: option.commercial === false,
|
|
359
|
+
// Only assert redistribution to the classifier when the option says so;
|
|
360
|
+
// unconfirmed options leave it unset so the classifier can fail safe.
|
|
361
|
+
...(option.redistribution === true ? { allows_redistribution: 1 } : {}),
|
|
362
|
+
});
|
|
363
|
+
const tierOk = PUBLIC_OK_TIERS.has(cls.tier);
|
|
364
|
+
const hintOk = option.publicEligible !== false;
|
|
365
|
+
const allowed = tierOk && hintOk;
|
|
366
|
+
|
|
367
|
+
let reason;
|
|
368
|
+
if (allowed) {
|
|
369
|
+
reason =
|
|
370
|
+
cls.tier === TIERS.SHAREALIKE
|
|
371
|
+
? `Share-alike license (${option.spdx}) — public OK; downstream must share alike.`
|
|
372
|
+
: `Redistribution-cleared license (${option.spdx}) — public OK.`;
|
|
373
|
+
} else if (cls.tier === TIERS.NONCOMMERCIAL) {
|
|
374
|
+
reason = `Non-commercial license (${option.spdx}) — excluded from public/commercial/ranked lanes. Register privately or local-only instead.`;
|
|
375
|
+
} else if (cls.tier === TIERS.NODERIVATIVES || cls.tier === TIERS.RESTRICTED_NOREDIST) {
|
|
376
|
+
reason = `License (${option.spdx}) forbids redistribution/derivatives — cannot publish a public pointer. Register privately or local-only instead.`;
|
|
377
|
+
} else {
|
|
378
|
+
reason = `License (${option.spdx}) is unconfirmed or not redistribution-cleared (gate tier: ${cls.tier}). Register privately or local-only instead.`;
|
|
379
|
+
}
|
|
380
|
+
return { allowed, tier: cls.tier, reason };
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// ---------------------------------------------------------------------------
|
|
384
|
+
// CARD BUILDER — assemble a schema-valid, content-FREE corpora card from the
|
|
385
|
+
// author's choices. Never embeds source/reference text; for private/local-only
|
|
386
|
+
// it never even names a hosted data file.
|
|
387
|
+
// ---------------------------------------------------------------------------
|
|
388
|
+
|
|
389
|
+
/** kebab-case slug from arbitrary text (for ids). */
|
|
390
|
+
export function slugify(text) {
|
|
391
|
+
return String(text || '')
|
|
392
|
+
.toLowerCase()
|
|
393
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
394
|
+
.replace(/^-+|-+$/g, '')
|
|
395
|
+
.replace(/-{2,}/g, '-');
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Derive a default card id from the pair + a name/publisher slug.
|
|
400
|
+
* Shape mirrors existing cards: eval-<src>-<tgt>-<slug>-dev-v<major>.
|
|
401
|
+
*/
|
|
402
|
+
export function deriveCardId({ source, target, slug, version = '1' }) {
|
|
403
|
+
const major = String(version).split('.')[0] || '1';
|
|
404
|
+
const tag = slug && slug.length ? slug : 'custom';
|
|
405
|
+
return `eval-${source}-${target}-${tag}-dev-v${major}`;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Build a corpora card object for a registration.
|
|
410
|
+
*
|
|
411
|
+
* @param {object} o
|
|
412
|
+
* @param {string} o.id
|
|
413
|
+
* @param {string} o.name
|
|
414
|
+
* @param {string} [o.version]
|
|
415
|
+
* @param {string} o.description
|
|
416
|
+
* @param {{source:string,target:string,direction?:string}} o.pair
|
|
417
|
+
* @param {string} o.publisher
|
|
418
|
+
* @param {string|null} [o.sourceUrl] canonical URL (public tier)
|
|
419
|
+
* @param {string|null} [o.repoUrl] fetch-from-source upstream (public tier)
|
|
420
|
+
* @param {string|null} [o.builder] builder adapter id (public tier)
|
|
421
|
+
* @param {string|null} [o.sha256] built-corpus hash (optional)
|
|
422
|
+
* @param {object} o.licenseOption a resolveLicense() result
|
|
423
|
+
* @param {object} o.tier a resolveTier() result
|
|
424
|
+
* @param {string} [o.contaminationRisk] NONE|LOW|MEDIUM|HIGH
|
|
425
|
+
* @param {string} [o.contaminationReasoning]
|
|
426
|
+
* @param {number} o.size
|
|
427
|
+
* @param {string} [o.sizeUnit]
|
|
428
|
+
* @param {string} o.domain
|
|
429
|
+
* @param {boolean} [o.doNotTrain]
|
|
430
|
+
* @param {string} o.addedAt ISO date (injected for determinism)
|
|
431
|
+
* @param {string} [o.populatedFrom]
|
|
432
|
+
* @param {object} [o.sealed] content-free sealed block (sealed tier
|
|
433
|
+
* only; from seal.buildSealedCardBlock — cipher/custodianGroupId/
|
|
434
|
+
* ciphertextDigest/aad + key + qualifier refs). NEVER contains text.
|
|
435
|
+
* @returns {object} a corpora card (metadata only — never contains text)
|
|
436
|
+
*/
|
|
437
|
+
export function buildCorpusCard(o) {
|
|
438
|
+
const tierKey = o.tier.key;
|
|
439
|
+
const isPublic = tierKey === 'public';
|
|
440
|
+
const isSealed = tierKey === 'sealed';
|
|
441
|
+
const license = deriveLicenseBlock(o.licenseOption);
|
|
442
|
+
|
|
443
|
+
// Contamination: private/sealed/unpublished data defaults to NONE per the
|
|
444
|
+
// schema's own guidance ("NONE = private/unpublished").
|
|
445
|
+
const risk = o.contaminationRisk || (isPublic ? 'LOW' : 'NONE');
|
|
446
|
+
const reasoning =
|
|
447
|
+
o.contaminationReasoning ||
|
|
448
|
+
(isPublic
|
|
449
|
+
? 'Author-asserted at registration. Public fetch-from-source corpus; review upstream exposure before relying on this rating.'
|
|
450
|
+
: isSealed
|
|
451
|
+
? 'Sealed at registration — corpus encrypted client-side under the custodian group’s threshold key; only ciphertext + a content-free card exist on our side, so the plaintext is not present in any training set via this project.'
|
|
452
|
+
: 'Private/unpublished at registration — text never uploaded or hosted, so not present in any training set via this project.');
|
|
453
|
+
|
|
454
|
+
// source block — a fetch-from-source pointer ONLY for the public tier.
|
|
455
|
+
const source = {
|
|
456
|
+
publisher: o.publisher,
|
|
457
|
+
url: isPublic ? (o.sourceUrl || o.repoUrl || null) : null,
|
|
458
|
+
};
|
|
459
|
+
if (isPublic) {
|
|
460
|
+
if (o.repoUrl) source.repo_url = o.repoUrl;
|
|
461
|
+
if (o.builder) source.builder = o.builder;
|
|
462
|
+
source.sha256 = o.sha256 || null;
|
|
463
|
+
source.license = license.spdx;
|
|
464
|
+
if (o.licenseUrl) source.license_url = o.licenseUrl;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// dev block — records the consumer-reports length/size + domain. dataFile
|
|
468
|
+
// (a hosted-content path) is set ONLY for the public tier, where the builder
|
|
469
|
+
// reproduces it from source; private/local-only never name a data file.
|
|
470
|
+
const dev = {
|
|
471
|
+
size: o.size,
|
|
472
|
+
sizeUnit: o.sizeUnit || 'entries',
|
|
473
|
+
domain: o.domain,
|
|
474
|
+
};
|
|
475
|
+
if (isPublic) {
|
|
476
|
+
dev.dataFile = `curated/${o.pair.source}-${o.pair.target}-dev-v1.json`;
|
|
477
|
+
dev.format = 'harness-json';
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
const doNotTrain = o.doNotTrain !== false; // default true (benchmark integrity)
|
|
481
|
+
|
|
482
|
+
const usageRestrictions = {
|
|
483
|
+
training: doNotTrain ? (license.commercial ? 'discouraged' : 'prohibited-by-license') : 'permitted',
|
|
484
|
+
commercialUse: license.commercial ? 'permitted' : 'prohibited-by-license',
|
|
485
|
+
redistribution: isPublic
|
|
486
|
+
? (o.licenseOption.spdx.toUpperCase().includes('-SA') ? 'same-terms' : 'permitted')
|
|
487
|
+
: 'prohibited',
|
|
488
|
+
communityNotes: null,
|
|
489
|
+
};
|
|
490
|
+
|
|
491
|
+
const card = {
|
|
492
|
+
id: o.id,
|
|
493
|
+
type: 'eval',
|
|
494
|
+
name: o.name,
|
|
495
|
+
version: o.version || '0.1.0',
|
|
496
|
+
pair: {
|
|
497
|
+
source: o.pair.source,
|
|
498
|
+
target: o.pair.target,
|
|
499
|
+
direction: o.pair.direction || 'unidirectional',
|
|
500
|
+
},
|
|
501
|
+
description: o.description,
|
|
502
|
+
source,
|
|
503
|
+
license,
|
|
504
|
+
dev,
|
|
505
|
+
contamination: { risk, reasoning },
|
|
506
|
+
doNotTrain,
|
|
507
|
+
exposureTier: tierKey,
|
|
508
|
+
usageRestrictions,
|
|
509
|
+
_provenance: {
|
|
510
|
+
addedAt: o.addedAt,
|
|
511
|
+
populatedFrom:
|
|
512
|
+
o.populatedFrom ||
|
|
513
|
+
`Registered via 'champollion register-corpus' (exposureTier=${tierKey}). Metadata only — corpus content was never read, uploaded, or hosted.`,
|
|
514
|
+
},
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
// Private/sovereign held-out sets are catalogued but never publicly runnable:
|
|
518
|
+
// quarantine keeps them out of the queue + leaderboard while preserving the
|
|
519
|
+
// metadata record (the WMT-style "a secret test set exists" announcement).
|
|
520
|
+
if (tierKey === 'private') {
|
|
521
|
+
card.quarantine = true;
|
|
522
|
+
card.quarantineReason =
|
|
523
|
+
'Private / sovereign held-out set — registered metadata only; corpus content is never uploaded or hosted (exposureTier=private). Custodian-controlled; evaluate with the harness in --private/--scores-only mode.';
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// Sealed sets carry the SAME invariant as private (content-free, quarantined)
|
|
527
|
+
// plus a content-free `sealed` block recording the cipher, custodian group,
|
|
528
|
+
// ciphertext digest, and AAD binding. The plaintext was encrypted client-side
|
|
529
|
+
// and is never present here — only ciphertext (in an off-git store) + this card.
|
|
530
|
+
if (isSealed) {
|
|
531
|
+
card.quarantine = true;
|
|
532
|
+
card.quarantineReason =
|
|
533
|
+
'Sealed / community-controlled secret test set — corpus encrypted client-side under the custodian group’s threshold key (exposureTier=sealed). Only ciphertext + this content-free card exist on our side; no single party can decrypt. Catalogued but never publicly runnable; a sealed run requires M-of-N custodian approval (see OCAP multisig plan).';
|
|
534
|
+
if (o.sealed) card.sealed = o.sealed;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
return card;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// ---------------------------------------------------------------------------
|
|
541
|
+
// Destination resolution — WHERE the card is written. local-only never touches
|
|
542
|
+
// the tracked corpora-cards SSOT; private/public land there.
|
|
543
|
+
// ---------------------------------------------------------------------------
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Decide where a card for the chosen tier should be written.
|
|
547
|
+
*
|
|
548
|
+
* @param {object} o
|
|
549
|
+
* @param {object} o.tier a resolveTier() result
|
|
550
|
+
* @param {string} o.id card id (filename stem)
|
|
551
|
+
* @param {string} o.corporaCardsDir absolute path to the tracked corpora-cards dir
|
|
552
|
+
* @param {string} o.localDir absolute path for local-only output (cwd or --out)
|
|
553
|
+
* @returns {{registered:boolean, tracked:boolean, dir:string, filename:string}}
|
|
554
|
+
*/
|
|
555
|
+
export function resolveDestination({ tier, id, corporaCardsDir, localDir }) {
|
|
556
|
+
const filename = `${id}.json`;
|
|
557
|
+
if (tier.key === 'local-only') {
|
|
558
|
+
return { registered: false, tracked: false, dir: localDir, filename };
|
|
559
|
+
}
|
|
560
|
+
// private + public — the tracked SSOT
|
|
561
|
+
return { registered: true, tracked: true, dir: corporaCardsDir, filename };
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Validate a registration request. Returns collected, plain-language errors so
|
|
566
|
+
* the command can fail loudly with everything wrong at once.
|
|
567
|
+
*
|
|
568
|
+
* @param {object} o
|
|
569
|
+
* @param {object} o.tier
|
|
570
|
+
* @param {object|null} o.licenseOption
|
|
571
|
+
* @param {{source?:string,target?:string}} o.pair
|
|
572
|
+
* @param {string} o.name
|
|
573
|
+
* @param {string|null} [o.repoUrl]
|
|
574
|
+
* @param {string|null} [o.builder]
|
|
575
|
+
* @param {number|null} [o.size]
|
|
576
|
+
* @param {string} [o.domain]
|
|
577
|
+
* @param {string|null} [o.custodianGroupId] sealed tier: custodian group id
|
|
578
|
+
* @param {string|null} [o.thresholdPublicKey] sealed tier: threshold pubkey (path/value)
|
|
579
|
+
* @param {string|null} [o.sealInput] sealed tier: local corpus file to encrypt
|
|
580
|
+
* @param {string|null} [o.qualifierId] sealed tier: paired public qualifier id
|
|
581
|
+
* @returns {{ok:boolean, errors:string[]}}
|
|
582
|
+
*/
|
|
583
|
+
export function validateRegistration(o) {
|
|
584
|
+
const errors = [];
|
|
585
|
+
if (!o.name || !String(o.name).trim()) errors.push('A corpus name is required (--name).');
|
|
586
|
+
if (!o.pair || !o.pair.source) errors.push('A source language code is required (--pair "src>tgt" or --source-lang).');
|
|
587
|
+
if (!o.pair || !o.pair.target) errors.push('A target language code is required (--pair "src>tgt" or --target-lang).');
|
|
588
|
+
if (!o.licenseOption) errors.push('A license is required (--license). Run with --list to see options.');
|
|
589
|
+
if (!(Number(o.size) > 0)) errors.push('A positive corpus size is required (--size).');
|
|
590
|
+
if (!o.domain || !String(o.domain).trim()) errors.push('A domain is required (--domain), e.g. news, conversational, educational.');
|
|
591
|
+
|
|
592
|
+
if (o.tier.key === 'public') {
|
|
593
|
+
if (!o.repoUrl) errors.push('Public registration needs a fetch-from-source pointer (--repo-url) — we never host content.');
|
|
594
|
+
if (!o.builder) errors.push('Public registration needs a builder adapter id (--builder) that rebuilds the corpus from source.');
|
|
595
|
+
if (o.licenseOption) {
|
|
596
|
+
const gate = gatePublicRegistration(o.licenseOption);
|
|
597
|
+
if (!gate.allowed) errors.push(`License gate blocked public registration: ${gate.reason}`);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
if (o.tier.key === 'sealed') {
|
|
602
|
+
if (!o.custodianGroupId) errors.push('Sealed registration needs a custodian group id (--custodian-group) — the community group whose threshold key controls the set.');
|
|
603
|
+
if (!o.thresholdPublicKey) errors.push('Sealed registration needs the custodian group’s threshold public key (--threshold-pubkey) to encrypt to — we never receive a key we could decrypt with.');
|
|
604
|
+
if (!o.sealInput) errors.push('Sealed registration needs the local corpus file to encrypt (--seal-input) — it is sealed on your device and never sent in readable form.');
|
|
605
|
+
if (!o.qualifierId) errors.push('Sealed registration needs a paired public qualifier (--qualifier-id) that methods must clear before any sealed run can be proposed.');
|
|
606
|
+
}
|
|
607
|
+
return { ok: errors.length === 0, errors };
|
|
608
|
+
}
|