champollion 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +133 -0
  2. package/README.md +387 -0
  3. package/bin/cli.js +278 -0
  4. package/index.js +135 -0
  5. package/lib/api-key.js +127 -0
  6. package/lib/autofix.js +432 -0
  7. package/lib/bridge/method_bridge.py +430 -0
  8. package/lib/card-source-resolution.mjs +284 -0
  9. package/lib/cards/cache.js +169 -0
  10. package/lib/cards/env.js +82 -0
  11. package/lib/cards/fetch-card-child.js +38 -0
  12. package/lib/cards/reader.js +435 -0
  13. package/lib/cards/refresh.js +111 -0
  14. package/lib/cards/remote.js +387 -0
  15. package/lib/cldf-export.mjs +540 -0
  16. package/lib/cldf-terms.mjs +62 -0
  17. package/lib/command-help.js +790 -0
  18. package/lib/commands/audit.js +49 -0
  19. package/lib/commands/card.js +454 -0
  20. package/lib/commands/doctor.js +559 -0
  21. package/lib/commands/fonts.js +489 -0
  22. package/lib/commands/help.js +91 -0
  23. package/lib/commands/init.js +1259 -0
  24. package/lib/commands/integrity.js +148 -0
  25. package/lib/commands/leaderboard.js +478 -0
  26. package/lib/commands/lint.js +30 -0
  27. package/lib/commands/models.js +177 -0
  28. package/lib/commands/plugin.js +103 -0
  29. package/lib/commands/provenance.js +45 -0
  30. package/lib/commands/recommend.js +75 -0
  31. package/lib/commands/register-corpus.js +678 -0
  32. package/lib/commands/repair-script.js +42 -0
  33. package/lib/commands/seal-corpus.js +355 -0
  34. package/lib/commands/seo.js +72 -0
  35. package/lib/commands/serve.js +147 -0
  36. package/lib/commands/status.js +265 -0
  37. package/lib/commands/submit.js +332 -0
  38. package/lib/commands/sync.js +89 -0
  39. package/lib/commands/tm.js +573 -0
  40. package/lib/commands/verify.js +39 -0
  41. package/lib/commands/watch.js +20 -0
  42. package/lib/commands/wrap.js +138 -0
  43. package/lib/commands/xliff.js +327 -0
  44. package/lib/commercial-eligibility.js +235 -0
  45. package/lib/concurrent.js +87 -0
  46. package/lib/config.js +523 -0
  47. package/lib/contamination-lane.js +76 -0
  48. package/lib/content-sync.js +731 -0
  49. package/lib/content.js +733 -0
  50. package/lib/corpus-registration.mjs +608 -0
  51. package/lib/cost-report.js +346 -0
  52. package/lib/diff.js +155 -0
  53. package/lib/docusaurus-sync.js +1256 -0
  54. package/lib/flatten.js +55 -0
  55. package/lib/format.js +954 -0
  56. package/lib/hash.js +159 -0
  57. package/lib/icu.js +473 -0
  58. package/lib/integrity.js +689 -0
  59. package/lib/license-gate.mjs +478 -0
  60. package/lib/license-identify.mjs +229 -0
  61. package/lib/lint.js +629 -0
  62. package/lib/method-manifest.js +60 -0
  63. package/lib/methods/anthropic.js +140 -0
  64. package/lib/methods/apertium.js +163 -0
  65. package/lib/methods/api.js +316 -0
  66. package/lib/methods/base.js +184 -0
  67. package/lib/methods/content-separator.js +45 -0
  68. package/lib/methods/deepl.js +426 -0
  69. package/lib/methods/direct-llm.js +586 -0
  70. package/lib/methods/external.js +332 -0
  71. package/lib/methods/fetch-with-retry.js +124 -0
  72. package/lib/methods/gemini.js +147 -0
  73. package/lib/methods/google-translate.js +402 -0
  74. package/lib/methods/http-utils.js +122 -0
  75. package/lib/methods/libretranslate.js +314 -0
  76. package/lib/methods/llm-coached.js +670 -0
  77. package/lib/methods/llm.js +592 -0
  78. package/lib/methods/local.js +76 -0
  79. package/lib/methods/microsoft-translator.js +331 -0
  80. package/lib/methods/openai.js +131 -0
  81. package/lib/methods/openrouter-client.js +327 -0
  82. package/lib/methods/openrouter-pricing.js +156 -0
  83. package/lib/methods/provider-env.js +115 -0
  84. package/lib/methods/provider-pricing.js +310 -0
  85. package/lib/methods/tilde.js +150 -0
  86. package/lib/methods/translated.js +229 -0
  87. package/lib/methods/translation-error.js +80 -0
  88. package/lib/models.js +258 -0
  89. package/lib/no-translate.js +233 -0
  90. package/lib/output.js +238 -0
  91. package/lib/pairs.js +547 -0
  92. package/lib/plugins.js +447 -0
  93. package/lib/provenance.js +323 -0
  94. package/lib/recommend.js +648 -0
  95. package/lib/registers.js +1185 -0
  96. package/lib/repair-script.js +266 -0
  97. package/lib/scripts.js +994 -0
  98. package/lib/seal.mjs +464 -0
  99. package/lib/sealed-qualifier.mjs +211 -0
  100. package/lib/security.js +59 -0
  101. package/lib/segment.js +369 -0
  102. package/lib/seo.js +275 -0
  103. package/lib/serve.js +854 -0
  104. package/lib/string-classify.js +85 -0
  105. package/lib/submit.mjs +344 -0
  106. package/lib/sync.js +969 -0
  107. package/lib/tags/bcp47.js +202 -0
  108. package/lib/tags/resolve.js +314 -0
  109. package/lib/terminology.js +111 -0
  110. package/lib/tm-seed.js +294 -0
  111. package/lib/tm.js +515 -0
  112. package/lib/translate-pair.js +197 -0
  113. package/lib/translate.js +203 -0
  114. package/lib/types.js +230 -0
  115. package/lib/validate.js +510 -0
  116. package/lib/verify.js +451 -0
  117. package/lib/watch.js +145 -0
  118. package/lib/xliff.js +184 -0
  119. package/package.json +93 -0
  120. package/shared/ATTRIBUTION.md +145 -0
  121. package/shared/CORPORA-CARDS.md +288 -0
  122. package/shared/DATA-SOVEREIGNTY.md +500 -0
  123. package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
  124. package/shared/card-lint-baseline.json +3189 -0
  125. package/shared/cards-fallback.json +1 -0
  126. package/shared/catalogue/card-config.json +6091 -0
  127. package/shared/catalogue/external-results.json +3888 -0
  128. package/shared/catalogue/gender-guidance.json +1038 -0
  129. package/shared/catalogue/method-coverage.json +1751 -0
  130. package/shared/catalogue/metric-coverage.json +170 -0
  131. package/shared/catalogue/metric-reliability.json +1 -0
  132. package/shared/catalogue/register-presets.json +3180 -0
  133. package/shared/catalogue/vitality-scales.json +55 -0
  134. package/shared/cldr-index.json +1115 -0
  135. package/shared/code-bridge.json +253 -0
  136. package/shared/corpora-cards-v1-reference.md +281 -0
  137. package/shared/curated-dictionary-flags.json +35 -0
  138. package/shared/curated-endonyms.json +35 -0
  139. package/shared/curated-fsts.json +51 -0
  140. package/shared/curated-orthography-conventions.json +26 -0
  141. package/shared/curated-sil-resources.json +374 -0
  142. package/shared/curated-tools.json +41 -0
  143. package/shared/docent/corpus.json +11333 -0
  144. package/shared/docent/faq.en.json +564 -0
  145. package/shared/docent/register-blocks.json +60 -0
  146. package/shared/docent/system-prompt.md +144 -0
  147. package/shared/domain-taxonomy.json +35 -0
  148. package/shared/explainers/glossary.json +2975 -0
  149. package/shared/explainers/tc-features.json +20112 -0
  150. package/shared/explainers/term-watchlist.json +147 -0
  151. package/shared/human-services.json +59 -0
  152. package/shared/license-corrections.json +261 -0
  153. package/shared/license-evidence.json +13452 -0
  154. package/shared/licenses.json +6781 -0
  155. package/shared/method-registry.json +236 -0
  156. package/shared/metric-registry.json +620 -0
  157. package/shared/model-aliases.json +7 -0
  158. package/shared/schemas/champollion-plugin.schema.json +206 -0
  159. package/shared/schemas/corpora-card.schema.json +957 -0
  160. package/shared/schemas/domain-taxonomy.schema.json +64 -0
  161. package/shared/schemas/external-results.schema.json +314 -0
  162. package/shared/schemas/human-services.schema.json +90 -0
  163. package/shared/schemas/language-card.schema.json +1308 -0
  164. package/shared/schemas/licenses.schema.json +155 -0
  165. package/shared/schemas/method-card.schema.json +412 -0
  166. package/shared/schemas/method-registry.schema.json +85 -0
  167. package/shared/schemas/metric-registry.schema.json +96 -0
  168. package/shared/schemas/metric-reliability.schema.json +178 -0
  169. package/shared/schemas/model-aliases.schema.json +27 -0
  170. package/shared/schemas/source-snapshot.schema.json +96 -0
@@ -0,0 +1,500 @@
1
+ # Data Sovereignty Field Reference
2
+
3
+ > **Version:** 1.0
4
+ > **Date:** 2026-06-09
5
+ > **Schema:** `cli/shared/schemas/corpora-card.schema.json`
6
+ > **Audience:** Contributors, eval set authors, governance tool implementers, downstream consumers
7
+
8
+ This document describes the sovereignty, stewardship, submission, and usage restriction fields in the Champollion corpora card schema. For each field you will find the JSON key, type, allowed values, a plain-English description, and guidance on when to use `null`.
9
+
10
+ For corpus identity, license, contamination, and split fields see the schema directly.
11
+ For language card fields see [LANGUAGE-CARD-FIELDS.md](./LANGUAGE-CARD-FIELDS.md).
12
+ For attribution and license obligations see [ATTRIBUTION.md](./ATTRIBUTION.md).
13
+
14
+ ---
15
+
16
+ ## 1. Purpose
17
+
18
+ The sovereignty fields record **governance facts** about a corpus:
19
+
20
+ - Who has formal authority over this data (if anyone).
21
+ - What frameworks the governing body has invoked (if any).
22
+ - What the community has said about acceptable use.
23
+ - How steward authorization and prize evaluation work.
24
+
25
+ They do **not** record:
26
+
27
+ - Invented governance where none exists. If no governance body has been identified, `sovereignty` is `null`.
28
+ - Inferred classifications. A language's endangerment status does not automatically populate sovereignty fields.
29
+ - Platform opinions about how data *should* be governed. That's the community's decision.
30
+ - Legal interpretations. The `license` field records legal facts. Sovereignty records governance assertions.
31
+
32
+ The fields are forward-looking infrastructure. Many corpora will have `sovereignty: null` and `usageRestrictions` that simply defer to the license. That's correct — it means no governance body has asserted terms beyond what the license already says.
33
+
34
+ ---
35
+
36
+ ## 2. Standards Map
37
+
38
+ Each sovereignty-related field maps to one or more external standards. The table below shows which standard provides the conceptual basis for each field and where to find the standard's specification.
39
+
40
+ | Field / Concept | Primary Standard | Standard URL | Relationship |
41
+ |---|---|---|---|
42
+ | `sovereignty.frameworks` | OCAP®, CARE, Te Mana Raraunga, FAIR, IEEE 2890 | See individual URLs below | Records which frameworks have been **explicitly invoked** by the governing body |
43
+ | `sovereignty.governanceOrg` | OCAP® Ownership | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | Maps to the "O" in OCAP — who owns the data |
44
+ | `sovereignty.custodian` | OCAP® Possession | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | Maps to the "P" in OCAP — who physically holds the data |
45
+ | `sovereignty.consentModel` | OCAP® Control, CARE Authority | [https://www.gida-global.org/care](https://www.gida-global.org/care) | Maps to the "C" in OCAP (who controls access) and CARE's Authority to Control principle |
46
+ | `sovereignty.tkLabels[]` | Local Contexts TK/BC Labels | [https://localcontexts.org/](https://localcontexts.org/) | Community-applied labels from the Local Contexts Hub |
47
+ | `usageRestrictions.training` | DUO (Data Use Ontology), ODRL | [DUO](https://github.com/EBISPOT/DUO), [ODRL](https://www.w3.org/TR/odrl-model/) | `prohibited-by-license` maps to DUO:0000004 (no general methods research) and ODRL `odrl:prohibition`. `prohibited-by-community` maps to a governance assertion (no DUO equivalent — DUO is license-derived) |
48
+ | `usageRestrictions.commercialUse` | DUO:0000018 (not for profit), ODRL | [DUO](https://github.com/EBISPOT/DUO), [ODRL](https://www.w3.org/TR/odrl-model/) | `prohibited-by-license` maps to DUO:0000018 and ODRL `odrl:prohibition` with `odrl:commercialize` action |
49
+ | `usageRestrictions.redistribution` | ODRL `odrl:distribute` | [ODRL](https://www.w3.org/TR/odrl-model/) | Maps to an ODRL permission/prohibition on the `odrl:distribute` action |
50
+ | `usageRestrictions.communityNotes` | CARE Ethics | [https://www.gida-global.org/care](https://www.gida-global.org/care) | Maps to CARE's Ethics principle — ensuring legitimate concerns are surfaced |
51
+ | `doNotTrain` | IEEE 2890 | No public URL (IEEE standard, paywall) | Methodological constraint for evaluation data; IEEE 2890 §7.4 recommends flagging datasets not intended for model training |
52
+ | `stewardship.authorizationModel` | OCAP® Control | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | Community controls how evaluation access is granted |
53
+ | `submission.transfer.*` | OCAP® Ownership + Possession, CARE Collective Benefit | [OCAP](https://fnigc.ca/ocap-training/), [CARE](https://www.gida-global.org/care) | Transfer provisions map to OCAP O+P (community owns and possesses the method). Revenue model maps to CARE Collective Benefit |
54
+ | `submission.admissibility.selfHostable` | OCAP® Possession | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | If the community can't run it independently, they don't possess it |
55
+
56
+ ### Standard References
57
+
58
+ | Standard | Full Name | URL |
59
+ |---|---|---|
60
+ | ODRL | Open Digital Rights Language 2.2 | [https://www.w3.org/TR/odrl-model/](https://www.w3.org/TR/odrl-model/) |
61
+ | DUO | Data Use Ontology | [https://github.com/EBISPOT/DUO](https://github.com/EBISPOT/DUO) |
62
+ | Local Contexts | TK and BC Labels | [https://localcontexts.org/](https://localcontexts.org/) |
63
+ | CARE | Collective Benefit, Authority to Control, Responsibility, Ethics | [https://www.gida-global.org/care](https://www.gida-global.org/care) |
64
+ | OCAP® | Ownership, Control, Access, Possession | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) |
65
+ | IEEE 2890 | IEEE Standard for Recommended Practice for Provenance of Datasets | IEEE (paywall; no stable public URL) |
66
+ | Te Mana Raraunga | Māori Data Sovereignty Network | [https://www.temanararaunga.maori.nz/](https://www.temanararaunga.maori.nz/) |
67
+ | FAIR | Findable, Accessible, Interoperable, Reusable | [https://www.go-fair.org/fair-principles/](https://www.go-fair.org/fair-principles/) |
68
+
69
+ ---
70
+
71
+ ## 3. Three-Source Restriction Model
72
+
73
+ Usage restrictions in `usageRestrictions` use enum values that encode **who imposed the restriction**, not just what the restriction is. This is deliberate — compliance tools need to know whether a restriction is legally binding or advisory.
74
+
75
+ ### The Three Sources
76
+
77
+ | Source | Enum suffix / value | Meaning | Legal force | Example |
78
+ |---|---|---|---|---|
79
+ | **License** | `prohibited-by-license` | The license text explicitly prohibits this use | Legally binding — violation is a license breach | CC-BY-NC-4.0 prohibits commercial use |
80
+ | **Community** | `prohibited-by-community` | A governance body has asserted this restriction, independent of the license | Advisory — not directly enforceable as contract, but ethically binding and may carry institutional consequences | A language trust requests no ML training, even though the license is CC-BY-4.0 |
81
+ | **Nobody** | `permitted` | No restriction from any source | Permitted under both license and community governance | CC-BY-4.0 corpus with no community governance body |
82
+
83
+ ### Why the Source Matters
84
+
85
+ 1. **Compliance automation.** A tool filtering datasets for training needs to know: "Is this a hard legal constraint or a community request?" Both should be respected, but they have different implications for institutional compliance workflows.
86
+
87
+ 2. **Layered restrictions.** `usageRestrictions` **adds to** the license — it never weakens it. If the license says non-commercial, `commercialUse` should be `prohibited-by-license`. If the license permits commercial use but the community objects, `commercialUse` should be `prohibited-by-community`.
88
+
89
+ 3. **Transparency.** Users see where each restriction comes from. No opaque "this is restricted" without knowing who said so and why.
90
+
91
+ ### The `null` / Defer-to-License Pattern
92
+
93
+ Several `usageRestrictions` fields accept `null`. Null means: "No additional guidance beyond what the `license` field already says." This avoids restating what's already in the license.
94
+
95
+ | `usageRestrictions` field | When to use `null` |
96
+ |---|---|
97
+ | `commercialUse` | The license `commercial` boolean already captures the full picture. No community has added restrictions or context. |
98
+ | `redistribution` | The license `redistribution` boolean already captures the full picture. |
99
+
100
+ The `training` field is **required** (not nullable) because the training question is central to evaluation corpora and must always be explicitly answered.
101
+
102
+ ---
103
+
104
+ ## 4. `sovereignty` Field Reference
105
+
106
+ The `sovereignty` object records governance structures that actually exist for this data. It is `null` when no formal governance structures are in place. That's not a failure state — most reference corpora (`ref-*` type) will have `sovereignty: null`.
107
+
108
+ | Field | Type | Allowed Values | Description | When to use `null` |
109
+ |---|---|---|---|---|
110
+ | `sovereignty` | `object \| null` | — | Container for governance metadata. | No governance body has been identified. No frameworks have been invoked. No labels have been applied. |
111
+ | `.frameworks` | `array` of `string` | `"OCAP"`, `"CARE"`, `"Te-Mana-Raraunga"`, `"FAIR"`, `"IEEE-2890"` | Data sovereignty frameworks the data creators or governing body have **explicitly invoked**. Only list frameworks where there is documented evidence of adoption. | *(Field is an array — use `[]` if sovereignty object exists but no frameworks have been invoked. But if no sovereignty object at all, the whole object is null.)* |
112
+ | `.governanceOrg` | `string \| null` | Free text | Organization or body with governance authority. For Indigenous-governed corpora, this is the language trust, tribal council, or delegated body. | No governance body has been identified or established. |
113
+ | `.custodian` | `string \| null` | Free text | Who physically holds the data. Distinct from `governanceOrg` — a university may be custodian while a community is the governance authority. | Custodian is the same as `source.publisher` (no need to repeat). |
114
+ | `.consentModel` | `string \| null` | `"per-submission"`, `"blanket"`, `"open-access"` | How consent is structured for use of this data in method evaluation. | No consent model has been established. |
115
+ | `.tkLabels` | `array` of objects | See sub-fields below | Local Contexts Traditional Knowledge or Biocultural Labels applied by the data's governing community. Empty array `[]` = sovereignty exists but no labels applied. | *(Field is an array — use `[]` when no labels applied.)* |
116
+ | `.tkLabels[].labelType` | `string` | TK/BC label codes (e.g., `"TK-A"`, `"TK-NC"`, `"BC-P"`) | Local Contexts label code. **Required** within each label entry. Only populated if the community has actually applied labels via the Local Contexts Hub. | *(Required — not nullable.)* |
117
+ | `.tkLabels[].projectId` | `string \| null` | Local Contexts Hub project ID | Enables dynamic label updates by communities without touching Champollion metadata. | Labels are referenced but not registered on the Hub. |
118
+ | `.tkLabels[].url` | `string \| null` (URI format) | URL | Direct link to the label on the Local Contexts Hub. | No Hub URL available. |
119
+ | `.notes` | `string \| null` | Free text | Governance context that structured fields can't capture. Use this for nuance. | No additional context needed. |
120
+
121
+ ### Example: Sovereignty with active governance
122
+
123
+ ```json
124
+ {
125
+ "sovereignty": {
126
+ "frameworks": ["OCAP", "CARE"],
127
+ "governanceOrg": "Plains Cree Language Trust (hypothetical)",
128
+ "custodian": "University of Alberta — ALTLab",
129
+ "consentModel": "per-submission",
130
+ "tkLabels": [
131
+ {
132
+ "labelType": "TK-A",
133
+ "projectId": "abc-123",
134
+ "url": "https://localcontextshub.org/projects/abc-123/labels/TK-A"
135
+ }
136
+ ],
137
+ "notes": "Governance body identification in progress."
138
+ }
139
+ }
140
+ ```
141
+
142
+ ### Example: No governance (most reference corpora)
143
+
144
+ ```json
145
+ {
146
+ "sovereignty": null
147
+ }
148
+ ```
149
+
150
+ ---
151
+
152
+ ## 5. `usageRestrictions` Field Reference
153
+
154
+ The `usageRestrictions` object records community assertions and guidance beyond what the `license` field captures. Required for `eval`-type cards. The license records legal facts. `usageRestrictions` records what communities have said — even when it differs from or supplements the license.
155
+
156
+ | Field | Type | Allowed Values | Description | When to use `null` |
157
+ |---|---|---|---|---|
158
+ | `usageRestrictions` | `object \| null` | — | Container for usage guidance. Required for eval-type cards. | Only null for reference-type cards where no community guidance exists beyond the license. |
159
+ | `.training` | `string` | `"prohibited-by-license"`, `"prohibited-by-community"`, `"discouraged"`, `"permitted"` | Whether this data may be used for ML model training. The enum value identifies **who** imposed the restriction. **Required.** | *(Not nullable — must always be explicitly set.)* |
160
+ | `.commercialUse` | `string \| null` | `"prohibited-by-license"`, `"prohibited-by-community"`, `"requires-agreement"`, `"permitted"` | Commercial use restrictions. Identifies source of restriction. | Defer to `license.commercial` — no additional community guidance exists. |
161
+ | `.redistribution` | `string \| null` | `"prohibited"`, `"same-terms"`, `"permitted"` | Redistribution restrictions. | Defer to `license.redistribution` — no additional community guidance exists. |
162
+ | `.communityNotes` | `string \| null` | Free text | Documented community concerns, context, or guidance that users should know about — even if the license technically permits the use. This is where legitimate community objections get surfaced. | No community concerns documented. |
163
+
164
+ ### `training` Enum Values
165
+
166
+ | Value | Who decided | Meaning | Consistency rule |
167
+ |---|---|---|---|
168
+ | `prohibited-by-license` | License author | The license text explicitly prohibits use in ML training | `doNotTrain` must be `true` |
169
+ | `prohibited-by-community` | Governance body | A community governance body prohibits training use, independent of license terms | `doNotTrain` must be `true` |
170
+ | `discouraged` | Community / custodian | Not prohibited, but the data custodians prefer it not be used for training | `doNotTrain` may be `true` or `false` depending on methodological need |
171
+ | `permitted` | Nobody (no restriction) | No restriction from any source | `doNotTrain` must not be `true` unless the corpus is also an eval set (see §8) |
172
+
173
+ ### `commercialUse` Enum Values
174
+
175
+ | Value | Who decided | Meaning |
176
+ |---|---|---|
177
+ | `prohibited-by-license` | License author | License terms prohibit commercial use (e.g., CC-NC) |
178
+ | `prohibited-by-community` | Governance body | Community prohibits commercial use, even if license permits |
179
+ | `requires-agreement` | Governance body / custodian | Commercial use is possible but requires a separate agreement |
180
+ | `permitted` | Nobody (no restriction) | No commercial use restrictions from any source |
181
+
182
+ ### Example: Community-governed eval corpus
183
+
184
+ ```json
185
+ {
186
+ "usageRestrictions": {
187
+ "training": "prohibited-by-community",
188
+ "commercialUse": "prohibited-by-community",
189
+ "redistribution": null,
190
+ "communityNotes": "Plains Cree is severely endangered; data created by L1 educators for educational purposes. Community interests in language revitalization should be considered."
191
+ }
192
+ }
193
+ ```
194
+
195
+ ### Example: Open reference corpus with no extra restrictions
196
+
197
+ ```json
198
+ {
199
+ "usageRestrictions": {
200
+ "training": "permitted",
201
+ "commercialUse": null,
202
+ "redistribution": null,
203
+ "communityNotes": null
204
+ }
205
+ }
206
+ ```
207
+
208
+ ---
209
+
210
+ ## 6. Prize Corpus Model
211
+
212
+ Prize evaluation sets use four coordinated structures to enable secure, community-controlled benchmarking. These four structures are defined across separate fields in the corpora card schema.
213
+
214
+ ### The Four Components
215
+
216
+ | Component | Schema field | Visibility | Purpose |
217
+ |---|---|---|---|
218
+ | **Dev split** | `dev` | Public | Local development and prompt tuning. Distributed openly. |
219
+ | **Public test** | `test` | Public | The non-secured portion of the test corpus. Published for reproducibility. Distinct from `dev` — `test` is for scoring, `dev` is for iteration. |
220
+ | **Secret test** | `secretTest` | Encrypted | Cryptographically secured holdout. Encrypted at rest, decrypted only inside the air-gapped evaluation sandbox. Never distributed. Steward authorization (TSS threshold signature) required for each evaluation run. |
221
+ | **Stewardship** | `stewardship` | Public metadata | Community steward roster, multi-signature threshold, and authorization model. Stewards are chosen by the language community. |
222
+
223
+ ### How They Work Together
224
+
225
+ ```
226
+ ┌────────────────────────────────────────────────────────────────────┐
227
+ │ Researcher │
228
+ │ │
229
+ │ 1. Downloads dev split → iterates on method locally │
230
+ │ 2. Downloads test split → validates scores locally │
231
+ │ 3. Submits self-hostable method for prize evaluation │
232
+ │ │
233
+ ├────────────────────────────────────────────────────────────────────┤
234
+ │ Stewards (community-chosen, minimum 5) │
235
+ │ │
236
+ │ 4. Review submission │
237
+ │ 5. Threshold-sign authorization (e.g., 3 of 5) │
238
+ │ │
239
+ ├────────────────────────────────────────────────────────────────────┤
240
+ │ Air-gapped sandbox (no network) │
241
+ │ │
242
+ │ 6. Decrypts secretTest with steward-authorized key │
243
+ │ 7. Runs submitted method against secret test data │
244
+ │ 8. Returns scores — never exposes test data │
245
+ └────────────────────────────────────────────────────────────────────┘
246
+ ```
247
+
248
+ ### Conditional Requirements
249
+
250
+ The schema enforces: when `secretTest.status` is `"active"`, the card **must** have:
251
+ - A `stewardship` object with at least 5 stewards
252
+ - `stewardship.threshold` and `stewardship.authorizationModel` defined
253
+
254
+ This is a schema-level `allOf` conditional, not a soft guideline.
255
+
256
+ ### `submission` — The Prize Deal
257
+
258
+ The `submission` field (§7) defines what happens when a method is accepted: what transfers to the governance org, what the researcher retains, and what methods are admissible.
259
+
260
+ > [!NOTE]
261
+ > The sandbox evaluation infrastructure (VPS endpoints, key ceremonies, TSS implementation) is not yet established. The schema fields are forward-looking — they define the data model for when infrastructure is deployed. `secretTest.serverEndpoint` and steward `publicKey` fields will be `null` until then.
262
+
263
+ ---
264
+
265
+ ## 7. Admissibility Rules
266
+
267
+ The `submission.admissibility` fields define what methods are eligible for prize evaluation. These constraints are technical, not arbitrary — they flow directly from OCAP® Possession requirements and the air-gapped sandbox architecture.
268
+
269
+ ### Why Coached API Calls Are Inadmissible
270
+
271
+ A "coached API call" is a method that wraps a third-party API (e.g., GPT-4, Google Translate) with prompt engineering. These are inadmissible for prize evaluation because:
272
+
273
+ 1. **OCAP® Possession violation.** The community cannot possess a method that depends on someone else's API. When the API key expires, the method dies. The community doesn't control the model, its weights, or its availability.
274
+
275
+ 2. **Non-reproducible.** API providers change models, pricing, and availability without notice. A score measured today may not be reproducible tomorrow.
276
+
277
+ 3. **Sandbox incompatibility.** The evaluation sandbox is air-gapped — no network access. Methods that phone home cannot execute.
278
+
279
+ ### What "Self-Hostable" Means
280
+
281
+ `admissibility.selfHostable: true` requires that the submitted method can run on community infrastructure without any third-party dependencies. Concretely:
282
+
283
+ - All model weights must be included in the submission
284
+ - All source code must be included
285
+ - No network calls during inference
286
+ - No license-restricted runtime dependencies that the community cannot independently obtain
287
+ - The community can inspect, modify, and deploy the method on their own hardware
288
+
289
+ ### OCAP® Possession Logic
290
+
291
+ The OCAP® Possession principle states that the community must physically hold the data and tools. In the context of prize evaluation:
292
+
293
+ | What | Possession requirement |
294
+ |---|---|
295
+ | Evaluation data (secretTest) | Encrypted, held by stewards. Decryption requires threshold signature. |
296
+ | Submitted method (source + weights) | Transferred to governance org upon acceptance. Community possesses every byte. |
297
+ | Evaluation infrastructure | Community-controlled sandbox. Not a platform service. |
298
+
299
+ ### Air-Gapped Sandbox Architecture
300
+
301
+ The evaluation sandbox operates with no network access:
302
+
303
+ - Methods are loaded as self-contained packages
304
+ - Secret test data is decrypted inside the sandbox
305
+ - Scoring happens locally — results exit, data does not
306
+ - The sandbox is a technical enforcement of Possession — if it can't run without network, the community doesn't possess it
307
+
308
+ ### `submission.admissibility` Fields
309
+
310
+ | Field | Type | Description |
311
+ |---|---|---|
312
+ | `selfHostable` | `boolean` | Method must run on community infrastructure without any third-party dependencies. |
313
+ | `inadmissible` | `array` of `string` | Explicitly inadmissible method types. e.g., `"coached-api-calls"`, `"proprietary-api-wrappers"`, `"external-api-dependencies"`. |
314
+ | `notes` | `string \| null` | Rationale for admissibility constraints. Should reference OCAP Possession and the air-gapped sandbox architecture. |
315
+
316
+ ### `submission.transfer` Fields — What the Community Gets
317
+
318
+ | Field | Type | Description | OCAP/CARE mapping |
319
+ |---|---|---|---|
320
+ | `sourceCode` | `boolean` | Source code ownership transfers to governance org | OCAP Ownership + Possession |
321
+ | `modelWeights` | `boolean` | Trained model weights transfer to governance org | OCAP Possession |
322
+ | `deploymentRights` | `boolean` | Exclusive deployment rights transfer to governance org | OCAP Control |
323
+ | `revenueModel` | `string \| null` | Community-set terms for any commercial deployment, held by the governance org (e.g., `"commercial use by written permission only"`) — Champollion is non-commercial and takes no share | CARE Collective Benefit |
324
+
325
+ ### `submission.retained` Fields — What the Researcher Keeps
326
+
327
+ | Field | Type | Description |
328
+ |---|---|---|
329
+ | `publicationRights` | `boolean` | Right to publish about the method |
330
+ | `techniqueReuse` | `boolean` | Right to reuse techniques and architectural ideas in other work |
331
+ | `attribution` | `boolean` | Attribution credit as the method's creator |
332
+
333
+ ---
334
+
335
+ ## 8. `doNotTrain` vs `usageRestrictions.training`
336
+
337
+ These two fields record two independent facts. They share the word "training" but operate on different axes.
338
+
339
+ ### `doNotTrain`: Methodological Constraint
340
+
341
+ `doNotTrain` is a **boolean** consumed by the evaluation harness. When `true`, it means: "This data must not be used for training any ML model."
342
+
343
+ For evaluation corpora, the primary reason is **methodological** — if the test data leaks into a model's training set, the benchmark is contaminated and scores become meaningless. This is a data science concern, not a governance one.
344
+
345
+ ### `usageRestrictions.training`: Governance / Legal Fact
346
+
347
+ `usageRestrictions.training` records **who** said training is restricted and **why**:
348
+
349
+ - `"prohibited-by-license"` — the license prohibits it (legal fact)
350
+ - `"prohibited-by-community"` — a governance body prohibits it (governance assertion)
351
+ - `"discouraged"` — not prohibited but unwanted
352
+ - `"permitted"` — no restriction
353
+
354
+ ### Independence of the Two Axes
355
+
356
+ | Scenario | `doNotTrain` | `usageRestrictions.training` | Why both values coexist |
357
+ |---|---|---|---|
358
+ | Eval corpus, license permits training | `true` | `"permitted"` | Training is methodologically invalid (contaminates the benchmark) but the license doesn't prohibit it. |
359
+ | Eval corpus, community prohibits training | `true` | `"prohibited-by-community"` | Both axes agree: don't train. But for different reasons — one methodological, one governance. |
360
+ | Reference corpus, NC license | `false` | `"prohibited-by-license"` | Not an eval set (no methodological concern), but the license prohibits training. |
361
+ | Open reference corpus | `false` | `"permitted"` | No restrictions from either axis. |
362
+
363
+ ### Consistency Rule
364
+
365
+ The schema description states: "These must be consistent: if `doNotTrain` is `true`, `usageRestrictions.training` must not be `'permitted'`" — **except** when `doNotTrain` is `true` for purely methodological reasons on an eval corpus where the license genuinely permits training. In that case, `usageRestrictions.training` being `"permitted"` is accurate (no one prohibited it; we're just not doing it because it would contaminate the eval).
366
+
367
+ > [!IMPORTANT]
368
+ > If `usageRestrictions.training` is `"prohibited-by-license"` or `"prohibited-by-community"`, then `doNotTrain` **must** be `true`. The converse is not always true — `doNotTrain` can be `true` for methodological reasons alone.
369
+
370
+ ---
371
+
372
+ ## 9. Croissant Export Mapping
373
+
374
+ When corpora card metadata is exported to [Croissant ML 1.1](https://mlcommons.org/croissant/) JSON-LD, the sovereignty and usage restriction fields map as follows. Croissant uses Schema.org vocabulary, ODRL policies, and DUO codes.
375
+
376
+ ### Field-by-Field Mapping
377
+
378
+ | Corpora card field | Croissant ML 1.1 property | Notes |
379
+ |---|---|---|
380
+ | `license.spdx` | `schema:license` | Direct mapping. SPDX identifier or URL to license text. |
381
+ | `doNotTrain` | `schema:usageInfo` | When `true`, emit `"This dataset must not be used for ML model training."` in `schema:usageInfo`. |
382
+ | `usageRestrictions.training = "prohibited-by-license"` | ODRL `odrl:prohibition` with `odrl:action = odrl:use` scoped to training | Emit an ODRL policy prohibiting training use. Also emit DUO:0000004 (no general methods research) as `schema:additionalType`. |
383
+ | `usageRestrictions.training = "prohibited-by-community"` | `schema:usageInfo` + custom annotation | No standard ODRL/DUO mapping for community assertions. Emit in `schema:usageInfo` as free text with `schema:creator` pointing to `sovereignty.governanceOrg`. |
384
+ | `usageRestrictions.commercialUse = "prohibited-by-license"` | ODRL `odrl:prohibition` with `odrl:action = odrl:commercialize` | Standard ODRL mapping. Also emit DUO:0000018 (not for profit, non-commercial use only). |
385
+ | `usageRestrictions.commercialUse = "prohibited-by-community"` | `schema:usageInfo` | Community assertion — no ODRL equivalent. Document in `schema:usageInfo`. |
386
+ | `usageRestrictions.redistribution = "prohibited"` | ODRL `odrl:prohibition` with `odrl:action = odrl:distribute` | Standard ODRL mapping. |
387
+ | `usageRestrictions.redistribution = "same-terms"` | ODRL `odrl:duty` with `odrl:action = odrl:shareAlike` | Maps to share-alike duty. |
388
+ | `sovereignty.frameworks` | `schema:additionalType` (array) | Emit framework URIs as type annotations on the dataset. |
389
+ | `sovereignty.governanceOrg` | `schema:maintainer` or `schema:funder` | Use `schema:maintainer` for governance authority. Distinct from `schema:publisher` (which maps to `source.publisher`). |
390
+ | `sovereignty.tkLabels[]` | `schema:usageInfo` + `schema:url` | No native Croissant support for TK Labels. Emit label codes in `schema:usageInfo` and link to Local Contexts Hub URLs. |
391
+ | `sovereignty.consentModel` | `schema:conditionsOfAccess` | Maps to Croissant's access conditions. `"per-submission"` → `"Authorization required per use."` |
392
+ | `usageRestrictions.communityNotes` | `schema:usageInfo` (appended) | Free text appended to usage info. |
393
+
394
+ ### DUO Code Mapping
395
+
396
+ | `usageRestrictions` value | DUO Code | DUO Term |
397
+ |---|---|---|
398
+ | `training = "prohibited-by-license"` | DUO:0000004 | No general methods research |
399
+ | `commercialUse = "prohibited-by-license"` | DUO:0000018 | Not for profit, non-commercial use only |
400
+ | `commercialUse = "requires-agreement"` | DUO:0000021 | Ethics approval required |
401
+
402
+ > [!NOTE]
403
+ > DUO codes only apply to license-derived restrictions. Community governance assertions (`*-by-community`) have no DUO equivalent — DUO is designed for consent-based data access in biomedical contexts, not Indigenous data sovereignty. Community assertions are emitted as `schema:usageInfo` free text.
404
+
405
+ ### ODRL Policy Structure
406
+
407
+ ```json
408
+ {
409
+ "@context": "http://www.w3.org/ns/odrl.jsonld",
410
+ "@type": "odrl:Set",
411
+ "odrl:prohibition": [
412
+ {
413
+ "odrl:action": "odrl:use",
414
+ "odrl:constraint": {
415
+ "odrl:leftOperand": "odrl:purpose",
416
+ "odrl:operator": "odrl:eq",
417
+ "odrl:rightOperand": "model-training"
418
+ }
419
+ }
420
+ ]
421
+ }
422
+ ```
423
+
424
+ > [!IMPORTANT]
425
+ > Croissant export is not yet implemented. The mappings above are the design specification. Implementation is a follow-up task.
426
+
427
+ ---
428
+
429
+ ## 10. Anti-Patterns
430
+
431
+ These are documented mistakes to avoid when populating sovereignty fields. Each anti-pattern is paired with the correct approach.
432
+
433
+ ### Don't invent governance
434
+
435
+ **Wrong:** Populating `sovereignty.governanceOrg` with a best-guess organization because a governance body "probably" exists.
436
+
437
+ **Right:** If no governance body has been identified, `sovereignty` is `null`. The `sovereignty.notes` field can document "Governance body identification in progress" if there's an active effort, but the structured fields remain empty until there's a confirmed answer.
438
+
439
+ ### Don't derive sovereignty from vitality
440
+
441
+ **Wrong:** Seeing that a language is `"severely-endangered"` in `vitality.unescoStatus` and automatically populating OCAP/CARE frameworks, TK Labels, or consent models.
442
+
443
+ **Right:** Sovereignty fields are populated from **documented evidence of governance adoption** — a formal resolution, a published data governance policy, or a registered Local Contexts Hub project. Endangerment status and sovereignty are orthogonal. A safe, widely-spoken language may have strong data sovereignty frameworks (e.g., Māori → Te Mana Raraunga). A critically endangered language may have no formal governance body.
444
+
445
+ ### Don't assign TK Labels
446
+
447
+ **Wrong:** A Champollion contributor decides that a corpus "should" have a TK-NC (Non-Commercial) label and adds it to `sovereignty.tkLabels`.
448
+
449
+ **Right:** TK and BC Labels are **community-asserted**. They can only be applied by the data's governing community through the [Local Contexts Hub](https://localcontexts.org/). Champollion records labels that communities have applied — it never assigns them. If no labels have been applied, use an empty array `[]`.
450
+
451
+ ### Don't suppress community objections
452
+
453
+ **Wrong:** A corpus has a permissive license (CC-BY-4.0) and the contributor sets `communityNotes: null` even though the community has expressed concerns about training use.
454
+
455
+ **Right:** `usageRestrictions.communityNotes` exists precisely to surface legitimate community concerns that the license doesn't capture. If a community has documented objections, record them — even when the license technically permits the use. The CARE Ethics principle requires surfacing these concerns.
456
+
457
+ ### Don't duplicate the license
458
+
459
+ **Wrong:** Setting `usageRestrictions.commercialUse` to `"prohibited-by-license"` and also writing "non-commercial use only" in `communityNotes` and also noting it in `sovereignty.notes`.
460
+
461
+ **Right:** `usageRestrictions` **adds to** the license, it doesn't restate it. If the license already captures a restriction, use `null` for the corresponding `usageRestrictions` field (the defer-to-license pattern, §3). Only populate `usageRestrictions` fields when there's information **beyond** what the license says.
462
+
463
+ ### Don't conflate custodian with governance
464
+
465
+ **Wrong:** Setting `sovereignty.governanceOrg` to `"University of Alberta"` because the university hosts the data.
466
+
467
+ **Right:** `governanceOrg` is who has governance **authority**. `custodian` is who physically holds the data. A university may be custodian while a tribal council is the governance authority. If there's no distinct custodian (i.e., the `source.publisher` is also the custodian), set `custodian` to `null`.
468
+
469
+ ### Don't populate sovereignty for reference corpora that don't need it
470
+
471
+ **Wrong:** Adding a skeleton `sovereignty` object with all-null fields to a FLORES+ reference card "just in case."
472
+
473
+ **Right:** If no governance structures exist, `sovereignty` is `null`. The schema accepts null. Don't create empty shells.
474
+
475
+ ---
476
+
477
+ ## 11. SQLite Design Note
478
+
479
+ Corpora metadata follows the atlas data architecture, where the **atlas store** (`cli/data/atlas.db`, projected into the language cards) is the single source of truth (SSOT) for structured data. (The earlier v2 database, `cli/data/champollion.db`, was retired and deleted 2026-08-18 — see `shared/cldf/deprecations.json`.) In this model:
480
+
481
+ - Structured data lives in SQLite (the atlas store, built by the ingest pipeline)
482
+ - JSON cards (language cards, corpora cards) become **generated output** — projected from the store, not edited directly
483
+ - Provenance is pinned per-value in the store, not per-card
484
+
485
+ For corpora cards specifically, this means:
486
+
487
+ | Current state | Future state |
488
+ |---|---|
489
+ | Corpora cards are hand-authored JSON files | Corpora metadata is stored in SQLite with sovereignty, stewardship, and restriction facts as structured rows |
490
+ | Schema validation checks JSON files directly | Schema validation runs against database-generated output |
491
+ | Sovereignty fields are populated by contributors editing JSON | Sovereignty facts are entered via ingestion scripts and validated against the database |
492
+
493
+ **This is a follow-up implementation.** The current corpora card schema defines the data model. The SQLite migration for corpora metadata has not yet been built. Until it is, corpora cards remain hand-authored JSON files validated against `corpora-card.schema.json`.
494
+
495
+ The data architecture behind this — the SQLite schema, the ingestion
496
+ pipeline, the CLDF strategy and the migration plan — is internal planning
497
+ material and is not published. The parts that affect anyone consuming this
498
+ package are the schemas shipped alongside it in `schemas/`, and the public
499
+ data-stewardship documentation at
500
+ <https://champollion.dev/docs/network/sovereignty/data-sovereignty>.