champollion 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +133 -0
- package/README.md +387 -0
- package/bin/cli.js +278 -0
- package/index.js +135 -0
- package/lib/api-key.js +127 -0
- package/lib/autofix.js +432 -0
- package/lib/bridge/method_bridge.py +430 -0
- package/lib/card-source-resolution.mjs +284 -0
- package/lib/cards/cache.js +169 -0
- package/lib/cards/env.js +82 -0
- package/lib/cards/fetch-card-child.js +38 -0
- package/lib/cards/reader.js +435 -0
- package/lib/cards/refresh.js +111 -0
- package/lib/cards/remote.js +387 -0
- package/lib/cldf-export.mjs +540 -0
- package/lib/cldf-terms.mjs +62 -0
- package/lib/command-help.js +790 -0
- package/lib/commands/audit.js +49 -0
- package/lib/commands/card.js +454 -0
- package/lib/commands/doctor.js +559 -0
- package/lib/commands/fonts.js +489 -0
- package/lib/commands/help.js +91 -0
- package/lib/commands/init.js +1259 -0
- package/lib/commands/integrity.js +148 -0
- package/lib/commands/leaderboard.js +478 -0
- package/lib/commands/lint.js +30 -0
- package/lib/commands/models.js +177 -0
- package/lib/commands/plugin.js +103 -0
- package/lib/commands/provenance.js +45 -0
- package/lib/commands/recommend.js +75 -0
- package/lib/commands/register-corpus.js +678 -0
- package/lib/commands/repair-script.js +42 -0
- package/lib/commands/seal-corpus.js +355 -0
- package/lib/commands/seo.js +72 -0
- package/lib/commands/serve.js +147 -0
- package/lib/commands/status.js +265 -0
- package/lib/commands/submit.js +332 -0
- package/lib/commands/sync.js +89 -0
- package/lib/commands/tm.js +573 -0
- package/lib/commands/verify.js +39 -0
- package/lib/commands/watch.js +20 -0
- package/lib/commands/wrap.js +138 -0
- package/lib/commands/xliff.js +327 -0
- package/lib/commercial-eligibility.js +235 -0
- package/lib/concurrent.js +87 -0
- package/lib/config.js +523 -0
- package/lib/contamination-lane.js +76 -0
- package/lib/content-sync.js +731 -0
- package/lib/content.js +733 -0
- package/lib/corpus-registration.mjs +608 -0
- package/lib/cost-report.js +346 -0
- package/lib/diff.js +155 -0
- package/lib/docusaurus-sync.js +1256 -0
- package/lib/flatten.js +55 -0
- package/lib/format.js +954 -0
- package/lib/hash.js +159 -0
- package/lib/icu.js +473 -0
- package/lib/integrity.js +689 -0
- package/lib/license-gate.mjs +478 -0
- package/lib/license-identify.mjs +229 -0
- package/lib/lint.js +629 -0
- package/lib/method-manifest.js +60 -0
- package/lib/methods/anthropic.js +140 -0
- package/lib/methods/apertium.js +163 -0
- package/lib/methods/api.js +316 -0
- package/lib/methods/base.js +184 -0
- package/lib/methods/content-separator.js +45 -0
- package/lib/methods/deepl.js +426 -0
- package/lib/methods/direct-llm.js +586 -0
- package/lib/methods/external.js +332 -0
- package/lib/methods/fetch-with-retry.js +124 -0
- package/lib/methods/gemini.js +147 -0
- package/lib/methods/google-translate.js +402 -0
- package/lib/methods/http-utils.js +122 -0
- package/lib/methods/libretranslate.js +314 -0
- package/lib/methods/llm-coached.js +670 -0
- package/lib/methods/llm.js +592 -0
- package/lib/methods/local.js +76 -0
- package/lib/methods/microsoft-translator.js +331 -0
- package/lib/methods/openai.js +131 -0
- package/lib/methods/openrouter-client.js +327 -0
- package/lib/methods/openrouter-pricing.js +156 -0
- package/lib/methods/provider-env.js +115 -0
- package/lib/methods/provider-pricing.js +310 -0
- package/lib/methods/tilde.js +150 -0
- package/lib/methods/translated.js +229 -0
- package/lib/methods/translation-error.js +80 -0
- package/lib/models.js +258 -0
- package/lib/no-translate.js +233 -0
- package/lib/output.js +238 -0
- package/lib/pairs.js +547 -0
- package/lib/plugins.js +447 -0
- package/lib/provenance.js +323 -0
- package/lib/recommend.js +648 -0
- package/lib/registers.js +1185 -0
- package/lib/repair-script.js +266 -0
- package/lib/scripts.js +994 -0
- package/lib/seal.mjs +464 -0
- package/lib/sealed-qualifier.mjs +211 -0
- package/lib/security.js +59 -0
- package/lib/segment.js +369 -0
- package/lib/seo.js +275 -0
- package/lib/serve.js +854 -0
- package/lib/string-classify.js +85 -0
- package/lib/submit.mjs +344 -0
- package/lib/sync.js +969 -0
- package/lib/tags/bcp47.js +202 -0
- package/lib/tags/resolve.js +314 -0
- package/lib/terminology.js +111 -0
- package/lib/tm-seed.js +294 -0
- package/lib/tm.js +515 -0
- package/lib/translate-pair.js +197 -0
- package/lib/translate.js +203 -0
- package/lib/types.js +230 -0
- package/lib/validate.js +510 -0
- package/lib/verify.js +451 -0
- package/lib/watch.js +145 -0
- package/lib/xliff.js +184 -0
- package/package.json +93 -0
- package/shared/ATTRIBUTION.md +145 -0
- package/shared/CORPORA-CARDS.md +288 -0
- package/shared/DATA-SOVEREIGNTY.md +500 -0
- package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
- package/shared/card-lint-baseline.json +3189 -0
- package/shared/cards-fallback.json +1 -0
- package/shared/catalogue/card-config.json +6091 -0
- package/shared/catalogue/external-results.json +3888 -0
- package/shared/catalogue/gender-guidance.json +1038 -0
- package/shared/catalogue/method-coverage.json +1751 -0
- package/shared/catalogue/metric-coverage.json +170 -0
- package/shared/catalogue/metric-reliability.json +1 -0
- package/shared/catalogue/register-presets.json +3180 -0
- package/shared/catalogue/vitality-scales.json +55 -0
- package/shared/cldr-index.json +1115 -0
- package/shared/code-bridge.json +253 -0
- package/shared/corpora-cards-v1-reference.md +281 -0
- package/shared/curated-dictionary-flags.json +35 -0
- package/shared/curated-endonyms.json +35 -0
- package/shared/curated-fsts.json +51 -0
- package/shared/curated-orthography-conventions.json +26 -0
- package/shared/curated-sil-resources.json +374 -0
- package/shared/curated-tools.json +41 -0
- package/shared/docent/corpus.json +11333 -0
- package/shared/docent/faq.en.json +564 -0
- package/shared/docent/register-blocks.json +60 -0
- package/shared/docent/system-prompt.md +144 -0
- package/shared/domain-taxonomy.json +35 -0
- package/shared/explainers/glossary.json +2975 -0
- package/shared/explainers/tc-features.json +20112 -0
- package/shared/explainers/term-watchlist.json +147 -0
- package/shared/human-services.json +59 -0
- package/shared/license-corrections.json +261 -0
- package/shared/license-evidence.json +13452 -0
- package/shared/licenses.json +6781 -0
- package/shared/method-registry.json +236 -0
- package/shared/metric-registry.json +620 -0
- package/shared/model-aliases.json +7 -0
- package/shared/schemas/champollion-plugin.schema.json +206 -0
- package/shared/schemas/corpora-card.schema.json +957 -0
- package/shared/schemas/domain-taxonomy.schema.json +64 -0
- package/shared/schemas/external-results.schema.json +314 -0
- package/shared/schemas/human-services.schema.json +90 -0
- package/shared/schemas/language-card.schema.json +1308 -0
- package/shared/schemas/licenses.schema.json +155 -0
- package/shared/schemas/method-card.schema.json +412 -0
- package/shared/schemas/method-registry.schema.json +85 -0
- package/shared/schemas/metric-registry.schema.json +96 -0
- package/shared/schemas/metric-reliability.schema.json +178 -0
- package/shared/schemas/model-aliases.schema.json +27 -0
- package/shared/schemas/source-snapshot.schema.json +96 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* card-source-resolution.mjs — the ONE definition of how a card's
|
|
3
|
+
* `_fieldSources` / `dataSources[]` stamp resolves to a license-register key.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS IS A SHARED MODULE
|
|
6
|
+
* This logic used to live only inside cli/scripts/lint-language-cards.mjs,
|
|
7
|
+
* unexported. So the license audit (cli/scripts/audit-license-gate.mjs) could
|
|
8
|
+
* not use it, and swept only `facts.source` in the (since-retired) legacy
|
|
9
|
+
* champollion.db facts table — 268 sources.
|
|
10
|
+
* Cards cite 801 distinct stamps, and PHOIBLE has NO facts row at all (it
|
|
11
|
+
* flows straight from phoible-raw.csv into cards), so the sole tone authority
|
|
12
|
+
* on 5,219 card stamps was outside the license sweep entirely.
|
|
13
|
+
*
|
|
14
|
+
* Any second copy of this vocabulary would drift — the alias map alone
|
|
15
|
+
* encodes 30 hand-justified renames. One definition, several consumers.
|
|
16
|
+
*
|
|
17
|
+
* Consumers: lint-language-cards.mjs (license-source-resolves rule),
|
|
18
|
+
* audit-license-gate.mjs --cards.
|
|
19
|
+
*
|
|
20
|
+
* @see docs/LICENSING.md
|
|
21
|
+
* @see shared/licenses.json
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import fs from 'node:fs';
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Build the resolution index from a license register file.
|
|
28
|
+
* Returns null if absent, or { sourceKeys, baseIndex, spdxSet }.
|
|
29
|
+
*/
|
|
30
|
+
export function buildRegisterIndex(filePath) {
|
|
31
|
+
if (!fs.existsSync(filePath)) return null;
|
|
32
|
+
let data;
|
|
33
|
+
try {
|
|
34
|
+
data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
35
|
+
} catch {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
const sources = data.sources || {};
|
|
39
|
+
const sourceKeys = new Set(Object.keys(sources));
|
|
40
|
+
|
|
41
|
+
// Base-name index: register keys with version tokens stripped, so
|
|
42
|
+
// "glottolog-5.3" (card) matches "glottolog-5.0" (register) via "glottolog".
|
|
43
|
+
const baseIndex = new Map();
|
|
44
|
+
for (const key of sourceKeys) {
|
|
45
|
+
const base = stripVersionTokens(key);
|
|
46
|
+
if (!baseIndex.has(base)) baseIndex.set(base, key);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const spdxSet = new Set();
|
|
50
|
+
for (const v of Object.values(sources)) {
|
|
51
|
+
if (v && typeof v.license_spdx === 'string' && v.license_spdx.trim()) {
|
|
52
|
+
spdxSet.add(v.license_spdx.trim());
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return { sourceKeys, baseIndex, spdxSet, sources };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Every distinct source stamp a card asserts, from BOTH conventions:
|
|
60
|
+
* - `_fieldSources` (rich cards), whose values may be a string, an array,
|
|
61
|
+
* or an object of per-subfield stamps; and
|
|
62
|
+
* - `dataSources[]` plus inline `<field>.source` keys, which is how the 751
|
|
63
|
+
* Glottolog-only stubs stamp — they carry no `_fieldSources` map at all.
|
|
64
|
+
*
|
|
65
|
+
* A sweep reading only `_fieldSources` silently skips those 751 cards.
|
|
66
|
+
*/
|
|
67
|
+
export function cardSourceStamps(card) {
|
|
68
|
+
const out = new Set();
|
|
69
|
+
const add = (v) => {
|
|
70
|
+
if (typeof v === 'string' && v.trim()) out.add(v.trim());
|
|
71
|
+
else if (Array.isArray(v)) v.forEach(add);
|
|
72
|
+
else if (v && typeof v === 'object') Object.values(v).forEach(add);
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
add(card._fieldSources);
|
|
76
|
+
add(card.dataSources);
|
|
77
|
+
|
|
78
|
+
// Inline `<field>.source`, ONLY on top-level object-valued fields.
|
|
79
|
+
//
|
|
80
|
+
// Deliberately not arrays. `evalDatasets[].source`, `resources.fsts[].source`
|
|
81
|
+
// and `orthographies[].source` all carry a `source` key meaning something
|
|
82
|
+
// else entirely — a publisher name, an FST repo URL, a script name like
|
|
83
|
+
// "Latin". Harvesting those produced 620+ phantom "unlicensed sources" that
|
|
84
|
+
// were never license claims at all. The stub convention this exists for is
|
|
85
|
+
// `coordinates.source` / `vitality.source` on the 751 Glottolog-only cards,
|
|
86
|
+
// which are plain objects.
|
|
87
|
+
for (const [key, value] of Object.entries(card)) {
|
|
88
|
+
if (key.startsWith('_') || !value || typeof value !== 'object') continue;
|
|
89
|
+
if (Array.isArray(value)) continue;
|
|
90
|
+
if (PROSE_SOURCE_FIELDS.has(key)) continue;
|
|
91
|
+
if (typeof value.source === 'string') add(value.source);
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Top-level fields whose `.source` is a BIBLIOGRAPHIC CITATION, not a dataset
|
|
98
|
+
* id — so it can never resolve against the license register and must not be
|
|
99
|
+
* reported as an unlicensed source.
|
|
100
|
+
*
|
|
101
|
+
* `culturalAphorism.source` (77 cards) reads e.g. "Traditional Turkish proverb
|
|
102
|
+
* (atasözü). Documented in Aksoy, Ö.A. (1988)…" — an attribution to oral
|
|
103
|
+
* literature with scholarly documentation. That is the card citing its work
|
|
104
|
+
* properly, which is the house rule, not a licensing gap.
|
|
105
|
+
*
|
|
106
|
+
* Kept as a named list rather than a prose-detecting heuristic: a heuristic
|
|
107
|
+
* would eventually swallow a REAL stamp and silently shrink the sweep, which is
|
|
108
|
+
* the failure this audit exists to prevent.
|
|
109
|
+
*/
|
|
110
|
+
export const PROSE_SOURCE_FIELDS = new Set(['culturalAphorism']);
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Strip version/date tokens from a source id.
|
|
114
|
+
* Handles every versioning style observed in card data:
|
|
115
|
+
* glottolog-5.3, wals-2024, asjp-v20, grambank-1.0.3, common-voice-20.0,
|
|
116
|
+
* glottolog-5.x-cldf (mid-string), wikidata-P282 (property suffix),
|
|
117
|
+
* unimorph-4.0, northeuralex-0.9, cldr-48
|
|
118
|
+
*/
|
|
119
|
+
export function stripVersionTokens(id) {
|
|
120
|
+
return id
|
|
121
|
+
// version token anywhere: -5.3, -2024, -v20, -1.0.3, -5.x, -P282
|
|
122
|
+
.replace(/-(?:v?\d+(?:[._]\d+)*[a-z]?|\d+\.x|P\d+)(?=-|$)/g, '')
|
|
123
|
+
// trailing parenthetical citation: "cldr-endonym (matched via fa)"
|
|
124
|
+
.replace(/\s*\(.*\)\s*$/, '')
|
|
125
|
+
.trim();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Source ids that denote INTERNAL derivation/curation steps rather than
|
|
130
|
+
* external datasets. These resolve to the register's
|
|
131
|
+
* 'champollion-derived' entry (LicenseRef-Champollion-Derived).
|
|
132
|
+
*/
|
|
133
|
+
export const INTERNAL_SOURCE_PATTERNS = [
|
|
134
|
+
/^derived([:\-]|$)/, // derived, derived:script, derived-from-…
|
|
135
|
+
/^champollion-derived([\s:(\[-]|$)/, // champollion-derived [family-level prior; …]
|
|
136
|
+
/^corpora-cards(\s*\(|$)/, // corpora-cards (sync-eval-datasets.mjs) — the
|
|
137
|
+
// repo's own eval registry; per-corpus licenses
|
|
138
|
+
// live on the corpora cards themselves
|
|
139
|
+
/^template-generated([:\-]|$)/, // template-generated:grambank-GB415 (…)
|
|
140
|
+
/^manual-curation(-|$)/, // manual-curation, manual-curation-verified
|
|
141
|
+
/^manual-curation\+\S+$/, // manual-curation+<citation-url> — curated
|
|
142
|
+
// endonym seed stamps (apply-curated-endonyms.mjs);
|
|
143
|
+
// the suffix is a citation, not a source id
|
|
144
|
+
/^cleanup(-resources)?-phase\d+$/,
|
|
145
|
+
/^api-verification(-\d+)?$/,
|
|
146
|
+
/^not-populated$/,
|
|
147
|
+
];
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Alias map: card-source vocabulary → register vocabulary.
|
|
151
|
+
* Applied AFTER version stripping (keys are base forms). Every entry is
|
|
152
|
+
* justified by an observed mismatch between the enrichment pipeline's
|
|
153
|
+
* source ids and the tc-licenses register keys — do not add speculative
|
|
154
|
+
* aliases.
|
|
155
|
+
*/
|
|
156
|
+
export const SOURCE_ALIASES = new Map(Object.entries({
|
|
157
|
+
'iso639-3': 'sil-iso639-3',
|
|
158
|
+
'iso639': 'sil-iso639-3', // 'iso639-3-2024' version-strips to 'iso639'
|
|
159
|
+
'olac-aggregator': 'olac',
|
|
160
|
+
'huggingface-datasets': 'huggingface',
|
|
161
|
+
'keyman-api': 'keyman',
|
|
162
|
+
'kaikki-wiktionary': 'wiktionary',
|
|
163
|
+
'kaikki-wiktextract': 'wiktionary',
|
|
164
|
+
'opus-nlpl': 'opus',
|
|
165
|
+
'opus-nlpl-api': 'opus-api',
|
|
166
|
+
'ucla-phonetics': 'uclaphoneticslabarchive',
|
|
167
|
+
'huntergatherer-db': 'huntergatherer',
|
|
168
|
+
'huntergatherer-lexibank': 'huntergatherer',
|
|
169
|
+
'nllb': 'meta-nllb', // nllb-200 → meta-nllb-200 (base forms)
|
|
170
|
+
'cldr': 'unicode-cldr',
|
|
171
|
+
'wikimedia-sitematrix': 'wikipedia-sitematrix',
|
|
172
|
+
'siteinfo': 'wikipedia-sitematrix', // "wikimedia-sitematrix+siteinfo" components
|
|
173
|
+
'numeralbank-channumerals': 'numeralbank',
|
|
174
|
+
'datsemshift-zalizniak': 'datsemshift',
|
|
175
|
+
'acd-blust': 'acd',
|
|
176
|
+
'valpal-hartmann': 'valpal',
|
|
177
|
+
'bowern-pny': 'bowernpny',
|
|
178
|
+
'vanuatu-voices': 'vanuatuvoices',
|
|
179
|
+
'glottolog-aes': 'glottolog',
|
|
180
|
+
'glottolog-languoid': 'glottolog', // glottolog-5.x-languoid
|
|
181
|
+
'sagart-sino-tibetan': 'sagartst', // same Sagart et al. Sino-Tibetan dataset
|
|
182
|
+
'ids-cldf': 'ids', // CLDF edition of IDS (same dataset/license)
|
|
183
|
+
'paradisec-olac': 'paradisec', // PARADISEC catalog metadata harvested via OLAC
|
|
184
|
+
'rosetta-project-ia': 'rosetta-project', // same collection, Internet Archive hosting
|
|
185
|
+
'language-atlas-pacific': 'wurm-hattori-pacific-atlas', // same atlas, two card vocabularies
|
|
186
|
+
'lexibank-batch': 'lexibank', // batch enrichment over the lexibank umbrella
|
|
187
|
+
'cldr-endonym': 'unicode-cldr', // CLDR endonym data ("cldr-endonym (matched via fa)")
|
|
188
|
+
'wikidata-rdfs-label-own': 'wikidata', // own-language rdfs:label tier of
|
|
189
|
+
// harvest-wikidata-endonyms.mjs (same CC0 dataset)
|
|
190
|
+
}));
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Resolve ONE source id component (no '+') against the register.
|
|
194
|
+
* Returns the register key it resolves to, or null.
|
|
195
|
+
*/
|
|
196
|
+
export function resolveSourceComponent(id, register) {
|
|
197
|
+
const { sourceKeys, baseIndex } = register;
|
|
198
|
+
|
|
199
|
+
// 1. Exact match
|
|
200
|
+
if (sourceKeys.has(id)) return id;
|
|
201
|
+
|
|
202
|
+
// 2. Internal derivation pseudo-sources → champollion-derived
|
|
203
|
+
for (const pattern of INTERNAL_SOURCE_PATTERNS) {
|
|
204
|
+
if (pattern.test(id)) return 'champollion-derived';
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// 3. Version-stripped base match (both sides base-normalized)
|
|
208
|
+
let base = stripVersionTokens(id);
|
|
209
|
+
if (sourceKeys.has(base)) return base;
|
|
210
|
+
if (baseIndex.has(base)) return baseIndex.get(base);
|
|
211
|
+
|
|
212
|
+
// 4. Alias rename
|
|
213
|
+
if (SOURCE_ALIASES.has(base)) {
|
|
214
|
+
const target = SOURCE_ALIASES.get(base);
|
|
215
|
+
if (sourceKeys.has(target)) return target;
|
|
216
|
+
if (baseIndex.has(target)) return baseIndex.get(target);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// 5. lexibank-<dataset> prefix / <dataset>-lexibank suffix: register
|
|
220
|
+
// keys are bare dataset names (e.g. 'dyenindoeuropean', 'abvd',
|
|
221
|
+
// 'uralex', 'crossandean'); cards tag them with the lexibank family.
|
|
222
|
+
if (base.startsWith('lexibank-')) {
|
|
223
|
+
const bare = base.slice('lexibank-'.length);
|
|
224
|
+
if (sourceKeys.has(bare)) return bare;
|
|
225
|
+
if (baseIndex.has(bare)) return baseIndex.get(bare);
|
|
226
|
+
}
|
|
227
|
+
if (base.endsWith('-lexibank')) {
|
|
228
|
+
const bare = base.slice(0, -'-lexibank'.length);
|
|
229
|
+
if (sourceKeys.has(bare)) return bare;
|
|
230
|
+
if (baseIndex.has(bare)) return baseIndex.get(bare);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// 6. FEATURE-ID citation: "<dataset>-<featureId>", e.g. grambank-GB415,
|
|
234
|
+
// wals-44A, wals-13A. The card is citing a specific typological feature,
|
|
235
|
+
// and the LICENSED thing is the dataset — Grambank's license governs
|
|
236
|
+
// GB415 exactly as it governs every other feature. Without this, 2,006
|
|
237
|
+
// card-citations of Grambank and WALS features read as unlicensed
|
|
238
|
+
// sources, which would have buried the real findings under noise.
|
|
239
|
+
// Deliberately narrow: the suffix must be a feature-code shape (letters
|
|
240
|
+
// then digits, or digits then a letter), never an arbitrary word.
|
|
241
|
+
const featureM = /^(.+)-(?:[A-Z]{2}\d+|\d+[A-Z])$/.exec(base);
|
|
242
|
+
if (featureM) {
|
|
243
|
+
const dataset = featureM[1];
|
|
244
|
+
if (sourceKeys.has(dataset)) return dataset;
|
|
245
|
+
if (baseIndex.has(dataset)) return baseIndex.get(dataset);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Resolve a full source value, which may be a compound "a+b+c" string.
|
|
253
|
+
* Returns { ok: boolean, unresolved: string[], malformed: boolean }.
|
|
254
|
+
*/
|
|
255
|
+
export function resolveSourceValue(value, register) {
|
|
256
|
+
// Stringified-array data bug: "['a', 'b']+c" — produced by an upstream
|
|
257
|
+
// enrichment step that joined a Python list repr instead of its items.
|
|
258
|
+
// Unparseable as source ids; flag the whole value as malformed. A bare
|
|
259
|
+
// '[' is NOT malformed by itself — internal stamps legitimately carry
|
|
260
|
+
// bracketed citations ("champollion-derived [family-level prior; …]"),
|
|
261
|
+
// so only the quote-bearing list-repr shape is flagged (2026-07-07).
|
|
262
|
+
if (value.includes("['") || value.includes("'")) {
|
|
263
|
+
return { ok: false, unresolved: [value], malformed: true };
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Whole-value internal derivations like "derived:cldr+script+keyboard"
|
|
267
|
+
// describe a derivation recipe, not a '+'-joined source list — the
|
|
268
|
+
// components after the first are recipe inputs ("script", "keyboard"),
|
|
269
|
+
// not source ids. Match internal patterns against the full value first.
|
|
270
|
+
for (const pattern of INTERNAL_SOURCE_PATTERNS) {
|
|
271
|
+
if (pattern.test(value)) return { ok: true, unresolved: [], malformed: false };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Parenthetical citations may legitimately contain spaces; only split
|
|
275
|
+
// on '+' that separates components.
|
|
276
|
+
const components = value.split('+').map(c => c.trim()).filter(Boolean);
|
|
277
|
+
const unresolved = [];
|
|
278
|
+
for (const component of components) {
|
|
279
|
+
if (!resolveSourceComponent(component, register)) {
|
|
280
|
+
unresolved.push(component);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return { ok: unresolved.length === 0, unresolved, malformed: false };
|
|
284
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-user card cache — ~/.champollion/cards/
|
|
3
|
+
*
|
|
4
|
+
* Layout:
|
|
5
|
+
* <code>.json — { v, code, updatedAt, fetchedAt, card }
|
|
6
|
+
* _state.json — { lastRefreshCheckAt, backoffUntil, missing: { code: ts } }
|
|
7
|
+
*
|
|
8
|
+
* updatedAt mirrors the per-language updated_at from the Supabase
|
|
9
|
+
* trading-card tables; invalidation works by deleting entries whose
|
|
10
|
+
* remote updated_at moved past the cached one (lib/cards/refresh.js)
|
|
11
|
+
* and refetching lazily on next use.
|
|
12
|
+
*
|
|
13
|
+
* `missing` tombstones record codes prod definitively doesn't have, so
|
|
14
|
+
* unknown locale codes don't trigger a network attempt on every CLI
|
|
15
|
+
* run. Tombstones expire with MISSING_TTL_MS and are cleared by the
|
|
16
|
+
* refresh pass when a language appears upstream.
|
|
17
|
+
*
|
|
18
|
+
* All writes are atomic (tmp + rename) and all reads tolerate missing
|
|
19
|
+
* or corrupt files — a broken cache must never break the CLI.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import fs from 'node:fs';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
import { CACHE_DIR, isValidCardCode } from './env.js';
|
|
25
|
+
|
|
26
|
+
const ENTRY_VERSION = 1;
|
|
27
|
+
const STATE_FILE = '_state.json';
|
|
28
|
+
|
|
29
|
+
/** Failed-fetch backoff — don't hammer the network on every invocation. */
|
|
30
|
+
export const BACKOFF_MS = 15 * 60 * 1000;
|
|
31
|
+
/** Missing-language tombstone lifetime. */
|
|
32
|
+
export const MISSING_TTL_MS = 24 * 60 * 60 * 1000;
|
|
33
|
+
|
|
34
|
+
function entryPath(code) {
|
|
35
|
+
return path.join(CACHE_DIR, `${code}.json`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function readJsonSafe(file) {
|
|
39
|
+
try {
|
|
40
|
+
return JSON.parse(fs.readFileSync(file, 'utf-8'));
|
|
41
|
+
} catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function writeJsonAtomic(file, value) {
|
|
47
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
48
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
49
|
+
fs.writeFileSync(tmp, JSON.stringify(value), 'utf-8');
|
|
50
|
+
fs.renameSync(tmp, file);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Read a cached card entry. Returns { card, updatedAt } or null. */
|
|
54
|
+
export function readCachedCard(code) {
|
|
55
|
+
if (!isValidCardCode(code)) return null;
|
|
56
|
+
const entry = readJsonSafe(entryPath(code));
|
|
57
|
+
if (!entry || entry.v !== ENTRY_VERSION || entry.code !== code || !entry.card) return null;
|
|
58
|
+
return { card: entry.card, updatedAt: entry.updatedAt || null };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function writeCachedCard(code, card, updatedAt) {
|
|
62
|
+
if (!isValidCardCode(code)) return false;
|
|
63
|
+
try {
|
|
64
|
+
writeJsonAtomic(entryPath(code), {
|
|
65
|
+
v: ENTRY_VERSION,
|
|
66
|
+
code,
|
|
67
|
+
updatedAt: updatedAt || null,
|
|
68
|
+
fetchedAt: new Date().toISOString(),
|
|
69
|
+
card,
|
|
70
|
+
});
|
|
71
|
+
return true;
|
|
72
|
+
} catch {
|
|
73
|
+
return false; // read-only HOME etc. — cache is best-effort
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function removeCachedCard(code) {
|
|
78
|
+
if (!isValidCardCode(code)) return;
|
|
79
|
+
try {
|
|
80
|
+
fs.unlinkSync(entryPath(code));
|
|
81
|
+
} catch {
|
|
82
|
+
// already gone
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Codes currently materialized in the cache (excludes state/tmp files). */
|
|
87
|
+
export function listCachedCodes() {
|
|
88
|
+
let entries;
|
|
89
|
+
try {
|
|
90
|
+
entries = fs.readdirSync(CACHE_DIR);
|
|
91
|
+
} catch {
|
|
92
|
+
return [];
|
|
93
|
+
}
|
|
94
|
+
const codes = [];
|
|
95
|
+
for (const name of entries) {
|
|
96
|
+
if (!name.endsWith('.json') || name === STATE_FILE) continue;
|
|
97
|
+
const code = name.slice(0, -5);
|
|
98
|
+
if (isValidCardCode(code)) codes.push(code);
|
|
99
|
+
}
|
|
100
|
+
return codes;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// -----------------------------------------------------------------
|
|
104
|
+
// Shared state — refresh stamp, network backoff, missing tombstones
|
|
105
|
+
// -----------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
export function readState() {
|
|
108
|
+
const state = readJsonSafe(path.join(CACHE_DIR, STATE_FILE));
|
|
109
|
+
return state && typeof state === 'object'
|
|
110
|
+
? state
|
|
111
|
+
: { lastRefreshCheckAt: null, backoffUntil: null, missing: {} };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function writeState(patch) {
|
|
115
|
+
try {
|
|
116
|
+
const next = { ...readState(), ...patch };
|
|
117
|
+
writeJsonAtomic(path.join(CACHE_DIR, STATE_FILE), next);
|
|
118
|
+
return next;
|
|
119
|
+
} catch {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** True while a previous network failure says "don't try again yet". */
|
|
125
|
+
export function inBackoff(state = readState()) {
|
|
126
|
+
return Boolean(state.backoffUntil && Date.parse(state.backoffUntil) > Date.now());
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function noteNetworkFailure() {
|
|
130
|
+
writeState({ backoffUntil: new Date(Date.now() + BACKOFF_MS).toISOString() });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export function clearBackoff() {
|
|
134
|
+
writeState({ backoffUntil: null });
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** True when `code` was recently confirmed absent from prod. */
|
|
138
|
+
export function isTombstoned(code, state = readState()) {
|
|
139
|
+
const ts = state.missing?.[code];
|
|
140
|
+
return Boolean(ts && Date.now() - Date.parse(ts) < MISSING_TTL_MS);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function tombstone(code) {
|
|
144
|
+
if (!isValidCardCode(code)) return;
|
|
145
|
+
const state = readState();
|
|
146
|
+
const missing = { ...(state.missing || {}) };
|
|
147
|
+
missing[code] = new Date().toISOString();
|
|
148
|
+
// Don't let the tombstone map grow without bound.
|
|
149
|
+
const keys = Object.keys(missing);
|
|
150
|
+
if (keys.length > 500) {
|
|
151
|
+
keys.sort((a, b) => Date.parse(missing[a]) - Date.parse(missing[b]));
|
|
152
|
+
for (const key of keys.slice(0, keys.length - 500)) delete missing[key];
|
|
153
|
+
}
|
|
154
|
+
writeState({ missing });
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function clearTombstones(codes) {
|
|
158
|
+
const state = readState();
|
|
159
|
+
if (!state.missing) return;
|
|
160
|
+
const missing = { ...state.missing };
|
|
161
|
+
let changed = false;
|
|
162
|
+
for (const code of codes) {
|
|
163
|
+
if (code in missing) {
|
|
164
|
+
delete missing[code];
|
|
165
|
+
changed = true;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (changed) writeState({ missing });
|
|
169
|
+
}
|
package/lib/cards/env.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Card-loading environment — paths, endpoints, and mode detection.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately tiny with zero heavy imports: bin/cli.js consults this
|
|
5
|
+
* module on every invocation to decide whether the background cache
|
|
6
|
+
* refresh applies, without paying the cost of loading the full card
|
|
7
|
+
* registry (lib/registers.js eager-loads thousands of files in repo
|
|
8
|
+
* checkouts).
|
|
9
|
+
*
|
|
10
|
+
* All values are read once at import. Override via environment for
|
|
11
|
+
* tests and staging:
|
|
12
|
+
* CHAMPOLLION_CARDS_DIR — full card directory (repo mode)
|
|
13
|
+
* CHAMPOLLION_CARDS_FALLBACK — bundled fallback JSON (packaged mode)
|
|
14
|
+
* CHAMPOLLION_CARDS_CACHE_DIR — per-user card cache
|
|
15
|
+
* CHAMPOLLION_SUPABASE_URL — REST endpoint base
|
|
16
|
+
* CHAMPOLLION_SUPABASE_ANON_KEY— read-only publishable key
|
|
17
|
+
* CHAMPOLLION_OFFLINE=1 — never touch the network
|
|
18
|
+
* CHAMPOLLION_CARDS_TTL_HOURS — staleness-check interval (default 24)
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import fs from 'node:fs';
|
|
22
|
+
import os from 'node:os';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Production Supabase project (MT Eval Arena). The publishable anon key
|
|
27
|
+
* is intentionally committed — Row Level Security restricts the anon
|
|
28
|
+
* role to SELECT on the public trading-card tables (see
|
|
29
|
+
* mt-eval-arena/supabase/migrations/012_create_trading_cards.sql).
|
|
30
|
+
* The CLI only ever reads.
|
|
31
|
+
*/
|
|
32
|
+
export const SUPABASE_URL =
|
|
33
|
+
process.env.CHAMPOLLION_SUPABASE_URL || 'https://sjdomynysdljkbemupqa.supabase.co';
|
|
34
|
+
export const SUPABASE_ANON_KEY =
|
|
35
|
+
process.env.CHAMPOLLION_SUPABASE_ANON_KEY || 'sb_publishable_bV6CFNFnzxhQI0wlBx2J0A_5Vm5gFBp';
|
|
36
|
+
|
|
37
|
+
/** Full card directory — present in repo checkouts, excluded from the npm tarball. */
|
|
38
|
+
export const CARDS_DIR =
|
|
39
|
+
process.env.CHAMPOLLION_CARDS_DIR ||
|
|
40
|
+
path.join(import.meta.dirname, '..', '..', 'shared', 'language-cards');
|
|
41
|
+
|
|
42
|
+
/** Bundled fallback (core cards + parents + manifest) — ships in the package. */
|
|
43
|
+
export const FALLBACK_FILE =
|
|
44
|
+
process.env.CHAMPOLLION_CARDS_FALLBACK ||
|
|
45
|
+
path.join(import.meta.dirname, '..', '..', 'shared', 'cards-fallback.json');
|
|
46
|
+
|
|
47
|
+
/** Per-user cache of cards fetched from Supabase. */
|
|
48
|
+
export const CACHE_DIR =
|
|
49
|
+
process.env.CHAMPOLLION_CARDS_CACHE_DIR ||
|
|
50
|
+
path.join(os.homedir(), '.champollion', 'cards');
|
|
51
|
+
|
|
52
|
+
export const OFFLINE = process.env.CHAMPOLLION_OFFLINE === '1';
|
|
53
|
+
|
|
54
|
+
/** How often (hours) the CLI checks Supabase for per-language updated_at bumps. */
|
|
55
|
+
export const REFRESH_TTL_HOURS = Number(process.env.CHAMPOLLION_CARDS_TTL_HOURS) > 0
|
|
56
|
+
? Number(process.env.CHAMPOLLION_CARDS_TTL_HOURS)
|
|
57
|
+
: 24;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Repo mode = the full card directory exists on disk (git checkout,
|
|
61
|
+
* monorepo dev). Packaged mode = npm install, where only the fallback
|
|
62
|
+
* bundle ships and the long tail is fetched + cached on demand.
|
|
63
|
+
*/
|
|
64
|
+
export function hasLocalCardsDir() {
|
|
65
|
+
try {
|
|
66
|
+
return fs.existsSync(CARDS_DIR);
|
|
67
|
+
} catch {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Locale codes flow from user config into cache file paths and REST
|
|
74
|
+
* query params, so gate them hard before any fs/network use.
|
|
75
|
+
* Accepts ISO-ish tags: letters/digits with - or _ separators
|
|
76
|
+
* ('fra', 'fra-CA', 'x-pirate', 'cmn-Hant'), max 5 segments.
|
|
77
|
+
*/
|
|
78
|
+
const CODE_RE = /^[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+){0,4}$/;
|
|
79
|
+
|
|
80
|
+
export function isValidCardCode(code) {
|
|
81
|
+
return typeof code === 'string' && code.length >= 2 && code.length <= 48 && CODE_RE.test(code);
|
|
82
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One-shot card fetcher — child-process entry point.
|
|
3
|
+
*
|
|
4
|
+
* The public registry API (getLanguageCard et al.) is synchronous, but
|
|
5
|
+
* Node has no synchronous HTTP. When the packaged CLI misses on a code
|
|
6
|
+
* that the manifest says exists, lib/registers.js runs this script via
|
|
7
|
+
* execFileSync and parses the single JSON object it prints:
|
|
8
|
+
*
|
|
9
|
+
* node fetch-card-child.js <code> [aliasesJson]
|
|
10
|
+
*
|
|
11
|
+
* stdout: { ok: true, card, updatedAt } — fetched
|
|
12
|
+
* { ok: true, missing: true } — not in prod (tombstone it)
|
|
13
|
+
* { ok: false, error } — network/server failure
|
|
14
|
+
*
|
|
15
|
+
* The parent owns all cache writes and backoff bookkeeping; this
|
|
16
|
+
* process only talks to Supabase (read-only) and prints.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { fetchRemoteCard } from './remote.js';
|
|
20
|
+
|
|
21
|
+
const code = process.argv[2];
|
|
22
|
+
let aliases;
|
|
23
|
+
try {
|
|
24
|
+
aliases = process.argv[3] ? JSON.parse(process.argv[3]) : undefined;
|
|
25
|
+
} catch {
|
|
26
|
+
aliases = undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
try {
|
|
30
|
+
const result = await fetchRemoteCard(code, { aliases });
|
|
31
|
+
if (result.missing) {
|
|
32
|
+
process.stdout.write(JSON.stringify({ ok: true, missing: true }));
|
|
33
|
+
} else {
|
|
34
|
+
process.stdout.write(JSON.stringify({ ok: true, card: result.card, updatedAt: result.updatedAt }));
|
|
35
|
+
}
|
|
36
|
+
} catch (err) {
|
|
37
|
+
process.stdout.write(JSON.stringify({ ok: false, error: err?.message || String(err) }));
|
|
38
|
+
}
|