champollion 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +133 -0
  2. package/README.md +387 -0
  3. package/bin/cli.js +278 -0
  4. package/index.js +135 -0
  5. package/lib/api-key.js +127 -0
  6. package/lib/autofix.js +432 -0
  7. package/lib/bridge/method_bridge.py +430 -0
  8. package/lib/card-source-resolution.mjs +284 -0
  9. package/lib/cards/cache.js +169 -0
  10. package/lib/cards/env.js +82 -0
  11. package/lib/cards/fetch-card-child.js +38 -0
  12. package/lib/cards/reader.js +435 -0
  13. package/lib/cards/refresh.js +111 -0
  14. package/lib/cards/remote.js +387 -0
  15. package/lib/cldf-export.mjs +540 -0
  16. package/lib/cldf-terms.mjs +62 -0
  17. package/lib/command-help.js +790 -0
  18. package/lib/commands/audit.js +49 -0
  19. package/lib/commands/card.js +454 -0
  20. package/lib/commands/doctor.js +559 -0
  21. package/lib/commands/fonts.js +489 -0
  22. package/lib/commands/help.js +91 -0
  23. package/lib/commands/init.js +1259 -0
  24. package/lib/commands/integrity.js +148 -0
  25. package/lib/commands/leaderboard.js +478 -0
  26. package/lib/commands/lint.js +30 -0
  27. package/lib/commands/models.js +177 -0
  28. package/lib/commands/plugin.js +103 -0
  29. package/lib/commands/provenance.js +45 -0
  30. package/lib/commands/recommend.js +75 -0
  31. package/lib/commands/register-corpus.js +678 -0
  32. package/lib/commands/repair-script.js +42 -0
  33. package/lib/commands/seal-corpus.js +355 -0
  34. package/lib/commands/seo.js +72 -0
  35. package/lib/commands/serve.js +147 -0
  36. package/lib/commands/status.js +265 -0
  37. package/lib/commands/submit.js +332 -0
  38. package/lib/commands/sync.js +89 -0
  39. package/lib/commands/tm.js +573 -0
  40. package/lib/commands/verify.js +39 -0
  41. package/lib/commands/watch.js +20 -0
  42. package/lib/commands/wrap.js +138 -0
  43. package/lib/commands/xliff.js +327 -0
  44. package/lib/commercial-eligibility.js +235 -0
  45. package/lib/concurrent.js +87 -0
  46. package/lib/config.js +523 -0
  47. package/lib/contamination-lane.js +76 -0
  48. package/lib/content-sync.js +731 -0
  49. package/lib/content.js +733 -0
  50. package/lib/corpus-registration.mjs +608 -0
  51. package/lib/cost-report.js +346 -0
  52. package/lib/diff.js +155 -0
  53. package/lib/docusaurus-sync.js +1256 -0
  54. package/lib/flatten.js +55 -0
  55. package/lib/format.js +954 -0
  56. package/lib/hash.js +159 -0
  57. package/lib/icu.js +473 -0
  58. package/lib/integrity.js +689 -0
  59. package/lib/license-gate.mjs +478 -0
  60. package/lib/license-identify.mjs +229 -0
  61. package/lib/lint.js +629 -0
  62. package/lib/method-manifest.js +60 -0
  63. package/lib/methods/anthropic.js +140 -0
  64. package/lib/methods/apertium.js +163 -0
  65. package/lib/methods/api.js +316 -0
  66. package/lib/methods/base.js +184 -0
  67. package/lib/methods/content-separator.js +45 -0
  68. package/lib/methods/deepl.js +426 -0
  69. package/lib/methods/direct-llm.js +586 -0
  70. package/lib/methods/external.js +332 -0
  71. package/lib/methods/fetch-with-retry.js +124 -0
  72. package/lib/methods/gemini.js +147 -0
  73. package/lib/methods/google-translate.js +402 -0
  74. package/lib/methods/http-utils.js +122 -0
  75. package/lib/methods/libretranslate.js +314 -0
  76. package/lib/methods/llm-coached.js +670 -0
  77. package/lib/methods/llm.js +592 -0
  78. package/lib/methods/local.js +76 -0
  79. package/lib/methods/microsoft-translator.js +331 -0
  80. package/lib/methods/openai.js +131 -0
  81. package/lib/methods/openrouter-client.js +327 -0
  82. package/lib/methods/openrouter-pricing.js +156 -0
  83. package/lib/methods/provider-env.js +115 -0
  84. package/lib/methods/provider-pricing.js +310 -0
  85. package/lib/methods/tilde.js +150 -0
  86. package/lib/methods/translated.js +229 -0
  87. package/lib/methods/translation-error.js +80 -0
  88. package/lib/models.js +258 -0
  89. package/lib/no-translate.js +233 -0
  90. package/lib/output.js +238 -0
  91. package/lib/pairs.js +547 -0
  92. package/lib/plugins.js +447 -0
  93. package/lib/provenance.js +323 -0
  94. package/lib/recommend.js +648 -0
  95. package/lib/registers.js +1185 -0
  96. package/lib/repair-script.js +266 -0
  97. package/lib/scripts.js +994 -0
  98. package/lib/seal.mjs +464 -0
  99. package/lib/sealed-qualifier.mjs +211 -0
  100. package/lib/security.js +59 -0
  101. package/lib/segment.js +369 -0
  102. package/lib/seo.js +275 -0
  103. package/lib/serve.js +854 -0
  104. package/lib/string-classify.js +85 -0
  105. package/lib/submit.mjs +344 -0
  106. package/lib/sync.js +969 -0
  107. package/lib/tags/bcp47.js +202 -0
  108. package/lib/tags/resolve.js +314 -0
  109. package/lib/terminology.js +111 -0
  110. package/lib/tm-seed.js +294 -0
  111. package/lib/tm.js +515 -0
  112. package/lib/translate-pair.js +197 -0
  113. package/lib/translate.js +203 -0
  114. package/lib/types.js +230 -0
  115. package/lib/validate.js +510 -0
  116. package/lib/verify.js +451 -0
  117. package/lib/watch.js +145 -0
  118. package/lib/xliff.js +184 -0
  119. package/package.json +93 -0
  120. package/shared/ATTRIBUTION.md +145 -0
  121. package/shared/CORPORA-CARDS.md +288 -0
  122. package/shared/DATA-SOVEREIGNTY.md +500 -0
  123. package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
  124. package/shared/card-lint-baseline.json +3189 -0
  125. package/shared/cards-fallback.json +1 -0
  126. package/shared/catalogue/card-config.json +6091 -0
  127. package/shared/catalogue/external-results.json +3888 -0
  128. package/shared/catalogue/gender-guidance.json +1038 -0
  129. package/shared/catalogue/method-coverage.json +1751 -0
  130. package/shared/catalogue/metric-coverage.json +170 -0
  131. package/shared/catalogue/metric-reliability.json +1 -0
  132. package/shared/catalogue/register-presets.json +3180 -0
  133. package/shared/catalogue/vitality-scales.json +55 -0
  134. package/shared/cldr-index.json +1115 -0
  135. package/shared/code-bridge.json +253 -0
  136. package/shared/corpora-cards-v1-reference.md +281 -0
  137. package/shared/curated-dictionary-flags.json +35 -0
  138. package/shared/curated-endonyms.json +35 -0
  139. package/shared/curated-fsts.json +51 -0
  140. package/shared/curated-orthography-conventions.json +26 -0
  141. package/shared/curated-sil-resources.json +374 -0
  142. package/shared/curated-tools.json +41 -0
  143. package/shared/docent/corpus.json +11333 -0
  144. package/shared/docent/faq.en.json +564 -0
  145. package/shared/docent/register-blocks.json +60 -0
  146. package/shared/docent/system-prompt.md +144 -0
  147. package/shared/domain-taxonomy.json +35 -0
  148. package/shared/explainers/glossary.json +2975 -0
  149. package/shared/explainers/tc-features.json +20112 -0
  150. package/shared/explainers/term-watchlist.json +147 -0
  151. package/shared/human-services.json +59 -0
  152. package/shared/license-corrections.json +261 -0
  153. package/shared/license-evidence.json +13452 -0
  154. package/shared/licenses.json +6781 -0
  155. package/shared/method-registry.json +236 -0
  156. package/shared/metric-registry.json +620 -0
  157. package/shared/model-aliases.json +7 -0
  158. package/shared/schemas/champollion-plugin.schema.json +206 -0
  159. package/shared/schemas/corpora-card.schema.json +957 -0
  160. package/shared/schemas/domain-taxonomy.schema.json +64 -0
  161. package/shared/schemas/external-results.schema.json +314 -0
  162. package/shared/schemas/human-services.schema.json +90 -0
  163. package/shared/schemas/language-card.schema.json +1308 -0
  164. package/shared/schemas/licenses.schema.json +155 -0
  165. package/shared/schemas/method-card.schema.json +412 -0
  166. package/shared/schemas/method-registry.schema.json +85 -0
  167. package/shared/schemas/metric-registry.schema.json +96 -0
  168. package/shared/schemas/metric-reliability.schema.json +178 -0
  169. package/shared/schemas/model-aliases.schema.json +27 -0
  170. package/shared/schemas/source-snapshot.schema.json +96 -0
@@ -0,0 +1,284 @@
1
+ /**
2
+ * card-source-resolution.mjs — the ONE definition of how a card's
3
+ * `_fieldSources` / `dataSources[]` stamp resolves to a license-register key.
4
+ *
5
+ * WHY THIS IS A SHARED MODULE
6
+ * This logic used to live only inside cli/scripts/lint-language-cards.mjs,
7
+ * unexported. So the license audit (cli/scripts/audit-license-gate.mjs) could
8
+ * not use it, and swept only `facts.source` in the (since-retired) legacy
9
+ * champollion.db facts table — 268 sources.
10
+ * Cards cite 801 distinct stamps, and PHOIBLE has NO facts row at all (it
11
+ * flows straight from phoible-raw.csv into cards), so the sole tone authority
12
+ * on 5,219 card stamps was outside the license sweep entirely.
13
+ *
14
+ * Any second copy of this vocabulary would drift — the alias map alone
15
+ * encodes 30 hand-justified renames. One definition, several consumers.
16
+ *
17
+ * Consumers: lint-language-cards.mjs (license-source-resolves rule),
18
+ * audit-license-gate.mjs --cards.
19
+ *
20
+ * @see docs/LICENSING.md
21
+ * @see shared/licenses.json
22
+ */
23
+
24
+ import fs from 'node:fs';
25
+
26
+ /**
27
+ * Build the resolution index from a license register file.
28
+ * Returns null if absent, or { sourceKeys, baseIndex, spdxSet }.
29
+ */
30
+ export function buildRegisterIndex(filePath) {
31
+ if (!fs.existsSync(filePath)) return null;
32
+ let data;
33
+ try {
34
+ data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
35
+ } catch {
36
+ return null;
37
+ }
38
+ const sources = data.sources || {};
39
+ const sourceKeys = new Set(Object.keys(sources));
40
+
41
+ // Base-name index: register keys with version tokens stripped, so
42
+ // "glottolog-5.3" (card) matches "glottolog-5.0" (register) via "glottolog".
43
+ const baseIndex = new Map();
44
+ for (const key of sourceKeys) {
45
+ const base = stripVersionTokens(key);
46
+ if (!baseIndex.has(base)) baseIndex.set(base, key);
47
+ }
48
+
49
+ const spdxSet = new Set();
50
+ for (const v of Object.values(sources)) {
51
+ if (v && typeof v.license_spdx === 'string' && v.license_spdx.trim()) {
52
+ spdxSet.add(v.license_spdx.trim());
53
+ }
54
+ }
55
+ return { sourceKeys, baseIndex, spdxSet, sources };
56
+ }
57
+
58
+ /**
59
+ * Every distinct source stamp a card asserts, from BOTH conventions:
60
+ * - `_fieldSources` (rich cards), whose values may be a string, an array,
61
+ * or an object of per-subfield stamps; and
62
+ * - `dataSources[]` plus inline `<field>.source` keys, which is how the 751
63
+ * Glottolog-only stubs stamp — they carry no `_fieldSources` map at all.
64
+ *
65
+ * A sweep reading only `_fieldSources` silently skips those 751 cards.
66
+ */
67
+ export function cardSourceStamps(card) {
68
+ const out = new Set();
69
+ const add = (v) => {
70
+ if (typeof v === 'string' && v.trim()) out.add(v.trim());
71
+ else if (Array.isArray(v)) v.forEach(add);
72
+ else if (v && typeof v === 'object') Object.values(v).forEach(add);
73
+ };
74
+
75
+ add(card._fieldSources);
76
+ add(card.dataSources);
77
+
78
+ // Inline `<field>.source`, ONLY on top-level object-valued fields.
79
+ //
80
+ // Deliberately not arrays. `evalDatasets[].source`, `resources.fsts[].source`
81
+ // and `orthographies[].source` all carry a `source` key meaning something
82
+ // else entirely — a publisher name, an FST repo URL, a script name like
83
+ // "Latin". Harvesting those produced 620+ phantom "unlicensed sources" that
84
+ // were never license claims at all. The stub convention this exists for is
85
+ // `coordinates.source` / `vitality.source` on the 751 Glottolog-only cards,
86
+ // which are plain objects.
87
+ for (const [key, value] of Object.entries(card)) {
88
+ if (key.startsWith('_') || !value || typeof value !== 'object') continue;
89
+ if (Array.isArray(value)) continue;
90
+ if (PROSE_SOURCE_FIELDS.has(key)) continue;
91
+ if (typeof value.source === 'string') add(value.source);
92
+ }
93
+ return out;
94
+ }
95
+
96
+ /**
97
+ * Top-level fields whose `.source` is a BIBLIOGRAPHIC CITATION, not a dataset
98
+ * id — so it can never resolve against the license register and must not be
99
+ * reported as an unlicensed source.
100
+ *
101
+ * `culturalAphorism.source` (77 cards) reads e.g. "Traditional Turkish proverb
102
+ * (atasözü). Documented in Aksoy, Ö.A. (1988)…" — an attribution to oral
103
+ * literature with scholarly documentation. That is the card citing its work
104
+ * properly, which is the house rule, not a licensing gap.
105
+ *
106
+ * Kept as a named list rather than a prose-detecting heuristic: a heuristic
107
+ * would eventually swallow a REAL stamp and silently shrink the sweep, which is
108
+ * the failure this audit exists to prevent.
109
+ */
110
+ export const PROSE_SOURCE_FIELDS = new Set(['culturalAphorism']);
111
+
112
+ /**
113
+ * Strip version/date tokens from a source id.
114
+ * Handles every versioning style observed in card data:
115
+ * glottolog-5.3, wals-2024, asjp-v20, grambank-1.0.3, common-voice-20.0,
116
+ * glottolog-5.x-cldf (mid-string), wikidata-P282 (property suffix),
117
+ * unimorph-4.0, northeuralex-0.9, cldr-48
118
+ */
119
+ export function stripVersionTokens(id) {
120
+ return id
121
+ // version token anywhere: -5.3, -2024, -v20, -1.0.3, -5.x, -P282
122
+ .replace(/-(?:v?\d+(?:[._]\d+)*[a-z]?|\d+\.x|P\d+)(?=-|$)/g, '')
123
+ // trailing parenthetical citation: "cldr-endonym (matched via fa)"
124
+ .replace(/\s*\(.*\)\s*$/, '')
125
+ .trim();
126
+ }
127
+
128
+ /**
129
+ * Source ids that denote INTERNAL derivation/curation steps rather than
130
+ * external datasets. These resolve to the register's
131
+ * 'champollion-derived' entry (LicenseRef-Champollion-Derived).
132
+ */
133
+ export const INTERNAL_SOURCE_PATTERNS = [
134
+ /^derived([:\-]|$)/, // derived, derived:script, derived-from-…
135
+ /^champollion-derived([\s:(\[-]|$)/, // champollion-derived [family-level prior; …]
136
+ /^corpora-cards(\s*\(|$)/, // corpora-cards (sync-eval-datasets.mjs) — the
137
+ // repo's own eval registry; per-corpus licenses
138
+ // live on the corpora cards themselves
139
+ /^template-generated([:\-]|$)/, // template-generated:grambank-GB415 (…)
140
+ /^manual-curation(-|$)/, // manual-curation, manual-curation-verified
141
+ /^manual-curation\+\S+$/, // manual-curation+<citation-url> — curated
142
+ // endonym seed stamps (apply-curated-endonyms.mjs);
143
+ // the suffix is a citation, not a source id
144
+ /^cleanup(-resources)?-phase\d+$/,
145
+ /^api-verification(-\d+)?$/,
146
+ /^not-populated$/,
147
+ ];
148
+
149
+ /**
150
+ * Alias map: card-source vocabulary → register vocabulary.
151
+ * Applied AFTER version stripping (keys are base forms). Every entry is
152
+ * justified by an observed mismatch between the enrichment pipeline's
153
+ * source ids and the tc-licenses register keys — do not add speculative
154
+ * aliases.
155
+ */
156
+ export const SOURCE_ALIASES = new Map(Object.entries({
157
+ 'iso639-3': 'sil-iso639-3',
158
+ 'iso639': 'sil-iso639-3', // 'iso639-3-2024' version-strips to 'iso639'
159
+ 'olac-aggregator': 'olac',
160
+ 'huggingface-datasets': 'huggingface',
161
+ 'keyman-api': 'keyman',
162
+ 'kaikki-wiktionary': 'wiktionary',
163
+ 'kaikki-wiktextract': 'wiktionary',
164
+ 'opus-nlpl': 'opus',
165
+ 'opus-nlpl-api': 'opus-api',
166
+ 'ucla-phonetics': 'uclaphoneticslabarchive',
167
+ 'huntergatherer-db': 'huntergatherer',
168
+ 'huntergatherer-lexibank': 'huntergatherer',
169
+ 'nllb': 'meta-nllb', // nllb-200 → meta-nllb-200 (base forms)
170
+ 'cldr': 'unicode-cldr',
171
+ 'wikimedia-sitematrix': 'wikipedia-sitematrix',
172
+ 'siteinfo': 'wikipedia-sitematrix', // "wikimedia-sitematrix+siteinfo" components
173
+ 'numeralbank-channumerals': 'numeralbank',
174
+ 'datsemshift-zalizniak': 'datsemshift',
175
+ 'acd-blust': 'acd',
176
+ 'valpal-hartmann': 'valpal',
177
+ 'bowern-pny': 'bowernpny',
178
+ 'vanuatu-voices': 'vanuatuvoices',
179
+ 'glottolog-aes': 'glottolog',
180
+ 'glottolog-languoid': 'glottolog', // glottolog-5.x-languoid
181
+ 'sagart-sino-tibetan': 'sagartst', // same Sagart et al. Sino-Tibetan dataset
182
+ 'ids-cldf': 'ids', // CLDF edition of IDS (same dataset/license)
183
+ 'paradisec-olac': 'paradisec', // PARADISEC catalog metadata harvested via OLAC
184
+ 'rosetta-project-ia': 'rosetta-project', // same collection, Internet Archive hosting
185
+ 'language-atlas-pacific': 'wurm-hattori-pacific-atlas', // same atlas, two card vocabularies
186
+ 'lexibank-batch': 'lexibank', // batch enrichment over the lexibank umbrella
187
+ 'cldr-endonym': 'unicode-cldr', // CLDR endonym data ("cldr-endonym (matched via fa)")
188
+ 'wikidata-rdfs-label-own': 'wikidata', // own-language rdfs:label tier of
189
+ // harvest-wikidata-endonyms.mjs (same CC0 dataset)
190
+ }));
191
+
192
+ /**
193
+ * Resolve ONE source id component (no '+') against the register.
194
+ * Returns the register key it resolves to, or null.
195
+ */
196
+ export function resolveSourceComponent(id, register) {
197
+ const { sourceKeys, baseIndex } = register;
198
+
199
+ // 1. Exact match
200
+ if (sourceKeys.has(id)) return id;
201
+
202
+ // 2. Internal derivation pseudo-sources → champollion-derived
203
+ for (const pattern of INTERNAL_SOURCE_PATTERNS) {
204
+ if (pattern.test(id)) return 'champollion-derived';
205
+ }
206
+
207
+ // 3. Version-stripped base match (both sides base-normalized)
208
+ let base = stripVersionTokens(id);
209
+ if (sourceKeys.has(base)) return base;
210
+ if (baseIndex.has(base)) return baseIndex.get(base);
211
+
212
+ // 4. Alias rename
213
+ if (SOURCE_ALIASES.has(base)) {
214
+ const target = SOURCE_ALIASES.get(base);
215
+ if (sourceKeys.has(target)) return target;
216
+ if (baseIndex.has(target)) return baseIndex.get(target);
217
+ }
218
+
219
+ // 5. lexibank-<dataset> prefix / <dataset>-lexibank suffix: register
220
+ // keys are bare dataset names (e.g. 'dyenindoeuropean', 'abvd',
221
+ // 'uralex', 'crossandean'); cards tag them with the lexibank family.
222
+ if (base.startsWith('lexibank-')) {
223
+ const bare = base.slice('lexibank-'.length);
224
+ if (sourceKeys.has(bare)) return bare;
225
+ if (baseIndex.has(bare)) return baseIndex.get(bare);
226
+ }
227
+ if (base.endsWith('-lexibank')) {
228
+ const bare = base.slice(0, -'-lexibank'.length);
229
+ if (sourceKeys.has(bare)) return bare;
230
+ if (baseIndex.has(bare)) return baseIndex.get(bare);
231
+ }
232
+
233
+ // 6. FEATURE-ID citation: "<dataset>-<featureId>", e.g. grambank-GB415,
234
+ // wals-44A, wals-13A. The card is citing a specific typological feature,
235
+ // and the LICENSED thing is the dataset — Grambank's license governs
236
+ // GB415 exactly as it governs every other feature. Without this, 2,006
237
+ // card-citations of Grambank and WALS features read as unlicensed
238
+ // sources, which would have buried the real findings under noise.
239
+ // Deliberately narrow: the suffix must be a feature-code shape (letters
240
+ // then digits, or digits then a letter), never an arbitrary word.
241
+ const featureM = /^(.+)-(?:[A-Z]{2}\d+|\d+[A-Z])$/.exec(base);
242
+ if (featureM) {
243
+ const dataset = featureM[1];
244
+ if (sourceKeys.has(dataset)) return dataset;
245
+ if (baseIndex.has(dataset)) return baseIndex.get(dataset);
246
+ }
247
+
248
+ return null;
249
+ }
250
+
251
+ /**
252
+ * Resolve a full source value, which may be a compound "a+b+c" string.
253
+ * Returns { ok: boolean, unresolved: string[], malformed: boolean }.
254
+ */
255
+ export function resolveSourceValue(value, register) {
256
+ // Stringified-array data bug: "['a', 'b']+c" — produced by an upstream
257
+ // enrichment step that joined a Python list repr instead of its items.
258
+ // Unparseable as source ids; flag the whole value as malformed. A bare
259
+ // '[' is NOT malformed by itself — internal stamps legitimately carry
260
+ // bracketed citations ("champollion-derived [family-level prior; …]"),
261
+ // so only the quote-bearing list-repr shape is flagged (2026-07-07).
262
+ if (value.includes("['") || value.includes("'")) {
263
+ return { ok: false, unresolved: [value], malformed: true };
264
+ }
265
+
266
+ // Whole-value internal derivations like "derived:cldr+script+keyboard"
267
+ // describe a derivation recipe, not a '+'-joined source list — the
268
+ // components after the first are recipe inputs ("script", "keyboard"),
269
+ // not source ids. Match internal patterns against the full value first.
270
+ for (const pattern of INTERNAL_SOURCE_PATTERNS) {
271
+ if (pattern.test(value)) return { ok: true, unresolved: [], malformed: false };
272
+ }
273
+
274
+ // Parenthetical citations may legitimately contain spaces; only split
275
+ // on '+' that separates components.
276
+ const components = value.split('+').map(c => c.trim()).filter(Boolean);
277
+ const unresolved = [];
278
+ for (const component of components) {
279
+ if (!resolveSourceComponent(component, register)) {
280
+ unresolved.push(component);
281
+ }
282
+ }
283
+ return { ok: unresolved.length === 0, unresolved, malformed: false };
284
+ }
@@ -0,0 +1,169 @@
1
+ /**
2
+ * Per-user card cache — ~/.champollion/cards/
3
+ *
4
+ * Layout:
5
+ * <code>.json — { v, code, updatedAt, fetchedAt, card }
6
+ * _state.json — { lastRefreshCheckAt, backoffUntil, missing: { code: ts } }
7
+ *
8
+ * updatedAt mirrors the per-language updated_at from the Supabase
9
+ * trading-card tables; invalidation works by deleting entries whose
10
+ * remote updated_at moved past the cached one (lib/cards/refresh.js)
11
+ * and refetching lazily on next use.
12
+ *
13
+ * `missing` tombstones record codes prod definitively doesn't have, so
14
+ * unknown locale codes don't trigger a network attempt on every CLI
15
+ * run. Tombstones expire with MISSING_TTL_MS and are cleared by the
16
+ * refresh pass when a language appears upstream.
17
+ *
18
+ * All writes are atomic (tmp + rename) and all reads tolerate missing
19
+ * or corrupt files — a broken cache must never break the CLI.
20
+ */
21
+
22
+ import fs from 'node:fs';
23
+ import path from 'node:path';
24
+ import { CACHE_DIR, isValidCardCode } from './env.js';
25
+
26
+ const ENTRY_VERSION = 1;
27
+ const STATE_FILE = '_state.json';
28
+
29
+ /** Failed-fetch backoff — don't hammer the network on every invocation. */
30
+ export const BACKOFF_MS = 15 * 60 * 1000;
31
+ /** Missing-language tombstone lifetime. */
32
+ export const MISSING_TTL_MS = 24 * 60 * 60 * 1000;
33
+
34
+ function entryPath(code) {
35
+ return path.join(CACHE_DIR, `${code}.json`);
36
+ }
37
+
38
+ function readJsonSafe(file) {
39
+ try {
40
+ return JSON.parse(fs.readFileSync(file, 'utf-8'));
41
+ } catch {
42
+ return null;
43
+ }
44
+ }
45
+
46
+ function writeJsonAtomic(file, value) {
47
+ fs.mkdirSync(path.dirname(file), { recursive: true });
48
+ const tmp = `${file}.${process.pid}.tmp`;
49
+ fs.writeFileSync(tmp, JSON.stringify(value), 'utf-8');
50
+ fs.renameSync(tmp, file);
51
+ }
52
+
53
+ /** Read a cached card entry. Returns { card, updatedAt } or null. */
54
+ export function readCachedCard(code) {
55
+ if (!isValidCardCode(code)) return null;
56
+ const entry = readJsonSafe(entryPath(code));
57
+ if (!entry || entry.v !== ENTRY_VERSION || entry.code !== code || !entry.card) return null;
58
+ return { card: entry.card, updatedAt: entry.updatedAt || null };
59
+ }
60
+
61
+ export function writeCachedCard(code, card, updatedAt) {
62
+ if (!isValidCardCode(code)) return false;
63
+ try {
64
+ writeJsonAtomic(entryPath(code), {
65
+ v: ENTRY_VERSION,
66
+ code,
67
+ updatedAt: updatedAt || null,
68
+ fetchedAt: new Date().toISOString(),
69
+ card,
70
+ });
71
+ return true;
72
+ } catch {
73
+ return false; // read-only HOME etc. — cache is best-effort
74
+ }
75
+ }
76
+
77
+ export function removeCachedCard(code) {
78
+ if (!isValidCardCode(code)) return;
79
+ try {
80
+ fs.unlinkSync(entryPath(code));
81
+ } catch {
82
+ // already gone
83
+ }
84
+ }
85
+
86
+ /** Codes currently materialized in the cache (excludes state/tmp files). */
87
+ export function listCachedCodes() {
88
+ let entries;
89
+ try {
90
+ entries = fs.readdirSync(CACHE_DIR);
91
+ } catch {
92
+ return [];
93
+ }
94
+ const codes = [];
95
+ for (const name of entries) {
96
+ if (!name.endsWith('.json') || name === STATE_FILE) continue;
97
+ const code = name.slice(0, -5);
98
+ if (isValidCardCode(code)) codes.push(code);
99
+ }
100
+ return codes;
101
+ }
102
+
103
+ // -----------------------------------------------------------------
104
+ // Shared state — refresh stamp, network backoff, missing tombstones
105
+ // -----------------------------------------------------------------
106
+
107
+ export function readState() {
108
+ const state = readJsonSafe(path.join(CACHE_DIR, STATE_FILE));
109
+ return state && typeof state === 'object'
110
+ ? state
111
+ : { lastRefreshCheckAt: null, backoffUntil: null, missing: {} };
112
+ }
113
+
114
+ export function writeState(patch) {
115
+ try {
116
+ const next = { ...readState(), ...patch };
117
+ writeJsonAtomic(path.join(CACHE_DIR, STATE_FILE), next);
118
+ return next;
119
+ } catch {
120
+ return null;
121
+ }
122
+ }
123
+
124
+ /** True while a previous network failure says "don't try again yet". */
125
+ export function inBackoff(state = readState()) {
126
+ return Boolean(state.backoffUntil && Date.parse(state.backoffUntil) > Date.now());
127
+ }
128
+
129
+ export function noteNetworkFailure() {
130
+ writeState({ backoffUntil: new Date(Date.now() + BACKOFF_MS).toISOString() });
131
+ }
132
+
133
+ export function clearBackoff() {
134
+ writeState({ backoffUntil: null });
135
+ }
136
+
137
+ /** True when `code` was recently confirmed absent from prod. */
138
+ export function isTombstoned(code, state = readState()) {
139
+ const ts = state.missing?.[code];
140
+ return Boolean(ts && Date.now() - Date.parse(ts) < MISSING_TTL_MS);
141
+ }
142
+
143
+ export function tombstone(code) {
144
+ if (!isValidCardCode(code)) return;
145
+ const state = readState();
146
+ const missing = { ...(state.missing || {}) };
147
+ missing[code] = new Date().toISOString();
148
+ // Don't let the tombstone map grow without bound.
149
+ const keys = Object.keys(missing);
150
+ if (keys.length > 500) {
151
+ keys.sort((a, b) => Date.parse(missing[a]) - Date.parse(missing[b]));
152
+ for (const key of keys.slice(0, keys.length - 500)) delete missing[key];
153
+ }
154
+ writeState({ missing });
155
+ }
156
+
157
+ export function clearTombstones(codes) {
158
+ const state = readState();
159
+ if (!state.missing) return;
160
+ const missing = { ...state.missing };
161
+ let changed = false;
162
+ for (const code of codes) {
163
+ if (code in missing) {
164
+ delete missing[code];
165
+ changed = true;
166
+ }
167
+ }
168
+ if (changed) writeState({ missing });
169
+ }
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Card-loading environment — paths, endpoints, and mode detection.
3
+ *
4
+ * Deliberately tiny with zero heavy imports: bin/cli.js consults this
5
+ * module on every invocation to decide whether the background cache
6
+ * refresh applies, without paying the cost of loading the full card
7
+ * registry (lib/registers.js eager-loads thousands of files in repo
8
+ * checkouts).
9
+ *
10
+ * All values are read once at import. Override via environment for
11
+ * tests and staging:
12
+ * CHAMPOLLION_CARDS_DIR — full card directory (repo mode)
13
+ * CHAMPOLLION_CARDS_FALLBACK — bundled fallback JSON (packaged mode)
14
+ * CHAMPOLLION_CARDS_CACHE_DIR — per-user card cache
15
+ * CHAMPOLLION_SUPABASE_URL — REST endpoint base
16
+ * CHAMPOLLION_SUPABASE_ANON_KEY— read-only publishable key
17
+ * CHAMPOLLION_OFFLINE=1 — never touch the network
18
+ * CHAMPOLLION_CARDS_TTL_HOURS — staleness-check interval (default 24)
19
+ */
20
+
21
+ import fs from 'node:fs';
22
+ import os from 'node:os';
23
+ import path from 'node:path';
24
+
25
+ /**
26
+ * Production Supabase project (MT Eval Arena). The publishable anon key
27
+ * is intentionally committed — Row Level Security restricts the anon
28
+ * role to SELECT on the public trading-card tables (see
29
+ * mt-eval-arena/supabase/migrations/012_create_trading_cards.sql).
30
+ * The CLI only ever reads.
31
+ */
32
+ export const SUPABASE_URL =
33
+ process.env.CHAMPOLLION_SUPABASE_URL || 'https://sjdomynysdljkbemupqa.supabase.co';
34
+ export const SUPABASE_ANON_KEY =
35
+ process.env.CHAMPOLLION_SUPABASE_ANON_KEY || 'sb_publishable_bV6CFNFnzxhQI0wlBx2J0A_5Vm5gFBp';
36
+
37
+ /** Full card directory — present in repo checkouts, excluded from the npm tarball. */
38
+ export const CARDS_DIR =
39
+ process.env.CHAMPOLLION_CARDS_DIR ||
40
+ path.join(import.meta.dirname, '..', '..', 'shared', 'language-cards');
41
+
42
+ /** Bundled fallback (core cards + parents + manifest) — ships in the package. */
43
+ export const FALLBACK_FILE =
44
+ process.env.CHAMPOLLION_CARDS_FALLBACK ||
45
+ path.join(import.meta.dirname, '..', '..', 'shared', 'cards-fallback.json');
46
+
47
+ /** Per-user cache of cards fetched from Supabase. */
48
+ export const CACHE_DIR =
49
+ process.env.CHAMPOLLION_CARDS_CACHE_DIR ||
50
+ path.join(os.homedir(), '.champollion', 'cards');
51
+
52
+ export const OFFLINE = process.env.CHAMPOLLION_OFFLINE === '1';
53
+
54
+ /** How often (hours) the CLI checks Supabase for per-language updated_at bumps. */
55
+ export const REFRESH_TTL_HOURS = Number(process.env.CHAMPOLLION_CARDS_TTL_HOURS) > 0
56
+ ? Number(process.env.CHAMPOLLION_CARDS_TTL_HOURS)
57
+ : 24;
58
+
59
+ /**
60
+ * Repo mode = the full card directory exists on disk (git checkout,
61
+ * monorepo dev). Packaged mode = npm install, where only the fallback
62
+ * bundle ships and the long tail is fetched + cached on demand.
63
+ */
64
+ export function hasLocalCardsDir() {
65
+ try {
66
+ return fs.existsSync(CARDS_DIR);
67
+ } catch {
68
+ return false;
69
+ }
70
+ }
71
+
72
+ /**
73
+ * Locale codes flow from user config into cache file paths and REST
74
+ * query params, so gate them hard before any fs/network use.
75
+ * Accepts ISO-ish tags: letters/digits with - or _ separators
76
+ * ('fra', 'fra-CA', 'x-pirate', 'cmn-Hant'), max 5 segments.
77
+ */
78
+ const CODE_RE = /^[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+){0,4}$/;
79
+
80
+ export function isValidCardCode(code) {
81
+ return typeof code === 'string' && code.length >= 2 && code.length <= 48 && CODE_RE.test(code);
82
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * One-shot card fetcher — child-process entry point.
3
+ *
4
+ * The public registry API (getLanguageCard et al.) is synchronous, but
5
+ * Node has no synchronous HTTP. When the packaged CLI misses on a code
6
+ * that the manifest says exists, lib/registers.js runs this script via
7
+ * execFileSync and parses the single JSON object it prints:
8
+ *
9
+ * node fetch-card-child.js <code> [aliasesJson]
10
+ *
11
+ * stdout: { ok: true, card, updatedAt } — fetched
12
+ * { ok: true, missing: true } — not in prod (tombstone it)
13
+ * { ok: false, error } — network/server failure
14
+ *
15
+ * The parent owns all cache writes and backoff bookkeeping; this
16
+ * process only talks to Supabase (read-only) and prints.
17
+ */
18
+
19
+ import { fetchRemoteCard } from './remote.js';
20
+
21
+ const code = process.argv[2];
22
+ let aliases;
23
+ try {
24
+ aliases = process.argv[3] ? JSON.parse(process.argv[3]) : undefined;
25
+ } catch {
26
+ aliases = undefined;
27
+ }
28
+
29
+ try {
30
+ const result = await fetchRemoteCard(code, { aliases });
31
+ if (result.missing) {
32
+ process.stdout.write(JSON.stringify({ ok: true, missing: true }));
33
+ } else {
34
+ process.stdout.write(JSON.stringify({ ok: true, card: result.card, updatedAt: result.updatedAt }));
35
+ }
36
+ } catch (err) {
37
+ process.stdout.write(JSON.stringify({ ok: false, error: err?.message || String(err) }));
38
+ }