champollion 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +133 -0
- package/README.md +387 -0
- package/bin/cli.js +278 -0
- package/index.js +135 -0
- package/lib/api-key.js +127 -0
- package/lib/autofix.js +432 -0
- package/lib/bridge/method_bridge.py +430 -0
- package/lib/card-source-resolution.mjs +284 -0
- package/lib/cards/cache.js +169 -0
- package/lib/cards/env.js +82 -0
- package/lib/cards/fetch-card-child.js +38 -0
- package/lib/cards/reader.js +435 -0
- package/lib/cards/refresh.js +111 -0
- package/lib/cards/remote.js +387 -0
- package/lib/cldf-export.mjs +540 -0
- package/lib/cldf-terms.mjs +62 -0
- package/lib/command-help.js +790 -0
- package/lib/commands/audit.js +49 -0
- package/lib/commands/card.js +454 -0
- package/lib/commands/doctor.js +559 -0
- package/lib/commands/fonts.js +489 -0
- package/lib/commands/help.js +91 -0
- package/lib/commands/init.js +1259 -0
- package/lib/commands/integrity.js +148 -0
- package/lib/commands/leaderboard.js +478 -0
- package/lib/commands/lint.js +30 -0
- package/lib/commands/models.js +177 -0
- package/lib/commands/plugin.js +103 -0
- package/lib/commands/provenance.js +45 -0
- package/lib/commands/recommend.js +75 -0
- package/lib/commands/register-corpus.js +678 -0
- package/lib/commands/repair-script.js +42 -0
- package/lib/commands/seal-corpus.js +355 -0
- package/lib/commands/seo.js +72 -0
- package/lib/commands/serve.js +147 -0
- package/lib/commands/status.js +265 -0
- package/lib/commands/submit.js +332 -0
- package/lib/commands/sync.js +89 -0
- package/lib/commands/tm.js +573 -0
- package/lib/commands/verify.js +39 -0
- package/lib/commands/watch.js +20 -0
- package/lib/commands/wrap.js +138 -0
- package/lib/commands/xliff.js +327 -0
- package/lib/commercial-eligibility.js +235 -0
- package/lib/concurrent.js +87 -0
- package/lib/config.js +523 -0
- package/lib/contamination-lane.js +76 -0
- package/lib/content-sync.js +731 -0
- package/lib/content.js +733 -0
- package/lib/corpus-registration.mjs +608 -0
- package/lib/cost-report.js +346 -0
- package/lib/diff.js +155 -0
- package/lib/docusaurus-sync.js +1256 -0
- package/lib/flatten.js +55 -0
- package/lib/format.js +954 -0
- package/lib/hash.js +159 -0
- package/lib/icu.js +473 -0
- package/lib/integrity.js +689 -0
- package/lib/license-gate.mjs +478 -0
- package/lib/license-identify.mjs +229 -0
- package/lib/lint.js +629 -0
- package/lib/method-manifest.js +60 -0
- package/lib/methods/anthropic.js +140 -0
- package/lib/methods/apertium.js +163 -0
- package/lib/methods/api.js +316 -0
- package/lib/methods/base.js +184 -0
- package/lib/methods/content-separator.js +45 -0
- package/lib/methods/deepl.js +426 -0
- package/lib/methods/direct-llm.js +586 -0
- package/lib/methods/external.js +332 -0
- package/lib/methods/fetch-with-retry.js +124 -0
- package/lib/methods/gemini.js +147 -0
- package/lib/methods/google-translate.js +402 -0
- package/lib/methods/http-utils.js +122 -0
- package/lib/methods/libretranslate.js +314 -0
- package/lib/methods/llm-coached.js +670 -0
- package/lib/methods/llm.js +592 -0
- package/lib/methods/local.js +76 -0
- package/lib/methods/microsoft-translator.js +331 -0
- package/lib/methods/openai.js +131 -0
- package/lib/methods/openrouter-client.js +327 -0
- package/lib/methods/openrouter-pricing.js +156 -0
- package/lib/methods/provider-env.js +115 -0
- package/lib/methods/provider-pricing.js +310 -0
- package/lib/methods/tilde.js +150 -0
- package/lib/methods/translated.js +229 -0
- package/lib/methods/translation-error.js +80 -0
- package/lib/models.js +258 -0
- package/lib/no-translate.js +233 -0
- package/lib/output.js +238 -0
- package/lib/pairs.js +547 -0
- package/lib/plugins.js +447 -0
- package/lib/provenance.js +323 -0
- package/lib/recommend.js +648 -0
- package/lib/registers.js +1185 -0
- package/lib/repair-script.js +266 -0
- package/lib/scripts.js +994 -0
- package/lib/seal.mjs +464 -0
- package/lib/sealed-qualifier.mjs +211 -0
- package/lib/security.js +59 -0
- package/lib/segment.js +369 -0
- package/lib/seo.js +275 -0
- package/lib/serve.js +854 -0
- package/lib/string-classify.js +85 -0
- package/lib/submit.mjs +344 -0
- package/lib/sync.js +969 -0
- package/lib/tags/bcp47.js +202 -0
- package/lib/tags/resolve.js +314 -0
- package/lib/terminology.js +111 -0
- package/lib/tm-seed.js +294 -0
- package/lib/tm.js +515 -0
- package/lib/translate-pair.js +197 -0
- package/lib/translate.js +203 -0
- package/lib/types.js +230 -0
- package/lib/validate.js +510 -0
- package/lib/verify.js +451 -0
- package/lib/watch.js +145 -0
- package/lib/xliff.js +184 -0
- package/package.json +93 -0
- package/shared/ATTRIBUTION.md +145 -0
- package/shared/CORPORA-CARDS.md +288 -0
- package/shared/DATA-SOVEREIGNTY.md +500 -0
- package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
- package/shared/card-lint-baseline.json +3189 -0
- package/shared/cards-fallback.json +1 -0
- package/shared/catalogue/card-config.json +6091 -0
- package/shared/catalogue/external-results.json +3888 -0
- package/shared/catalogue/gender-guidance.json +1038 -0
- package/shared/catalogue/method-coverage.json +1751 -0
- package/shared/catalogue/metric-coverage.json +170 -0
- package/shared/catalogue/metric-reliability.json +1 -0
- package/shared/catalogue/register-presets.json +3180 -0
- package/shared/catalogue/vitality-scales.json +55 -0
- package/shared/cldr-index.json +1115 -0
- package/shared/code-bridge.json +253 -0
- package/shared/corpora-cards-v1-reference.md +281 -0
- package/shared/curated-dictionary-flags.json +35 -0
- package/shared/curated-endonyms.json +35 -0
- package/shared/curated-fsts.json +51 -0
- package/shared/curated-orthography-conventions.json +26 -0
- package/shared/curated-sil-resources.json +374 -0
- package/shared/curated-tools.json +41 -0
- package/shared/docent/corpus.json +11333 -0
- package/shared/docent/faq.en.json +564 -0
- package/shared/docent/register-blocks.json +60 -0
- package/shared/docent/system-prompt.md +144 -0
- package/shared/domain-taxonomy.json +35 -0
- package/shared/explainers/glossary.json +2975 -0
- package/shared/explainers/tc-features.json +20112 -0
- package/shared/explainers/term-watchlist.json +147 -0
- package/shared/human-services.json +59 -0
- package/shared/license-corrections.json +261 -0
- package/shared/license-evidence.json +13452 -0
- package/shared/licenses.json +6781 -0
- package/shared/method-registry.json +236 -0
- package/shared/metric-registry.json +620 -0
- package/shared/model-aliases.json +7 -0
- package/shared/schemas/champollion-plugin.schema.json +206 -0
- package/shared/schemas/corpora-card.schema.json +957 -0
- package/shared/schemas/domain-taxonomy.schema.json +64 -0
- package/shared/schemas/external-results.schema.json +314 -0
- package/shared/schemas/human-services.schema.json +90 -0
- package/shared/schemas/language-card.schema.json +1308 -0
- package/shared/schemas/licenses.schema.json +155 -0
- package/shared/schemas/method-card.schema.json +412 -0
- package/shared/schemas/method-registry.schema.json +85 -0
- package/shared/schemas/metric-registry.schema.json +96 -0
- package/shared/schemas/metric-reliability.schema.json +178 -0
- package/shared/schemas/model-aliases.schema.json +27 -0
- package/shared/schemas/source-snapshot.schema.json +96 -0
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLDF StructureDataset export — pure builders (no I/O).
|
|
3
|
+
*
|
|
4
|
+
* These functions turn rows from the Champollion facts DB (the SSOT) into the
|
|
5
|
+
* tables of a valid CLDF StructureDataset: languages.csv, parameters.csv,
|
|
6
|
+
* values.csv, codes.csv, sources.bib and the StructureDataset-metadata.json
|
|
7
|
+
* descriptor. They perform no file or database I/O — `cli/scripts/export-cldf.mjs`
|
|
8
|
+
* reads the DB + cards and calls these — so every builder is trivially unit
|
|
9
|
+
* testable (see cli/test/cldf-export.test.js).
|
|
10
|
+
*
|
|
11
|
+
* Round-trip framing (docs/DATA-ARCHITECTURE.md §4):
|
|
12
|
+
* download-cldf-datasets.mjs → ingest-cldf.mjs → [facts DB] → export-cldf.mjs (this)
|
|
13
|
+
*
|
|
14
|
+
* Design notes
|
|
15
|
+
* ────────────
|
|
16
|
+
* • IDENTIFIERS ARE COLLISION-SAFE. CLDF requires every table ID to be unique
|
|
17
|
+
* and to match `[a-zA-Z0-9_\-]+`, and every *_ID reference to resolve. We
|
|
18
|
+
* therefore assign IDs up front (assignLanguageIds / assignParameterIds /
|
|
19
|
+
* assignSourceKeys) and pass the resulting maps into the row builders, so the
|
|
20
|
+
* LanguageTable/ParameterTable/CodeTable and the ValueTable references they
|
|
21
|
+
* point at are guaranteed consistent. This is why the builder signatures take
|
|
22
|
+
* id-maps rather than re-deriving ids per table.
|
|
23
|
+
* • PROVENANCE IS PRESERVED, NEVER REWRITTEN. A fact whose source is
|
|
24
|
+
* 'champollion-derived' is emitted with Source = the champollion-derived bib
|
|
25
|
+
* key and keeps its `[derived from <upstream>]` note in the Comment column.
|
|
26
|
+
* It is never re-attributed to the upstream's name. Mirrors the doctrine in
|
|
27
|
+
* cli/scripts/audit-fact-provenance.mjs.
|
|
28
|
+
* • EXPORT FROM THE FACTS DB, NOT FROM CARD JSON. Card JSON is lossy/ambiguous
|
|
29
|
+
* for typed values; the DB carries value_type + per-fact provenance. Cards
|
|
30
|
+
* are used ONLY for language-level metadata (name, glottocode, coordinates).
|
|
31
|
+
*
|
|
32
|
+
* @module cldf-export
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { CLDF_TYPES, CLDF_DATASET_TYPES, CLDF_PROPERTIES } from './cldf-terms.mjs';
|
|
36
|
+
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// SLUG + COLLISION-SAFE ID ASSIGNMENT
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Slugify an arbitrary string to `[a-z0-9_]` (CLDF/BibTeX-safe). Strips
|
|
43
|
+
* diacritics, lowercases, collapses runs of non-alphanumerics to '_'. Returns
|
|
44
|
+
* '' for input that contains no Latin alphanumerics (callers supply a fallback).
|
|
45
|
+
*/
|
|
46
|
+
export function slug(s) {
|
|
47
|
+
return String(s == null ? '' : s)
|
|
48
|
+
.normalize('NFKD').replace(/\p{M}/gu, '') // strip combining marks (diacritics)
|
|
49
|
+
.toLowerCase()
|
|
50
|
+
.replace(/[^a-z0-9]+/g, '_')
|
|
51
|
+
.replace(/^_+|_+$/g, '')
|
|
52
|
+
.replace(/_+/g, '_');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Stable composite key for a (domain, property) parameter. */
|
|
56
|
+
export function paramKey(domain, property) {
|
|
57
|
+
return JSON.stringify([domain, property]);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function uniquify(base, used) {
|
|
61
|
+
let id = base, i = 1;
|
|
62
|
+
while (used.has(id)) id = `${base}-${++i}`;
|
|
63
|
+
used.add(id);
|
|
64
|
+
return id;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Assign a unique CLDF Language ID per language code. Prefers the Glottocode
|
|
69
|
+
* (so crk → "plai1258"); falls back to the ISO 639-3 code when the glottocode
|
|
70
|
+
* is missing or shared by more than one code (ISO codes are globally unique).
|
|
71
|
+
*
|
|
72
|
+
* @param {Array<{code:string, glottocode?:string}>} records
|
|
73
|
+
* @returns {Map<string,string>} language_code → Language_ID
|
|
74
|
+
*/
|
|
75
|
+
export function assignLanguageIds(records) {
|
|
76
|
+
const glottoCount = new Map();
|
|
77
|
+
for (const r of records) {
|
|
78
|
+
const g = (r.glottocode || '').trim();
|
|
79
|
+
if (g) glottoCount.set(g, (glottoCount.get(g) || 0) + 1);
|
|
80
|
+
}
|
|
81
|
+
const map = new Map();
|
|
82
|
+
const used = new Set();
|
|
83
|
+
// Pass 1: glottocode when it is present AND unique within the export set.
|
|
84
|
+
for (const r of records) {
|
|
85
|
+
const g = (r.glottocode || '').trim();
|
|
86
|
+
if (g && glottoCount.get(g) === 1 && /^[a-zA-Z0-9_-]+$/.test(g)) {
|
|
87
|
+
map.set(r.code, g);
|
|
88
|
+
used.add(g);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// Pass 2: everyone else gets a unique id derived from their (unique) ISO code.
|
|
92
|
+
for (const r of records) {
|
|
93
|
+
if (map.has(r.code)) continue;
|
|
94
|
+
const base = slug(r.code) || slug(r.glottocode) || 'lang';
|
|
95
|
+
map.set(r.code, uniquify(base, used));
|
|
96
|
+
}
|
|
97
|
+
return map;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Assign a unique CLDF Parameter ID per (domain, property) pair.
|
|
102
|
+
* @param {Array<{domain:string, property:string}>} params
|
|
103
|
+
* @returns {Map<string,string>} paramKey(domain,property) → Parameter_ID
|
|
104
|
+
*/
|
|
105
|
+
export function assignParameterIds(params) {
|
|
106
|
+
const map = new Map();
|
|
107
|
+
const used = new Set();
|
|
108
|
+
for (const p of params) {
|
|
109
|
+
const key = paramKey(p.domain, p.property);
|
|
110
|
+
if (map.has(key)) continue;
|
|
111
|
+
let base = `${slug(p.domain)}__${slug(p.property)}`;
|
|
112
|
+
if (base === '__' || base === '') base = 'param';
|
|
113
|
+
map.set(key, uniquify(base, used));
|
|
114
|
+
}
|
|
115
|
+
return map;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Assign a unique BibTeX/source key per source label.
|
|
120
|
+
* @param {Array<string|{source:string}>} sources
|
|
121
|
+
* @returns {Map<string,string>} source label → bib key
|
|
122
|
+
*/
|
|
123
|
+
export function assignSourceKeys(sources) {
|
|
124
|
+
const map = new Map();
|
|
125
|
+
const used = new Set();
|
|
126
|
+
for (const s0 of sources) {
|
|
127
|
+
const source = typeof s0 === 'string' ? s0 : s0.source;
|
|
128
|
+
if (map.has(source)) continue;
|
|
129
|
+
const base = slug(source) || 'source';
|
|
130
|
+
map.set(source, uniquify(base, used));
|
|
131
|
+
}
|
|
132
|
+
return map;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Deterministic CLDF Code ID for a (parameter, raw-code) pair. */
|
|
136
|
+
export function codeId(paramId, rawCode) {
|
|
137
|
+
return `${paramId}-${slug(rawCode) || 'x'}`;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* True only for a bare ISO 639-3 individual-language code shape (three lowercase
|
|
142
|
+
* letters). Champollion's `code` namespace is wider than ISO 639-3 — it also
|
|
143
|
+
* holds hub cards (`family-*`, `genus-*`, `macrolanguage-*`), BCP-47 regional
|
|
144
|
+
* variants (`fra-CA`, `cmn-Hant`) and conlangs (`x-yoda`). Those are real
|
|
145
|
+
* entities with facts, but they are NOT ISO 639-3 codes, so the CLDF
|
|
146
|
+
* `ISO639P3code` column must be left blank for them (their `code` still drives
|
|
147
|
+
* the Language_ID). This keeps `cldf validate` warning-free.
|
|
148
|
+
*/
|
|
149
|
+
export function isIso639_3(code) {
|
|
150
|
+
return /^[a-z]{3}$/.test(code || '');
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
// NORMALIZERS + NOTE PARSING
|
|
155
|
+
// ---------------------------------------------------------------------------
|
|
156
|
+
|
|
157
|
+
function num(v) {
|
|
158
|
+
if (v == null || v === '') return null;
|
|
159
|
+
const n = Number(v);
|
|
160
|
+
return Number.isNaN(n) ? null : n;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Normalize a language card OR a `languages`-table row into one shape.
|
|
165
|
+
* Cards carry coordinates as { coordinates: { lat, lng } }; DB rows carry flat
|
|
166
|
+
* `lat`/`lng`. Either is accepted.
|
|
167
|
+
*/
|
|
168
|
+
export function normalizeLanguageRecord(src) {
|
|
169
|
+
let lat = null, lng = null;
|
|
170
|
+
if (src.coordinates && typeof src.coordinates === 'object') {
|
|
171
|
+
lat = num(src.coordinates.lat);
|
|
172
|
+
lng = num(src.coordinates.lng);
|
|
173
|
+
}
|
|
174
|
+
if (lat == null) lat = num(src.lat);
|
|
175
|
+
if (lng == null) lng = num(src.lng);
|
|
176
|
+
return {
|
|
177
|
+
code: src.code,
|
|
178
|
+
name: src.name || src.code,
|
|
179
|
+
glottocode: src.glottocode || '',
|
|
180
|
+
macroarea: src.macroarea || '',
|
|
181
|
+
lat,
|
|
182
|
+
lng,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Extract the raw categorical code from a fact's `notes`. The ingester writes
|
|
188
|
+
* `code:<Code_ID>` (e.g. "code:1A-1"), sometimes combined as "comment | code:X".
|
|
189
|
+
* Returns the code string, or null.
|
|
190
|
+
*/
|
|
191
|
+
export function parseCodeFromNotes(notes) {
|
|
192
|
+
if (!notes) return null;
|
|
193
|
+
const m = String(notes).match(/code:\s*([^|\s]+)/);
|
|
194
|
+
return m ? m[1].trim() : null;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// ---------------------------------------------------------------------------
|
|
198
|
+
// TABLE COLUMN SPECS — single source of truth for CSV headers + csvw columns
|
|
199
|
+
// ---------------------------------------------------------------------------
|
|
200
|
+
|
|
201
|
+
const ID_DATATYPE = { base: 'string', format: '[a-zA-Z0-9_\\-]+' };
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Each table's columns, in order. `prop` (a CLDF term URI) marks a standard
|
|
205
|
+
* CLDF column; columns without `prop` are Champollion extension columns.
|
|
206
|
+
* `foreignKeys` are [localColumn, targetResource, targetColumn] triples.
|
|
207
|
+
*/
|
|
208
|
+
export const TABLES = [
|
|
209
|
+
{
|
|
210
|
+
name: 'values',
|
|
211
|
+
url: 'values.csv',
|
|
212
|
+
conformsTo: CLDF_TYPES.VALUE_TABLE,
|
|
213
|
+
columns: [
|
|
214
|
+
{ name: 'ID', prop: CLDF_PROPERTIES.ID, datatype: ID_DATATYPE, required: true },
|
|
215
|
+
{ name: 'Language_ID', prop: CLDF_PROPERTIES.LANGUAGE_REF, datatype: 'string', required: true },
|
|
216
|
+
{ name: 'Parameter_ID', prop: CLDF_PROPERTIES.PARAMETER_REF, datatype: 'string', required: true },
|
|
217
|
+
{ name: 'Value', prop: CLDF_PROPERTIES.VALUE, datatype: 'string', null: [''] },
|
|
218
|
+
{ name: 'Code_ID', prop: CLDF_PROPERTIES.CODE_REF, datatype: 'string', null: [''] },
|
|
219
|
+
{ name: 'Source', prop: CLDF_PROPERTIES.SOURCE, datatype: 'string', separator: ';' },
|
|
220
|
+
{ name: 'Source_URL', datatype: 'string' },
|
|
221
|
+
{ name: 'Confidence', datatype: 'string' },
|
|
222
|
+
{ name: 'Value_Type', datatype: 'string' },
|
|
223
|
+
{ name: 'Comment', prop: CLDF_PROPERTIES.COMMENT, datatype: 'string' },
|
|
224
|
+
],
|
|
225
|
+
primaryKey: ['ID'],
|
|
226
|
+
foreignKeys: [
|
|
227
|
+
['Language_ID', 'languages.csv', 'ID'],
|
|
228
|
+
['Parameter_ID', 'parameters.csv', 'ID'],
|
|
229
|
+
['Code_ID', 'codes.csv', 'ID'],
|
|
230
|
+
],
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
name: 'languages',
|
|
234
|
+
url: 'languages.csv',
|
|
235
|
+
conformsTo: CLDF_TYPES.LANGUAGE_TABLE,
|
|
236
|
+
columns: [
|
|
237
|
+
{ name: 'ID', prop: CLDF_PROPERTIES.ID, datatype: ID_DATATYPE, required: true },
|
|
238
|
+
{ name: 'Name', prop: CLDF_PROPERTIES.NAME, datatype: 'string' },
|
|
239
|
+
{ name: 'ISO639P3code', prop: CLDF_PROPERTIES.ISO639P3, datatype: 'string' },
|
|
240
|
+
{ name: 'Glottocode', prop: CLDF_PROPERTIES.GLOTTOCODE, datatype: 'string' },
|
|
241
|
+
{ name: 'Macroarea', prop: CLDF_PROPERTIES.MACROAREA, datatype: 'string' },
|
|
242
|
+
{ name: 'Latitude', prop: CLDF_PROPERTIES.LATITUDE, datatype: { base: 'decimal', minimum: -90, maximum: 90 }, null: [''] },
|
|
243
|
+
{ name: 'Longitude', prop: CLDF_PROPERTIES.LONGITUDE, datatype: { base: 'decimal', minimum: -180, maximum: 180 }, null: [''] },
|
|
244
|
+
],
|
|
245
|
+
primaryKey: ['ID'],
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
name: 'parameters',
|
|
249
|
+
url: 'parameters.csv',
|
|
250
|
+
conformsTo: CLDF_TYPES.PARAMETER_TABLE,
|
|
251
|
+
columns: [
|
|
252
|
+
{ name: 'ID', prop: CLDF_PROPERTIES.ID, datatype: ID_DATATYPE, required: true },
|
|
253
|
+
{ name: 'Name', prop: CLDF_PROPERTIES.NAME, datatype: 'string' },
|
|
254
|
+
{ name: 'Description', prop: CLDF_PROPERTIES.DESCRIPTION, datatype: 'string' },
|
|
255
|
+
{ name: 'Value_Type', datatype: 'string' },
|
|
256
|
+
],
|
|
257
|
+
primaryKey: ['ID'],
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
name: 'codes',
|
|
261
|
+
url: 'codes.csv',
|
|
262
|
+
conformsTo: CLDF_TYPES.CODE_TABLE,
|
|
263
|
+
columns: [
|
|
264
|
+
{ name: 'ID', prop: CLDF_PROPERTIES.ID, datatype: ID_DATATYPE, required: true },
|
|
265
|
+
{ name: 'Parameter_ID', prop: CLDF_PROPERTIES.PARAMETER_REF, datatype: 'string', required: true },
|
|
266
|
+
{ name: 'Name', prop: CLDF_PROPERTIES.NAME, datatype: 'string' },
|
|
267
|
+
{ name: 'Description', prop: CLDF_PROPERTIES.DESCRIPTION, datatype: 'string' },
|
|
268
|
+
],
|
|
269
|
+
primaryKey: ['ID'],
|
|
270
|
+
foreignKeys: [
|
|
271
|
+
['Parameter_ID', 'parameters.csv', 'ID'],
|
|
272
|
+
],
|
|
273
|
+
},
|
|
274
|
+
];
|
|
275
|
+
|
|
276
|
+
/** Ordered column names for a table (CSV header order). */
|
|
277
|
+
export function tableColumnNames(tableName) {
|
|
278
|
+
const t = TABLES.find((x) => x.name === tableName);
|
|
279
|
+
if (!t) throw new Error(`unknown CLDF table: ${tableName}`);
|
|
280
|
+
return t.columns.map((c) => c.name);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// ---------------------------------------------------------------------------
|
|
284
|
+
// ROW BUILDERS
|
|
285
|
+
// ---------------------------------------------------------------------------
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* @param {Array<object>} records language cards and/or `languages` rows
|
|
289
|
+
* @param {Map<string,string>} langIdByCode from assignLanguageIds
|
|
290
|
+
* @returns {Array<object>} languages.csv rows
|
|
291
|
+
*/
|
|
292
|
+
export function buildLanguageTable(records, langIdByCode) {
|
|
293
|
+
const rows = [];
|
|
294
|
+
const seen = new Set();
|
|
295
|
+
for (const src of records) {
|
|
296
|
+
const r = normalizeLanguageRecord(src);
|
|
297
|
+
const id = langIdByCode.get(r.code);
|
|
298
|
+
if (!id || seen.has(id)) continue;
|
|
299
|
+
seen.add(id);
|
|
300
|
+
rows.push({
|
|
301
|
+
ID: id,
|
|
302
|
+
Name: r.name,
|
|
303
|
+
ISO639P3code: isIso639_3(r.code) ? r.code : '',
|
|
304
|
+
Glottocode: r.glottocode,
|
|
305
|
+
Macroarea: r.macroarea,
|
|
306
|
+
Latitude: r.lat == null ? '' : String(r.lat),
|
|
307
|
+
Longitude: r.lng == null ? '' : String(r.lng),
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
return rows;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* @param {Array<{domain:string, property:string, value_type?:string}>} params distinct parameters
|
|
315
|
+
* @param {Map<string,string>} paramIdMap from assignParameterIds
|
|
316
|
+
* @param {Map<string,string>} [descByProperty] optional property → description (P2)
|
|
317
|
+
* @returns {Array<object>} parameters.csv rows
|
|
318
|
+
*/
|
|
319
|
+
export function buildParameterTable(params, paramIdMap, descByProperty = new Map()) {
|
|
320
|
+
const rows = [];
|
|
321
|
+
const seen = new Set();
|
|
322
|
+
for (const p of params) {
|
|
323
|
+
const id = paramIdMap.get(paramKey(p.domain, p.property));
|
|
324
|
+
if (!id || seen.has(id)) continue;
|
|
325
|
+
seen.add(id);
|
|
326
|
+
rows.push({
|
|
327
|
+
ID: id,
|
|
328
|
+
Name: p.property,
|
|
329
|
+
Description: descByProperty.get(p.property) || '',
|
|
330
|
+
Value_Type: p.value_type || p.valueType || '',
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
return rows;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* The heart of the export: one ValueTable row per fact, with resolvable
|
|
338
|
+
* Language_ID / Parameter_ID / Code_ID / Source references. Facts whose
|
|
339
|
+
* language or parameter cannot be resolved are skipped (the CLI logs the count
|
|
340
|
+
* — fail-loud, never silent on aggregate). Source provenance is preserved
|
|
341
|
+
* verbatim; derived facts keep their `[derived from …]` note in Comment.
|
|
342
|
+
*
|
|
343
|
+
* @param {Array<object>} factRows facts-table rows
|
|
344
|
+
* @param {{langIdByCode:Map, paramIdMap:Map, sourceKeyMap:Map}} maps
|
|
345
|
+
* @returns {Array<object>} values.csv rows
|
|
346
|
+
*/
|
|
347
|
+
export function buildValueTable(factRows, { langIdByCode, paramIdMap, sourceKeyMap }) {
|
|
348
|
+
const rows = [];
|
|
349
|
+
const counter = new Map(); // `${langId}|${paramId}` → running n
|
|
350
|
+
for (const f of factRows) {
|
|
351
|
+
const langId = langIdByCode.get(f.language_code);
|
|
352
|
+
const paramId = paramIdMap.get(paramKey(f.domain, f.property));
|
|
353
|
+
if (!langId || !paramId) continue; // unresolvable reference → skip
|
|
354
|
+
const rawCode = parseCodeFromNotes(f.notes);
|
|
355
|
+
const ck = `${langId}|${paramId}`;
|
|
356
|
+
const n = (counter.get(ck) || 0) + 1;
|
|
357
|
+
counter.set(ck, n);
|
|
358
|
+
rows.push({
|
|
359
|
+
ID: `${langId}-${paramId}-${n}`,
|
|
360
|
+
Language_ID: langId,
|
|
361
|
+
Parameter_ID: paramId,
|
|
362
|
+
Value: f.value == null ? '' : String(f.value),
|
|
363
|
+
Code_ID: rawCode ? codeId(paramId, rawCode) : '',
|
|
364
|
+
Source: sourceKeyMap.get(f.source) || '',
|
|
365
|
+
Source_URL: f.source_url || '',
|
|
366
|
+
Confidence: f.confidence || '',
|
|
367
|
+
Value_Type: f.value_type || '',
|
|
368
|
+
Comment: f.notes || '',
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
return rows;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* CodeTable (P2): one row per distinct (parameter, raw-code). The human-readable
|
|
376
|
+
* label is the fact `value` (the ingester stores labels, not bare codes). IDs
|
|
377
|
+
* match codeId() so every ValueTable.Code_ID resolves.
|
|
378
|
+
*
|
|
379
|
+
* @param {Array<{domain:string, property:string, value:any, notes?:string, code?:string}>} codeRows
|
|
380
|
+
* @param {Map<string,string>} paramIdMap
|
|
381
|
+
* @returns {Array<object>} codes.csv rows
|
|
382
|
+
*/
|
|
383
|
+
export function buildCodeTable(codeRows, paramIdMap) {
|
|
384
|
+
const rows = [];
|
|
385
|
+
const seen = new Set();
|
|
386
|
+
for (const c of codeRows) {
|
|
387
|
+
const paramId = paramIdMap.get(paramKey(c.domain, c.property));
|
|
388
|
+
const rawCode = c.code != null ? c.code : parseCodeFromNotes(c.notes);
|
|
389
|
+
if (!paramId || !rawCode) continue;
|
|
390
|
+
const id = codeId(paramId, rawCode);
|
|
391
|
+
if (seen.has(id)) continue;
|
|
392
|
+
seen.add(id);
|
|
393
|
+
rows.push({
|
|
394
|
+
ID: id,
|
|
395
|
+
Parameter_ID: paramId,
|
|
396
|
+
Name: c.value == null ? '' : String(c.value),
|
|
397
|
+
Description: '',
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
return rows;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// ---------------------------------------------------------------------------
|
|
404
|
+
// sources.bib
|
|
405
|
+
// ---------------------------------------------------------------------------
|
|
406
|
+
|
|
407
|
+
function bibValue(v) {
|
|
408
|
+
// BibTeX field values: drop braces (we wrap in {}), collapse newlines.
|
|
409
|
+
return String(v == null ? '' : v).replace(/[{}]/g, '').replace(/\s*\n\s*/g, ' ').trim();
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function bibEntry(key, source, url, license) {
|
|
413
|
+
const isDerived = source === 'champollion-derived';
|
|
414
|
+
const fields = [];
|
|
415
|
+
fields.push(` title = {${bibValue(isDerived
|
|
416
|
+
? 'Champollion-derived values (computed; see per-value "[derived from <upstream>]" notes)'
|
|
417
|
+
: source)}}`);
|
|
418
|
+
fields.push(` author = {${bibValue(isDerived ? 'Champollion' : source)}}`);
|
|
419
|
+
if (url) fields.push(` url = {${bibValue(url)}}`);
|
|
420
|
+
|
|
421
|
+
const note = [];
|
|
422
|
+
if (isDerived) note.push('Champollion derivation — not an assertion by any upstream dataset');
|
|
423
|
+
if (license) {
|
|
424
|
+
if (license.license_spdx) note.push(`License: ${license.license_spdx}`);
|
|
425
|
+
else if (license.license_url) note.push(`License: ${license.license_url}`);
|
|
426
|
+
if (license.attribution) note.push(`Attribution: ${license.attribution}`);
|
|
427
|
+
if (license.non_commercial_only) note.push('Non-commercial use only');
|
|
428
|
+
if (license.requires_sharealike) note.push('ShareAlike required');
|
|
429
|
+
}
|
|
430
|
+
if (note.length) fields.push(` note = {${bibValue(note.join('. '))}}`);
|
|
431
|
+
|
|
432
|
+
return `@misc{${key},\n${fields.join(',\n')}\n}`;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Build a deduplicated BibTeX sources file: one @misc entry per distinct source.
|
|
437
|
+
*
|
|
438
|
+
* @param {Array<string|{source:string, source_url?:string}>} sources
|
|
439
|
+
* @param {Map<string,string>} sourceKeyMap from assignSourceKeys
|
|
440
|
+
* @param {Map<string,object>} [licenses] source → source_licenses row (P2)
|
|
441
|
+
* @returns {string} BibTeX text
|
|
442
|
+
*/
|
|
443
|
+
export function buildSourcesBib(sources, sourceKeyMap, licenses = new Map()) {
|
|
444
|
+
const seen = new Set();
|
|
445
|
+
const entries = [];
|
|
446
|
+
for (const s0 of sources) {
|
|
447
|
+
const source = typeof s0 === 'string' ? s0 : s0.source;
|
|
448
|
+
const key = sourceKeyMap.get(source);
|
|
449
|
+
if (!key || seen.has(key)) continue;
|
|
450
|
+
seen.add(key);
|
|
451
|
+
const url = typeof s0 === 'object' ? (s0.source_url || '') : '';
|
|
452
|
+
entries.push(bibEntry(key, source, url, licenses.get(source)));
|
|
453
|
+
}
|
|
454
|
+
return entries.join('\n\n') + (entries.length ? '\n' : '');
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// ---------------------------------------------------------------------------
|
|
458
|
+
// metadata descriptor
|
|
459
|
+
// ---------------------------------------------------------------------------
|
|
460
|
+
|
|
461
|
+
function toCsvwColumn(c) {
|
|
462
|
+
const out = { name: c.name };
|
|
463
|
+
if (c.datatype) out.datatype = c.datatype;
|
|
464
|
+
if (c.prop) out.propertyUrl = c.prop;
|
|
465
|
+
if (c.separator) out.separator = c.separator;
|
|
466
|
+
if (c.null) out.null = c.null;
|
|
467
|
+
if (c.required) out.required = true;
|
|
468
|
+
return out;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Build the StructureDataset-metadata.json descriptor (a plain JS object).
|
|
473
|
+
* Mirrors the canonical CLDF metadata shape (see any StructureDataset-metadata
|
|
474
|
+
* .json under cli/data/) so `cldf validate` recognises the module and checks
|
|
475
|
+
* referential integrity.
|
|
476
|
+
*/
|
|
477
|
+
export function buildMetadata({
|
|
478
|
+
id = 'champollion',
|
|
479
|
+
title = 'Champollion Language Facts (CLDF StructureDataset)',
|
|
480
|
+
license = 'CC-BY-4.0',
|
|
481
|
+
accessURL = '',
|
|
482
|
+
citation = 'Champollion — open translation infrastructure for low-resource languages. Per-source licenses and citations in sources.bib.',
|
|
483
|
+
} = {}) {
|
|
484
|
+
return {
|
|
485
|
+
'@context': ['http://www.w3.org/ns/csvw', { '@language': 'en' }],
|
|
486
|
+
'dc:conformsTo': CLDF_DATASET_TYPES.STRUCTURE,
|
|
487
|
+
'dc:source': 'sources.bib',
|
|
488
|
+
'dc:title': title,
|
|
489
|
+
'dc:license': license,
|
|
490
|
+
'dc:bibliographicCitation': citation,
|
|
491
|
+
...(accessURL ? { 'dcat:accessURL': accessURL } : {}),
|
|
492
|
+
'rdf:ID': id,
|
|
493
|
+
'rdf:type': 'http://www.w3.org/ns/dcat#Distribution',
|
|
494
|
+
tables: TABLES.map((t) => ({
|
|
495
|
+
url: t.url,
|
|
496
|
+
'dc:conformsTo': t.conformsTo,
|
|
497
|
+
tableSchema: {
|
|
498
|
+
columns: t.columns.map(toCsvwColumn),
|
|
499
|
+
...(t.foreignKeys
|
|
500
|
+
? {
|
|
501
|
+
foreignKeys: t.foreignKeys.map(([col, resource, refcol]) => ({
|
|
502
|
+
columnReference: [col],
|
|
503
|
+
reference: { resource, columnReference: [refcol] },
|
|
504
|
+
})),
|
|
505
|
+
}
|
|
506
|
+
: {}),
|
|
507
|
+
primaryKey: t.primaryKey,
|
|
508
|
+
},
|
|
509
|
+
})),
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// ---------------------------------------------------------------------------
|
|
514
|
+
// CSV serialization (RFC-4180 minimal quoter — no new dependency)
|
|
515
|
+
// ---------------------------------------------------------------------------
|
|
516
|
+
|
|
517
|
+
/** Quote a single CSV field if it contains a comma, quote, or newline. */
|
|
518
|
+
export function csvField(v) {
|
|
519
|
+
const s = v == null ? '' : String(v);
|
|
520
|
+
return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/** Serialize one row (array of values) to a CSV line (no trailing newline). */
|
|
524
|
+
export function csvLine(values) {
|
|
525
|
+
return values.map(csvField).join(',');
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* Serialize a whole table (header + rows) to a CSV string. Convenient for the
|
|
530
|
+
* small tables and for tests; the CLI streams large tables with csvLine().
|
|
531
|
+
*
|
|
532
|
+
* @param {string} tableName one of TABLES[].name
|
|
533
|
+
* @param {Array<object>} rows row objects keyed by CLDF column name
|
|
534
|
+
*/
|
|
535
|
+
export function serializeCsv(tableName, rows) {
|
|
536
|
+
const names = tableColumnNames(tableName);
|
|
537
|
+
const lines = [csvLine(names)];
|
|
538
|
+
for (const row of rows) lines.push(csvLine(names.map((n) => row[n])));
|
|
539
|
+
return lines.join('\n') + '\n';
|
|
540
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLDF term URIs — the single source of truth for Cross-Linguistic Data Format
|
|
3
|
+
* identifiers used across Champollion's CLDF pipeline.
|
|
4
|
+
*
|
|
5
|
+
* These constants name the standard CLDF table types, dataset types, and column
|
|
6
|
+
* property URIs (http://cldf.clld.org/v1.0/terms.rdf#…). They were originally
|
|
7
|
+
* declared inline in `cli/scripts/ingest-cldf.mjs`; they now live here so that
|
|
8
|
+
* BOTH the ingester (CLDF → facts DB) and the exporter (facts DB → CLDF) read
|
|
9
|
+
* the exact same vocabulary. Round-trip framing, see docs/DATA-ARCHITECTURE.md §4:
|
|
10
|
+
*
|
|
11
|
+
* download-cldf-datasets.mjs → ingest-cldf.mjs → [facts DB] → export-cldf.mjs
|
|
12
|
+
*
|
|
13
|
+
* The ingester imports CLDF_TYPES / CLDF_DATASET_TYPES / CLDF_PROPERTIES and uses
|
|
14
|
+
* a subset of CLDF_PROPERTIES. The exporter uses the full set (including the
|
|
15
|
+
* latitude/longitude/description/codeReference terms it needs to write a valid
|
|
16
|
+
* StructureDataset). Adding terms here is therefore behaviour-preserving for the
|
|
17
|
+
* ingester — it simply ignores the keys it does not reference.
|
|
18
|
+
*
|
|
19
|
+
* @module cldf-terms
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
// CLDF dc:conformsTo URIs that identify table types
|
|
23
|
+
export const CLDF_TYPES = {
|
|
24
|
+
FORM_TABLE: 'http://cldf.clld.org/v1.0/terms.rdf#FormTable',
|
|
25
|
+
VALUE_TABLE: 'http://cldf.clld.org/v1.0/terms.rdf#ValueTable',
|
|
26
|
+
LANGUAGE_TABLE: 'http://cldf.clld.org/v1.0/terms.rdf#LanguageTable',
|
|
27
|
+
PARAMETER_TABLE: 'http://cldf.clld.org/v1.0/terms.rdf#ParameterTable',
|
|
28
|
+
CODE_TABLE: 'http://cldf.clld.org/v1.0/terms.rdf#CodeTable',
|
|
29
|
+
COGNATE_TABLE: 'http://cldf.clld.org/v1.0/terms.rdf#CognateTable',
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export const CLDF_DATASET_TYPES = {
|
|
33
|
+
WORDLIST: 'http://cldf.clld.org/v1.0/terms.rdf#Wordlist',
|
|
34
|
+
STRUCTURE: 'http://cldf.clld.org/v1.0/terms.rdf#StructureDataset',
|
|
35
|
+
DICTIONARY: 'http://cldf.clld.org/v1.0/terms.rdf#Dictionary',
|
|
36
|
+
PARALLEL_TEXT: 'http://cldf.clld.org/v1.0/terms.rdf#ParallelText',
|
|
37
|
+
GENERIC: 'http://cldf.clld.org/v1.0/terms.rdf#Generic',
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
// CLDF property URIs that identify standard columns.
|
|
41
|
+
// The first block is the set the ingester references; the second block adds the
|
|
42
|
+
// extra terms the exporter writes (latitude/longitude/description/codeReference).
|
|
43
|
+
export const CLDF_PROPERTIES = {
|
|
44
|
+
// --- referenced by ingest-cldf.mjs (do not remove) ---
|
|
45
|
+
ID: 'http://cldf.clld.org/v1.0/terms.rdf#id',
|
|
46
|
+
NAME: 'http://cldf.clld.org/v1.0/terms.rdf#name',
|
|
47
|
+
LANGUAGE_REF: 'http://cldf.clld.org/v1.0/terms.rdf#languageReference',
|
|
48
|
+
PARAMETER_REF: 'http://cldf.clld.org/v1.0/terms.rdf#parameterReference',
|
|
49
|
+
VALUE: 'http://cldf.clld.org/v1.0/terms.rdf#value',
|
|
50
|
+
FORM: 'http://cldf.clld.org/v1.0/terms.rdf#form',
|
|
51
|
+
GLOTTOCODE: 'http://cldf.clld.org/v1.0/terms.rdf#glottocode',
|
|
52
|
+
ISO639P3: 'http://cldf.clld.org/v1.0/terms.rdf#iso639P3code',
|
|
53
|
+
MACROAREA: 'http://cldf.clld.org/v1.0/terms.rdf#macroarea',
|
|
54
|
+
COMMENT: 'http://cldf.clld.org/v1.0/terms.rdf#comment',
|
|
55
|
+
SOURCE: 'http://cldf.clld.org/v1.0/terms.rdf#source',
|
|
56
|
+
|
|
57
|
+
// --- additionally used by export-cldf.mjs ---
|
|
58
|
+
CODE_REF: 'http://cldf.clld.org/v1.0/terms.rdf#codeReference',
|
|
59
|
+
DESCRIPTION: 'http://cldf.clld.org/v1.0/terms.rdf#description',
|
|
60
|
+
LATITUDE: 'http://cldf.clld.org/v1.0/terms.rdf#latitude',
|
|
61
|
+
LONGITUDE: 'http://cldf.clld.org/v1.0/terms.rdf#longitude',
|
|
62
|
+
};
|