@glossarist/concept-browser 0.7.134 → 0.7.136

package/scripts/generate-data.ts

@@ -1,7 +1,19 @@
 import fs from 'fs';
 import path from 'path';
 import yaml from 'js-yaml';
-import { naturalSort, Register, parseMention, parseMentions, InvalidMentionError } from 'glossarist';
+import {
+  naturalSort,
+  Register,
+  parseMention,
+  parseMentions,
+  InvalidMentionError,
+  parseConceptYaml,
+  conceptToTurtle,
+  conceptsToTbx,
+  conceptsToCsv,
+  emitOutputSet,
+  type Concept,
+} from 'glossarist';
 import { loadSiteConfig } from './load-site-config';
 import { getGroups } from './lib/concept-groups';
 import { consumeDatasetEntities } from './lib/build/non-verbal-consumer';
@@ -23,7 +35,7 @@ import type {
   YamlNewsFrontmatter,
   YamlContentPage,
 } from './lib/yaml-types';
-import { buildConceptUri } from './lib/concept-uri';
+import { buildConceptUri, buildConceptUriPrefix } from './lib/concept-uri';
 import {
   loadConceptFile,
   readYaml,
@@ -41,13 +53,6 @@ import {
   refsToJsonLd,
   citationToJsonLd,
 } from './lib/jsonld-emitter';
-import {
-  escapeTurtle,
-  escapeXml,
-  conceptJsonToTurtle,
-  conceptJsonToSkosJsonLd,
-  conceptJsonToTbx,
-} from './lib/concept-formats';
 
 // MECE partitive multiplicity: 2 independent axes (ISO 704:2022).
 //   presence × count
@@ -689,9 +694,10 @@ async function processDataset(dir, register, opts) {
     for (const f of fs.readdirSync(conceptsDir)) fs.unlinkSync(path.join(conceptsDir, f));
   }
   const concepts = [];
+  const conceptInstances: Concept[] = [];
   const langTermCounts = {};
   const langDefCounts = {};
-  const availableFormats = ['ttl', 'jsonld', 'yaml', 'tbx'];
+  const availableFormats = ['ttl', 'jsonld', 'yaml', 'tbx', 'csv'];
 
   // Pre-scan: build designation → concept ID lookup for same-dataset designation refs
   const designationLookup = new Map();
@@ -802,16 +808,34 @@ async function processDataset(dir, register, opts) {
       const termid = String(conceptYaml.termid);
       const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
       writeJson(path.join(conceptsDir, `${termid}.json`), jsonld);
-
-      const ttlContent = conceptJsonToTurtle(jsonld);
-      fs.writeFileSync(path.join(conceptsDir, `${termid}.ttl`), ttlContent);
-
-      const skosJsonLd = conceptJsonToSkosJsonLd(jsonld);
-      fs.writeFileSync(path.join(conceptsDir, `${termid}.jsonld`), skosJsonLd);
-
-      const tbxContent = conceptJsonToTbx(jsonld);
-      if (tbxContent) {
-        fs.writeFileSync(path.join(conceptsDir, `${termid}.tbx`), tbxContent);
+      // Canonical interchange exports — routed through glossarist's
+      // output API (one model walk; predicates generated from the
+      // concept-model context, wire shape from schemas/v3). The hand-rolled
+      // gl:-shaped emitters in scripts/lib/concept-formats.ts are retired.
+      // Note: TBX now includes <term> entries — the retired emitter
+      // computed termEntries but never wrote them into the document.
+      let conceptInstance: Concept | null = null;
+      try {
+        conceptInstance = parseConceptYaml(fs.readFileSync(path.join(dir, file), 'utf8'), termid);
+      } catch (e) {
+        console.warn(`  ${termid}: canonical parse failed (${e.message}); skipping glossarist exports`);
+      }
+      if (conceptInstance) {
+        conceptInstances.push(conceptInstance);
+        fs.writeFileSync(
+          path.join(conceptsDir, `${termid}.ttl`),
+          conceptToTurtle(conceptInstance, { uriBase: refMaps.uriBase, registerId: register }),
+        );
+        const conceptJsonLd = await emitOutputSet([conceptInstance], {
+          registerId: register,
+          uriBase: refMaps.uriBase,
+          formats: ['jsonld'],
+        });
+        fs.writeFileSync(path.join(conceptsDir, `${termid}.jsonld`), conceptJsonLd.jsonld!);
+        const tbxContent = conceptsToTbx([conceptInstance], { registerId: register });
+        if (tbxContent) {
+          fs.writeFileSync(path.join(conceptsDir, `${termid}.tbx`), tbxContent);
+        }
       }
 
       fs.copyFileSync(path.join(dir, file), path.join(conceptsDir, `${termid}.yaml`));
@@ -831,6 +855,16 @@ async function processDataset(dir, register, opts) {
     }
   }
 
+  // Aggregate distributions: single-file exports of the whole register,
+  // emitted by glossarist's output-set API from the canonical model walk.
+  const aggregate = await emitOutputSet(conceptInstances, {
+    registerId: register,
+    uriBase: refMaps.uriBase,
+    languageOrder: opts.languageOrder || opts.languages,
+  });
+  fs.writeFileSync(path.join(DATA, register, `${register}.csv`), aggregate.csv!);
+  fs.writeFileSync(path.join(DATA, register, `${register}.jsonld`), aggregate.jsonld!);
+
   const CHUNK_SIZE = 500;
   const chunks = [];
   for (let i = 0; i < concepts.length; i += CHUNK_SIZE) {
@@ -877,7 +911,7 @@ async function processDataset(dir, register, opts) {
   fs.writeFileSync(
     path.join(DATA, register, 'graph-nodes.json'),
     JSON.stringify({
-      uriPrefix: `${refMaps.uriBase}/${register}/concept/`,
+      uriPrefix: buildConceptUriPrefix(refMaps.uriBase, register),
       registerId: register,
       nodes: graphNodeEntries,
     }),
@@ -908,29 +942,37 @@ async function processDataset(dir, register, opts) {
     };
   }
 
-  // Copy bulk format files from compiled/ directory (full GCR)
+  // Aggregate bulk exports offered on the dataset page: single-file
+  // distributions generated by this build plus any files shipped in the
+  // dataset's compiled/ directory. site-config `bulk_exports` restricts
+  // which formats are offered; absent/empty config offers all available.
+  const BULK_FORMAT_BY_EXT = {
+    '.ttl': 'turtle',
+    '.jsonld': 'jsonld',
+    '.xml': 'tbx',
+    '.jsonl': 'jsonl',
+    '.yaml': 'yaml',
+    '.csv': 'csv',
+  };
+  const bulkFormats = [];
+  const addBulkFormat = file => {
+    const dest = path.join(DATA, register, file);
+    if (!fs.existsSync(dest) || bulkFormats.some(f => f.file === file)) return;
+    const formatName = BULK_FORMAT_BY_EXT[path.extname(file)] || path.extname(file).slice(1);
+    bulkFormats.push({ file, format: formatName, size: fs.statSync(dest).size });
+  };
+  for (const file of [`${register}.csv`, `${register}.jsonld`]) addBulkFormat(file);
+
   const sourceRoot = path.dirname(dir);
   const compiledDir = path.join(sourceRoot, 'compiled');
-  const bulkFormats = [];
   if (fs.existsSync(compiledDir)) {
+    let copied = 0;
     for (const file of fs.readdirSync(compiledDir)) {
-      const src = path.join(compiledDir, file);
-      const dest = path.join(DATA, register, file);
-      fs.copyFileSync(src, dest);
-      const ext = path.extname(file);
-      const formatMap = {
-        '.ttl': 'turtle',
-        '.jsonld': 'jsonld',
-        '.xml': 'tbx',
-        '.jsonl': 'jsonl',
-        '.yaml': 'yaml',
-      };
-      const formatName = formatMap[ext] || ext.slice(1);
-      bulkFormats.push({ file, format: formatName, size: fs.statSync(src).size });
-    }
-    if (bulkFormats.length) {
-      console.log(`  Copied ${bulkFormats.length} bulk format files`);
+      fs.copyFileSync(path.join(compiledDir, file), path.join(DATA, register, file));
+      addBulkFormat(file);
+      copied += 1;
     }
+    if (copied) console.log(`  Copied ${copied} bulk format files from compiled/`);
   }
 
   const sourceStats = Array.from(stats.sourceMap.values())
@@ -977,7 +1019,6 @@ async function processDataset(dir, register, opts) {
     year: opts.year ?? undefined,
     languageStats: langStats,
     availableFormats,
-    bulkFormats,
     hasBibliography: opts.hasBibliography,
     hasImages: opts.hasImages,
   };
@@ -987,10 +1028,16 @@ async function processDataset(dir, register, opts) {
   if (opts.status) manifest.editionStatus = opts.status;
   if (opts.ordering) manifest.ordering = opts.ordering;
   if (opts.sections && opts.sections.length > 0) manifest.sections = opts.sections;
-  writeJson(path.join(DATA, register, 'manifest.json'), manifest);
 
   // Dataset-level RDF (WS J2/J5): dcat:Dataset + skos:ConceptScheme + skos:Collection per section.
+  // Runs before the manifest write so the ttl distribution is on disk when
+  // bulkFormats is assembled.
   await writeDatasetRdf(register, manifest, concepts, refMaps, opts);
+  addBulkFormat(`${register}.ttl`);
+  manifest.bulkFormats = opts.bulkExports?.length
+    ? bulkFormats.filter(f => opts.bulkExports.includes(f.format))
+    : bulkFormats;
+  writeJson(path.join(DATA, register, 'manifest.json'), manifest);
 
   // Copy bibliography.yaml → bibliography.json
   // ALWAYS emit bibliography.json — empty stub `{ bibliography: [] }` when no
@@ -1058,6 +1105,11 @@ for (const ds of config.datasets) {
 
 const refMaps = buildRefMaps(config, registerCache);
 
+// site-config `bulk_exports`: which aggregate formats the dataset pages
+// offer for download. 'ttl' is accepted as an alias of the canonical
+// format id 'turtle'. Absent/empty → offer all available formats.
+const bulkExports = (config.bulkExports || []).map(f => (f === 'ttl' ? 'turtle' : f));
+
 for (let i = 0; i < config.datasets.length; i++) {
   const ds = config.datasets[i];
 
@@ -1110,6 +1162,7 @@ for (let i = 0; i < config.datasets.length; i++) {
     sections: reg?.sections ? reg.sections.map(s => s.toJSON()) : [],
     hasBibliography: fs.existsSync(path.join(datasetDir(ds), 'bibliography.yaml')),
     hasImages: fs.existsSync(path.join(datasetDir(ds), 'images')),
+    bulkExports,
   });
   const resolvedOwner = firstNonEmpty(reg?.owner, ds.owner) ?? '';
   const resolvedTags = firstNonEmpty(reg?.tags, ds.tags) ?? [];

package/scripts/lib/concept-uri.ts

@@ -15,3 +15,14 @@
 export function buildConceptUri(uriBase: string, registerId: string, conceptId: string | number): string {
   return `${uriBase}/dataset/${registerId}/concept/${conceptId}`;
 }
+
+/** Prefix of every concept URI of a register (for keying node maps and
+ *  matching edge endpoints against a register's URI space). */
+export function buildConceptUriPrefix(uriBase: string, registerId: string): string {
+  return `${uriBase}/dataset/${registerId}/concept/`;
+}
+
+/** Prefix of any URI belonging to a register (for the cross-ref index). */
+export function buildDatasetUriPrefix(uriBase: string, registerId: string): string {
+  return `${uriBase}/dataset/${registerId}/`;
+}

package/scripts/load-site-config.js

@@ -22,6 +22,7 @@ var SNAKE_TO_CAMEL = {
   footer_nav: "footerNav",
   footer_logo: "footerLogo",
   news_dir: "newsDir",
+  bulk_exports: "bulkExports",
   // Dataset-level
   gcr_package: "gcrPackage",
   source_repo: "sourceRepo",

package/scripts/load-site-config.ts

@@ -47,6 +47,7 @@ const SNAKE_TO_CAMEL = {
   footer_nav: 'footerNav',
   footer_logo: 'footerLogo',
   news_dir: 'newsDir',
+  bulk_exports: 'bulkExports',
   // Dataset-level
   gcr_package: 'gcrPackage',
   source_repo: 'sourceRepo',

package/scripts/process-about-pages.js

@@ -28,6 +28,7 @@ var SNAKE_TO_CAMEL = {
   footer_nav: "footerNav",
   footer_logo: "footerLogo",
   news_dir: "newsDir",
+  bulk_exports: "bulkExports",
   // Dataset-level
   gcr_package: "gcrPackage",
   source_repo: "sourceRepo",

package/src/__tests__/scripts/concept-csv.test.ts

@@ -0,0 +1,106 @@
+// CSV aggregate export — now routed through glossarist's canonical
+// output API (glossarist@0.4.58, glossarist-js#132). These specs pin the
+// production contract that shipped in concept-browser#214: column order,
+// preferred/alt split, RFC 4180 quoting, BOM + CRLF, languageOrder,
+// and no-drop rows for concepts without localizations.
+
+import { describe, it, expect } from 'vitest';
+import { Concept, CSV_COLUMNS, conceptsToCsv } from 'glossarist';
+
+function parseCsv(csv: string): string[][] {
+  const body = csv.replace(/^/, '');
+  const rows: string[][] = [];
+  let row: string[] = [];
+  let field = '';
+  let quoted = false;
+  for (let i = 0; i < body.length; i++) {
+    const ch = body[i];
+    if (quoted) {
+      if (ch === '"') {
+        if (body[i + 1] === '"') { field += '"'; i++; }
+        else quoted = false;
+      } else {
+        field += ch;
+      }
+    } else if (ch === '"') {
+      quoted = true;
+    } else if (ch === ',') {
+      row.push(field); field = '';
+    } else if (ch === '\r' && body[i + 1] === '\n') {
+      row.push(field); rows.push(row); row = []; field = ''; i++;
+    } else {
+      field += ch;
+    }
+  }
+  return rows;
+}
+
+const baseConcept = () => new Concept({
+  id: '1-1-010',
+  termid: '1-1-010',
+  status: 'valid',
+  uri: 'https://example.com/vocab/concept/1-1-010',
+  domains: [{ conceptId: 'section-1' }],
+  sources: [{ type: 'authoritative', origin: { ref: { source: 'IALA Dictionary' }, link: 'https://example.com/source' } }],
+  localizations: {
+    eng: {
+      terms: [
+        { designation: 'Buoyage and beaconage', normative_status: 'preferred' },
+        { designation: 'Sea-marking (G.B.)', normative_status: 'admitted' },
+      ],
+      definition: [{ content: 'Ensemble of beacons, buoys, seamarks and small lights.' }],
+      notes: [{ content: 'Note one.' }, { content: 'Note two.' }],
+    },
+    fra: {
+      terms: [{ designation: 'Balisage', normative_status: 'preferred' }],
+      definition: [{ content: 'Ensemble de balises.' }],
+    },
+  },
+});
+
+describe('conceptsToCsv (glossarist output API)', () => {
+  it('emits UTF-8 BOM, CRLF line endings, and the model-driven header', () => {
+    const csv = conceptsToCsv([baseConcept()]);
+    expect(csv.startsWith('')).toBe(true);
+    expect(csv.endsWith('\r\n')).toBe(true);
+    const rows = parseCsv(csv);
+    expect(rows[0]!.join(',')).toBe(CSV_COLUMNS.join(','));
+  });
+
+  it('one row per concept × language; preferred designation first', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()]));
+    expect(rows).toHaveLength(3);
+    const eng = rows[1]!;
+    expect(eng[0]).toBe('1-1-010');
+    expect(eng[4]).toBe('eng');
+    expect(eng[5]).toBe('Buoyage and beaconage');
+    expect(eng[6]).toBe('Sea-marking (G.B.)');
+  });
+
+  it('definition/notes columns carry the full content', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()]));
+    expect(rows[1]![7]).toBe('Ensemble of beacons, buoys, seamarks and small lights.');
+    expect(rows[1]![8]).toBe('Note one.\nNote two.');
+    expect(rows[2]![7]).toBe('Ensemble de balises.');
+  });
+
+  it('sources and source_links from origin ref/link', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()]));
+    expect(rows[1]![10]).toBe('IALA Dictionary');
+    expect(rows[1]![11]).toBe('https://example.com/source');
+  });
+
+  it('languageOrder reorders rows; unlisted languages append after', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()], { languageOrder: ['fra', 'deu'] }));
+    expect(rows[1]![4]).toBe('fra');
+    expect(rows[2]![4]).toBe('eng');
+  });
+
+  it('concept without localizations still gets a row (no silent drops)', () => {
+    const bare = new Concept({ id: '222-02-02', termid: '222-02-02' });
+    const rows = parseCsv(conceptsToCsv([bare]));
+    expect(rows).toHaveLength(2);
+    expect(rows[1]![0]).toBe('222-02-02');
+    expect(rows[1]![4]).toBe('');
+  });
+});

package/src/composables/use-format-registry.ts

@@ -40,3 +40,4 @@ registerFormat({ id: 'jsonld', extension: 'jsonld',  mediaType: 'application/ld+
 registerFormat({ id: 'yaml',   extension: 'yaml',   mediaType: 'application/yaml',       label: 'YAML',        available: 'per-concept',  serialize: 'build' });
 registerFormat({ id: 'tbx',    extension: 'tbx.xml', mediaType: 'application/x-tbx',     label: 'TBX',         available: 'aggregate',    serialize: 'build' });
 registerFormat({ id: 'jsonl',  extension: 'jsonl',  mediaType: 'application/jsonl+json', label: 'JSON-Lines',  available: 'aggregate',    serialize: 'build' });
+registerFormat({ id: 'csv',    extension: 'csv',    mediaType: 'text/csv',              label: 'CSV',         available: 'aggregate',    serialize: 'build' });

package/src/config/types.ts

@@ -186,6 +186,7 @@ export const FORMAT_LABELS: Record<string, string> = {
   tbx: 'TBX-XML',
   jsonl: 'JSONL',
   yaml: 'YAML',
+  csv: 'CSV (Excel)',
 };
 
 // === Pages ===