@glossarist/concept-browser 0.7.135 → 0.7.136

package/scripts/generate-data.ts

@@ -1,7 +1,19 @@
 import fs from 'fs';
 import path from 'path';
 import yaml from 'js-yaml';
-import { naturalSort, Register, parseMention, parseMentions, InvalidMentionError } from 'glossarist';
+import {
+  naturalSort,
+  Register,
+  parseMention,
+  parseMentions,
+  InvalidMentionError,
+  parseConceptYaml,
+  conceptToTurtle,
+  conceptsToTbx,
+  conceptsToCsv,
+  emitOutputSet,
+  type Concept,
+} from 'glossarist';
 import { loadSiteConfig } from './load-site-config';
 import { getGroups } from './lib/concept-groups';
 import { consumeDatasetEntities } from './lib/build/non-verbal-consumer';
@@ -23,7 +35,7 @@ import type {
   YamlNewsFrontmatter,
   YamlContentPage,
 } from './lib/yaml-types';
-import { buildConceptUri } from './lib/concept-uri';
+import { buildConceptUri, buildConceptUriPrefix } from './lib/concept-uri';
 import {
   loadConceptFile,
   readYaml,
@@ -41,15 +53,6 @@ import {
   refsToJsonLd,
   citationToJsonLd,
 } from './lib/jsonld-emitter';
-import {
-  escapeTurtle,
-  escapeXml,
-  conceptJsonToTurtle,
-  conceptJsonToSkosJsonLd,
-  conceptJsonToTbx,
-  conceptsToCsv,
-  conceptsToSkosJsonLdGraph,
-} from './lib/concept-formats';
 
 // MECE partitive multiplicity: 2 independent axes (ISO 704:2022).
 //   presence × count
@@ -691,7 +694,7 @@ async function processDataset(dir, register, opts) {
     for (const f of fs.readdirSync(conceptsDir)) fs.unlinkSync(path.join(conceptsDir, f));
   }
   const concepts = [];
-  const jsonlds = [];
+  const conceptInstances: Concept[] = [];
   const langTermCounts = {};
   const langDefCounts = {};
   const availableFormats = ['ttl', 'jsonld', 'yaml', 'tbx', 'csv'];
@@ -805,17 +808,34 @@ async function processDataset(dir, register, opts) {
       const termid = String(conceptYaml.termid);
       const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
       writeJson(path.join(conceptsDir, `${termid}.json`), jsonld);
-      jsonlds.push(jsonld);
-
-      const ttlContent = conceptJsonToTurtle(jsonld);
-      fs.writeFileSync(path.join(conceptsDir, `${termid}.ttl`), ttlContent);
-
-      const skosJsonLd = conceptJsonToSkosJsonLd(jsonld);
-      fs.writeFileSync(path.join(conceptsDir, `${termid}.jsonld`), skosJsonLd);
-
-      const tbxContent = conceptJsonToTbx(jsonld);
-      if (tbxContent) {
-        fs.writeFileSync(path.join(conceptsDir, `${termid}.tbx`), tbxContent);
+      // Canonical interchange exports — routed through glossarist's
+      // output API (one model walk; predicates generated from the
+      // concept-model context, wire shape from schemas/v3). The hand-rolled
+      // gl:-shaped emitters in scripts/lib/concept-formats.ts are retired.
+      // Note: TBX now includes <term> entries — the retired emitter
+      // computed termEntries but never wrote them into the document.
+      let conceptInstance: Concept | null = null;
+      try {
+        conceptInstance = parseConceptYaml(fs.readFileSync(path.join(dir, file), 'utf8'), termid);
+      } catch (e) {
+        console.warn(`  ${termid}: canonical parse failed (${e.message}); skipping glossarist exports`);
+      }
+      if (conceptInstance) {
+        conceptInstances.push(conceptInstance);
+        fs.writeFileSync(
+          path.join(conceptsDir, `${termid}.ttl`),
+          conceptToTurtle(conceptInstance, { uriBase: refMaps.uriBase, registerId: register }),
+        );
+        const conceptJsonLd = await emitOutputSet([conceptInstance], {
+          registerId: register,
+          uriBase: refMaps.uriBase,
+          formats: ['jsonld'],
+        });
+        fs.writeFileSync(path.join(conceptsDir, `${termid}.jsonld`), conceptJsonLd.jsonld!);
+        const tbxContent = conceptsToTbx([conceptInstance], { registerId: register });
+        if (tbxContent) {
+          fs.writeFileSync(path.join(conceptsDir, `${termid}.tbx`), tbxContent);
+        }
       }
 
       fs.copyFileSync(path.join(dir, file), path.join(conceptsDir, `${termid}.yaml`));
@@ -835,9 +855,15 @@ async function processDataset(dir, register, opts) {
     }
   }
 
-  // Aggregate distributions: single-file exports of the whole register
-  fs.writeFileSync(path.join(DATA, register, `${register}.csv`), conceptsToCsv(jsonlds, { languageOrder: opts.languageOrder || opts.languages }));
-  fs.writeFileSync(path.join(DATA, register, `${register}.jsonld`), conceptsToSkosJsonLdGraph(jsonlds));
+  // Aggregate distributions: single-file exports of the whole register,
+  // emitted by glossarist's output-set API from the canonical model walk.
+  const aggregate = await emitOutputSet(conceptInstances, {
+    registerId: register,
+    uriBase: refMaps.uriBase,
+    languageOrder: opts.languageOrder || opts.languages,
+  });
+  fs.writeFileSync(path.join(DATA, register, `${register}.csv`), aggregate.csv!);
+  fs.writeFileSync(path.join(DATA, register, `${register}.jsonld`), aggregate.jsonld!);
 
   const CHUNK_SIZE = 500;
   const chunks = [];
@@ -885,7 +911,7 @@ async function processDataset(dir, register, opts) {
   fs.writeFileSync(
     path.join(DATA, register, 'graph-nodes.json'),
     JSON.stringify({
-      uriPrefix: `${refMaps.uriBase}/${register}/concept/`,
+      uriPrefix: buildConceptUriPrefix(refMaps.uriBase, register),
       registerId: register,
       nodes: graphNodeEntries,
     }),

package/scripts/lib/concept-uri.ts

@@ -15,3 +15,14 @@
 export function buildConceptUri(uriBase: string, registerId: string, conceptId: string | number): string {
   return `${uriBase}/dataset/${registerId}/concept/${conceptId}`;
 }
+
+/** Prefix of every concept URI of a register (for keying node maps and
+ *  matching edge endpoints against a register's URI space). */
+export function buildConceptUriPrefix(uriBase: string, registerId: string): string {
+  return `${uriBase}/dataset/${registerId}/concept/`;
+}
+
+/** Prefix of any URI belonging to a register (for the cross-ref index). */
+export function buildDatasetUriPrefix(uriBase: string, registerId: string): string {
+  return `${uriBase}/dataset/${registerId}/`;
+}

package/src/__tests__/scripts/concept-csv.test.ts

@@ -1,8 +1,14 @@
+// CSV aggregate export — now routed through glossarist's canonical
+// output API (glossarist@0.4.58, glossarist-js#132). These specs pin the
+// production contract that shipped in concept-browser#214: column order,
+// preferred/alt split, RFC 4180 quoting, BOM + CRLF, languageOrder,
+// and no-drop rows for concepts without localizations.
+
 import { describe, it, expect } from 'vitest';
-import { conceptsToCsv, conceptsToSkosJsonLdGraph, CSV_COLUMNS } from '../../../scripts/lib/concept-formats';
+import { Concept, CSV_COLUMNS, conceptsToCsv } from 'glossarist';
 
 function parseCsv(csv: string): string[][] {
-  const body = csv.replace(/^\uFEFF/, '');
+  const body = csv.replace(/^/, '');
   const rows: string[][] = [];
   let row: string[] = [];
   let field = '';
@@ -29,114 +35,72 @@ function parseCsv(csv: string): string[][] {
   return rows;
 }
 
-const baseConcept = {
-  '@id': 'https://example.com/vocab/concept/1-1-010',
-  'gl:identifier': '1-1-010',
-  'gl:status': 'valid',
-  'gl:domain': [{ 'gl:conceptId': 'section-1' }],
-  'gl:source': [{
-    'gl:sourceType': 'authoritative',
-    'gl:origin': { 'gl:ref': { 'gl:source': 'IALA Dictionary' }, 'gl:link': 'https://example.com/source' },
-  }],
-  'gl:localizedConcept': {
+const baseConcept = () => new Concept({
+  id: '1-1-010',
+  termid: '1-1-010',
+  status: 'valid',
+  uri: 'https://example.com/vocab/concept/1-1-010',
+  domains: [{ conceptId: 'section-1' }],
+  sources: [{ type: 'authoritative', origin: { ref: { source: 'IALA Dictionary' }, link: 'https://example.com/source' } }],
+  localizations: {
     eng: {
-      'gl:designation': [
-        { 'gl:normativeStatus': 'preferred', 'gl:term': 'Buoyage and beaconage' },
-        { 'gl:normativeStatus': 'admitted', 'gl:term': 'Sea-marking (G.B.)' },
+      terms: [
+        { designation: 'Buoyage and beaconage', normative_status: 'preferred' },
+        { designation: 'Sea-marking (G.B.)', normative_status: 'admitted' },
       ],
-      'gl:definition': [{ 'gl:content': 'Ensemble of beacons, buoys, seamarks and small lights.' }],
-      'gl:notes': [{ 'gl:content': 'Note one.' }, { 'gl:content': 'Note two.' }],
+      definition: [{ content: 'Ensemble of beacons, buoys, seamarks and small lights.' }],
+      notes: [{ content: 'Note one.' }, { content: 'Note two.' }],
     },
     fra: {
-      'gl:designation': [{ 'gl:normativeStatus': 'preferred', 'gl:term': 'Balisage' }],
-      'gl:definition': [{ 'gl:content': 'Ensemble de balises.' }],
+      terms: [{ designation: 'Balisage', normative_status: 'preferred' }],
+      definition: [{ content: 'Ensemble de balises.' }],
     },
   },
-};
+});
 
-describe('conceptsToCsv', () => {
+describe('conceptsToCsv (glossarist output API)', () => {
   it('emits UTF-8 BOM, CRLF line endings, and the model-driven header', () => {
-    const csv = conceptsToCsv([baseConcept]);
-    expect(csv.startsWith('\uFEFF')).toBe(true);
+    const csv = conceptsToCsv([baseConcept()]);
+    expect(csv.startsWith('')).toBe(true);
     expect(csv.endsWith('\r\n')).toBe(true);
-    expect(csv).toContain('\r\n');
     const rows = parseCsv(csv);
-    expect(rows[0]).toEqual([...CSV_COLUMNS]);
+    expect(rows[0]!.join(',')).toBe(CSV_COLUMNS.join(','));
   });
 
-  it('emits one row per concept and language', () => {
-    const rows = parseCsv(conceptsToCsv([baseConcept]));
+  it('one row per concept × language; preferred designation first', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()]));
     expect(rows).toHaveLength(3);
-    expect(rows[1][CSV_COLUMNS.indexOf('termid')]).toBe('1-1-010');
-    expect(rows[1][CSV_COLUMNS.indexOf('language')]).toBe('eng');
-    expect(rows[2][CSV_COLUMNS.indexOf('language')]).toBe('fra');
-  });
-
-  it('honors languageOrder from the deployment config', () => {
-    const rows = parseCsv(conceptsToCsv([baseConcept], { languageOrder: ['fra', 'eng'] }));
-    expect(rows[1][CSV_COLUMNS.indexOf('language')]).toBe('fra');
-    expect(rows[2][CSV_COLUMNS.indexOf('language')]).toBe('eng');
-  });
-
-  it('keeps languages missing from languageOrder at the end', () => {
-    const rows = parseCsv(conceptsToCsv([baseConcept], { languageOrder: ['deu', 'eng'] }));
-    expect(rows.slice(1).map(r => r[CSV_COLUMNS.indexOf('language')])).toEqual(['eng', 'fra']);
-  });
-
-  it('separates the preferred term from alt terms', () => {
-    const rows = parseCsv(conceptsToCsv([baseConcept]));
-    const eng = rows[1];
-    expect(eng[CSV_COLUMNS.indexOf('term')]).toBe('Buoyage and beaconage');
-    expect(eng[CSV_COLUMNS.indexOf('alt_terms')]).toBe('Sea-marking (G.B.)');
+    const eng = rows[1]!;
+    expect(eng[0]).toBe('1-1-010');
+    expect(eng[4]).toBe('eng');
+    expect(eng[5]).toBe('Buoyage and beaconage');
+    expect(eng[6]).toBe('Sea-marking (G.B.)');
   });
 
-  it('joins multi-entry notes with a newline inside a quoted cell', () => {
-    const rows = parseCsv(conceptsToCsv([baseConcept]));
-    expect(rows[1][CSV_COLUMNS.indexOf('notes')]).toBe('Note one.\nNote two.');
+  it('definition/notes columns carry the full content', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()]));
+    expect(rows[1]![7]).toBe('Ensemble of beacons, buoys, seamarks and small lights.');
+    expect(rows[1]![8]).toBe('Note one.\nNote two.');
+    expect(rows[2]![7]).toBe('Ensemble de balises.');
   });
 
-  it('escapes quotes and commas per RFC 4180', () => {
-    const concept = JSON.parse(JSON.stringify(baseConcept));
-    concept['gl:localizedConcept'].eng['gl:definition'] = [{ 'gl:content': 'Says "hello", loudly' }];
-    const csv = conceptsToCsv([concept]);
-    expect(csv).toContain('"Says ""hello"", loudly"');
-    expect(parseCsv(csv)[1][CSV_COLUMNS.indexOf('definition')]).toBe('Says "hello", loudly');
+  it('sources and source_links from origin ref/link', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()]));
+    expect(rows[1]![10]).toBe('IALA Dictionary');
+    expect(rows[1]![11]).toBe('https://example.com/source');
   });
 
-  it('falls back to managed sources when the localization has none', () => {
-    const rows = parseCsv(conceptsToCsv([baseConcept]));
-    for (const row of rows.slice(1)) {
-      expect(row[CSV_COLUMNS.indexOf('sources')]).toBe('IALA Dictionary');
-      expect(row[CSV_COLUMNS.indexOf('source_links')]).toBe('https://example.com/source');
-    }
+  it('languageOrder reorders rows; unlisted languages append after', () => {
+    const rows = parseCsv(conceptsToCsv([baseConcept()], { languageOrder: ['fra', 'deu'] }));
+    expect(rows[1]![4]).toBe('fra');
+    expect(rows[2]![4]).toBe('eng');
   });
 
-  it('still emits a row for concepts with no localizations', () => {
-    const bare = {
-      '@id': 'https://example.com/vocab/concept/9-9-999',
-      'gl:identifier': '9-9-999',
-      'gl:status': 'retired',
-    };
+  it('concept without localizations still gets a row (no silent drops)', () => {
+    const bare = new Concept({ id: '222-02-02', termid: '222-02-02' });
     const rows = parseCsv(conceptsToCsv([bare]));
     expect(rows).toHaveLength(2);
-    expect(rows[1][CSV_COLUMNS.indexOf('termid')]).toBe('9-9-999');
-    expect(rows[1][CSV_COLUMNS.indexOf('language')]).toBe('');
-    expect(rows[1][CSV_COLUMNS.indexOf('term')]).toBe('');
-  });
-
-  it('carries section domains and status columns', () => {
-    const rows = parseCsv(conceptsToCsv([baseConcept]));
-    expect(rows[1][CSV_COLUMNS.indexOf('section')]).toBe('section-1');
-    expect(rows[1][CSV_COLUMNS.indexOf('status')]).toBe('valid');
-  });
-});
-
-describe('conceptsToSkosJsonLdGraph', () => {
-  it('wraps concepts in a shared @context and @graph', () => {
-    const parsed = JSON.parse(conceptsToSkosJsonLdGraph([baseConcept]));
-    expect(parsed['@context']['skos']).toBe('http://www.w3.org/2004/02/skos/core#');
-    expect(parsed['@graph']).toHaveLength(1);
-    expect(parsed['@graph'][0]['@id']).toBe(baseConcept['@id']);
-    expect(parsed['@graph'][0]['skos:prefLabel']['eng']).toBe('Buoyage and beaconage');
+    expect(rows[1]![0]).toBe('222-02-02');
+    expect(rows[1]![4]).toBe('');
   });
 });