@glossarist/concept-browser 0.7.139 → 0.7.141

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glossarist/concept-browser",
-  "version": "0.7.139",
+  "version": "0.7.141",
   "description": "Vue SPA for browsing Glossarist terminology datasets with cross-reference resolution, graph visualization, and multi-language support",
   "type": "module",
   "bin": {

package/scripts/__tests__/standards-index.test.ts

@@ -0,0 +1,121 @@
+// @vitest-environment node
+import { describe, expect, it } from 'vitest';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+
+describe('generate-data standards index', () => {
+  it('emits unique authoritative standards per register and preserves source stats', () => {
+    const root = mkdtempSync(join(tmpdir(), 'generate-data-standards-'));
+    const conceptsDir = join(root, 'source', 'concepts');
+    const outputDir = join(root, 'public', 'data', 'fixture');
+    const configPath = join(root, 'site-config.yml');
+    const generator = resolve('scripts/generate-data.ts');
+    const tsx = resolve('node_modules/tsx/dist/loader.mjs');
+
+    try {
+      mkdirSync(conceptsDir, { recursive: true });
+      writeFileSync(configPath, JSON.stringify({
+        id: 'fixture', title: 'Fixture', uri_base: 'https://example.test',
+        datasets: [{ id: 'fixture', local_path: 'source', languages: ['eng', 'fra'] }],
+      }));
+      const writeConcept = (id: string, sources: unknown[], engSources: unknown[] = []) => {
+        writeFileSync(join(conceptsDir, `${id}.yaml`), JSON.stringify({
+          termid: id,
+          _sources: sources,
+          eng: {
+            terms: [{ designation: `Fixture term ${id}` }],
+            sources: engSources,
+          },
+        }));
+      };
+      writeConcept('1', [
+        { type: 'authoritative', origin: { ref: { source: 'ISO', id: '19165-2:2020', clause: '3.1' } } },
+        { type: 'authoritative', origin: { ref: { source: 'ISO', id: '19165-2:2020', clause: '4.2' } } },
+        { type: 'authoritative', origin: { ref: { source: 'IEC', id: '17000:2020' } } },
+        { type: 'lineage', origin: { ref: { source: 'ISO', id: '704:2022' } } },
+      ], [
+        { type: 'authoritative', origin: { ref: { source: 'iso', id: '19165-2:2020', clause: '5.1' } } },
+      ]);
+      writeConcept('2', [
+        { type: 'authoritative', origin: { ref: { source: ' ISO ', id: ' 19165-2:2020 ', clause: '6.3' } } },
+      ]);
+      writeConcept('3', [
+        { type: 'lineage', origin: { ref: { source: 'ISO', id: '999:2020' } } },
+      ], [
+        { type: 'authoritative', origin: { ref: { source: 'ISO', id: '1087:2019' } } },
+      ]);
+
+      execFileSync(process.execPath, ['--import', tsx, generator], {
+        cwd: root,
+        env: { ...process.env, SITE_CONFIG: configPath },
+        stdio: 'pipe',
+        timeout: 30_000,
+      });
+
+      const standards = JSON.parse(readFileSync(join(outputDir, 'standards.json'), 'utf8'));
+      expect(standards).toEqual({
+        'iso 19165-2:2020': { label: 'ISO 19165-2:2020', conceptIds: ['1', '2'] },
+        'iec 17000:2020': { label: 'IEC 17000:2020', conceptIds: ['1'] },
+        'iso 1087:2019': { label: 'ISO 1087:2019', conceptIds: ['3'] },
+      });
+
+      const stats = JSON.parse(readFileSync(join(outputDir, 'stats.json'), 'utf8'));
+      expect(stats.sourceCount).toBe(4);
+      expect(stats.sources).toEqual(expect.arrayContaining([
+        expect.objectContaining({ ref: 'ISO', conceptCount: 2, conceptIds: ['1', '3'] }),
+        expect.objectContaining({ ref: 'IEC', conceptCount: 1, conceptIds: ['1'] }),
+      ]));
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  }, 60_000);
+
+  it('includes authoritative standards from languages absent from the dataset config', () => {
+    const root = mkdtempSync(join(tmpdir(), 'generate-data-unconfigured-language-'));
+    const conceptsDir = join(root, 'source', 'concepts');
+    const configPath = join(root, 'site-config.yml');
+    try {
+      mkdirSync(conceptsDir, { recursive: true });
+      writeFileSync(configPath, JSON.stringify({
+        id: 'fixture', title: 'Fixture', uri_base: 'https://example.test',
+        datasets: [{ id: 'fixture', local_path: 'source', languages: ['eng'] }],
+      }));
+      writeFileSync(join(conceptsDir, '1.yaml'), JSON.stringify({
+        termid: '1',
+        eng: {
+          terms: [{ designation: 'Term' }],
+          sources: [{ type: 'authoritative', origin: { ref: { source: 'ISO' } } }],
+        },
+        deu: {
+          terms: [{ designation: 'Begriff' }],
+          sources: [
+            { type: 'authoritative', origin: { ref: { source: 'DIN', id: '1' } } },
+            { type: 'lineage', origin: { ref: { source: 'DIN', id: '2' } } },
+          ],
+        },
+      }));
+      execFileSync(process.execPath, [
+        '--import', resolve('node_modules/tsx/dist/loader.mjs'), resolve('scripts/generate-data.ts'),
+      ], {
+        cwd: root,
+        env: { ...process.env, SITE_CONFIG: configPath },
+        stdio: 'pipe',
+        timeout: 30_000,
+      });
+      const outputDir = join(root, 'public', 'data', 'fixture');
+      expect(JSON.parse(readFileSync(join(outputDir, 'standards.json'), 'utf8'))).toEqual({
+        iso: { label: 'ISO', conceptIds: ['1'] },
+        'din 1': { label: 'DIN 1', conceptIds: ['1'] },
+      });
+      const stats = JSON.parse(readFileSync(join(outputDir, 'stats.json'), 'utf8'));
+      expect(stats.sourceCount).toBe(1);
+      expect(stats.sources).toEqual([
+        { ref: 'ISO', types: ['authoritative'], conceptCount: 1, conceptIds: ['1'] },
+      ]);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  }, 60_000);
+});

package/scripts/build-edges.js

@@ -3,28 +3,6 @@ function buildConceptUri(uriBase, registerId, conceptId) {
   return `${uriBase}/dataset/${registerId}/concept/${conceptId}`;
 }
 
-// scripts/extract-source-refs.js
-function extractSourceRefs(concept, registerId) {
-  const refs = /* @__PURE__ */ new Set();
-  for (const src of concept["gl:source"] || []) {
-    const origin = src["gl:origin"];
-    if (origin) {
-      const ref = origin["gl:ref"];
-      if (ref?.["gl:source"]) refs.add(ref["gl:source"]);
-    }
-  }
-  for (const lc of Object.values(concept["gl:localizedConcept"] || {})) {
-    for (const src of lc["gl:source"] || []) {
-      const origin = src["gl:origin"];
-      if (origin) {
-        const ref = origin["gl:ref"];
-        if (ref?.["gl:source"]) refs.add(ref["gl:source"]);
-      }
-    }
-  }
-  return [...refs].map((source) => ({ source, registerId }));
-}
-
 // scripts/build-edges.ts
 import { readFileSync, writeFileSync, readdirSync, existsSync, realpathSync } from "fs";
 import { join, dirname } from "path";
@@ -285,7 +263,7 @@ function buildEdgesForDataset(datasetDir, registerId, uriBase, urnMap, manifest)
       warnBinaryPartitiveRedundancy(edges, relations, registerId);
       allEdges.push(...edges);
       allPartitiveRelations.push(...relations);
-      allSourceRefs.push(...extractSourceRefs(data, registerId));
+      allSourceRefs.push(...(void 0)(data, registerId));
       for (const edge of edges) {
         if (edge.type === "domain" || edge.type === "section") {
           domainConceptCount.set(edge.target, (domainConceptCount.get(edge.target) || 0) + 1);

package/scripts/generate-data.js

@@ -1,6 +1,9 @@
 // scripts/generate-data.ts
 import fs2 from "fs";
+import os from "os";
 import path3 from "path";
+import { Worker as WorkerThread } from "node:worker_threads";
+import { createHash as createHash2 } from "node:crypto";
 import yaml4 from "js-yaml";
 import {
   naturalSort,
@@ -89,6 +92,28 @@ function loadSiteConfig(args = []) {
   return { config: config2, configPath };
 }
 
+// src/utils/source-standard.ts
+function standardLabel(ref) {
+  const parts = [ref?.source, ref?.id].map((part) => part?.trim().replace(/\s+/g, " ") ?? "").filter(Boolean);
+  return parts.join(" ") || null;
+}
+function standardKey(ref) {
+  return standardLabel(ref)?.toLowerCase() ?? null;
+}
+function collectStandardKeys(concept) {
+  const sources = [
+    ...concept.sources,
+    ...concept.languages.flatMap((lang) => concept.localization(lang)?.sources ?? [])
+  ];
+  const keys = /* @__PURE__ */ new Set();
+  for (const source of sources) {
+    if (source.type !== "authoritative") continue;
+    const key = standardKey(source.origin?.ref);
+    if (key !== null) keys.add(key);
+  }
+  return [...keys];
+}
+
 // scripts/lib/concept-groups.ts
 function getGroups(conceptYaml) {
   if (conceptYaml.eng && conceptYaml.eng.groups) return conceptYaml.eng.groups;
@@ -1229,18 +1254,104 @@ function getPrimaryDesignation(conceptYaml) {
   }
   return descs;
 }
+async function inlineEmission(job, register, uriBase) {
+  try {
+    const outputs = await emitOutputSet([job.conceptInstance], {
+      registerId: register,
+      uriBase,
+      formats: ["jsonld"]
+    });
+    return {
+      termid: job.termid,
+      ttl: conceptToTurtle(job.conceptInstance, { uriBase, registerId: register }),
+      jsonld: outputs.jsonld,
+      tbx: conceptsToTbx([job.conceptInstance], { registerId: register }) || null
+    };
+  } catch (e) {
+    return { termid: job.termid, error: String(e && e.message || e) };
+  }
+}
+function createEmissionPool(register, uriBase) {
+  const configured = Number(process.env.GCB_WORKERS);
+  if (Number.isFinite(configured) && configured <= 0) return null;
+  const size = Number.isFinite(configured) ? configured : Math.max(1, Math.min(4, (os.availableParallelism ? os.availableParallelism() : os.cpus().length) - 1));
+  const workerPath = path3.join(__dirname3, "lib", "workers", "emit-concept-worker.mjs");
+  let pool;
+  try {
+    pool = Array.from({ length: size }, () => new WorkerThread(workerPath, { workerData: { registerId: register, uriBase } }));
+  } catch {
+    return null;
+  }
+  const active = /* @__PURE__ */ new Map();
+  const pending = [];
+  const idle = /* @__PURE__ */ new Set();
+  function dispatch() {
+    while (pending.length > 0) {
+      const worker = idle.values().next();
+      if (worker.done) return;
+      idle.delete(worker.value);
+      const entry = pending.shift();
+      active.set(worker.value, entry);
+      worker.value.postMessage({ termid: entry.job.termid, sourceText: entry.job.sourceText });
+    }
+  }
+  for (const worker of pool) {
+    idle.add(worker);
+    worker.on("message", (msg) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.add(worker);
+      entry?.resolve(msg);
+      dispatch();
+    });
+    const fail = (err) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.delete(worker);
+      pool = pool.filter((w) => w !== worker);
+      entry?.reject(err);
+      dispatch();
+    };
+    worker.on("error", fail);
+    worker.on("exit", () => fail(new Error("emission worker exited")));
+  }
+  return {
+    emit(job) {
+      return new Promise((resolve2, reject) => {
+        pending.push({ job, resolve: resolve2, reject });
+        dispatch();
+      });
+    },
+    async close() {
+      await Promise.allSettled(pool.map((w) => w.terminate()));
+    }
+  };
+}
 async function processDataset(dir, register, opts) {
   const files = fs2.readdirSync(dir).filter((f) => f.endsWith(".yaml")).sort((a, b) => naturalSort(a.replace(".yaml", ""), b.replace(".yaml", "")));
   console.log(`Processing ${register}: ${files.length} files`);
   const conceptsDir = path3.join(DATA, register, "concepts");
-  if (fs2.existsSync(conceptsDir)) {
-    for (const f of fs2.readdirSync(conceptsDir)) fs2.unlinkSync(path3.join(conceptsDir, f));
+  fs2.mkdirSync(conceptsDir, { recursive: true });
+  const EMISSION_CACHE_VERSION = 1;
+  const EMISSION_EXTENSIONS = ["json", "ttl", "jsonld", "tbx", "yaml"];
+  const cacheDir = path3.join(ROOT, ".gcb-cache");
+  const cachePath = path3.join(cacheDir, `${register}.json`);
+  const emissionCache = {
+    version: EMISSION_CACHE_VERSION,
+    files: {}
+  };
+  try {
+    const loaded = JSON.parse(fs2.readFileSync(cachePath, "utf8"));
+    if (loaded?.version === EMISSION_CACHE_VERSION && loaded.files) emissionCache.files = loaded.files;
+  } catch {
   }
   const concepts = [];
   const conceptInstances = [];
   const langTermCounts = {};
   const langDefCounts = {};
   const availableFormats = ["ttl", "jsonld", "yaml", "tbx", "csv"];
+  const producedOutputs = /* @__PURE__ */ new Set();
+  const emissionCacheFiles = {};
   const designationLookup = /* @__PURE__ */ new Map();
   for (const file of files) {
     try {
@@ -1275,7 +1386,26 @@ async function processDataset(dir, register, opts) {
       withPlurality: 0
     }
   };
+  const standards = /* @__PURE__ */ new Map();
   const STATS_PROCESSORS = [
+    function collectStandards(cy, termid) {
+      const concept = {
+        sources: cy._sources || [],
+        languages: Object.keys(cy).filter((lang) => !lang.startsWith("_") && cy[lang] && typeof cy[lang] === "object" && !Array.isArray(cy[lang])),
+        localization: (lang) => cy[lang]
+      };
+      const sources = [
+        ...concept.sources,
+        ...concept.languages.flatMap((lang) => concept.localization(lang).sources || [])
+      ];
+      for (const key of collectStandardKeys(concept)) {
+        if (!standards.has(key)) {
+          const source = sources.find((source2) => source2.type === "authoritative" && standardKey(source2.origin?.ref) === key);
+          standards.set(key, { label: standardLabel(source.origin.ref), conceptIds: [] });
+        }
+        standards.get(key).conceptIds.push(termid);
+      }
+    },
     function collectSources(cy, termid, s) {
       const refs = /* @__PURE__ */ new Set();
       const allSources = [
@@ -1334,38 +1464,44 @@ async function processDataset(dir, register, opts) {
       }
     }
   ];
+  const loopStart = Date.now();
+  const emissionJobs = [];
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
+    if (i > 0 && i % 250 === 0) console.log(`  ${i}/${files.length} concepts`);
     try {
-      const conceptYaml = loadConceptFile(path3.join(dir, file));
+      const filePath = path3.join(dir, file);
+      const sourceText = fs2.readFileSync(filePath, "utf8");
+      const sourceHash = createHash2("sha256").update(sourceText).digest("hex");
+      const conceptYaml = loadConceptFile(filePath);
       if (!conceptYaml || !conceptYaml.termid) continue;
       const termid = String(conceptYaml.termid);
-      const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
-      writeJson(path3.join(conceptsDir, `${termid}.json`), jsonld);
+      const cached = emissionCache.files[file];
+      const unchanged = cached?.hash === sourceHash && EMISSION_EXTENSIONS.every((ext) => fs2.existsSync(path3.join(conceptsDir, `${termid}.${ext}`)));
       let conceptInstance = null;
       try {
-        conceptInstance = parseConceptYaml(fs2.readFileSync(path3.join(dir, file), "utf8"), termid);
+        conceptInstance = parseConceptYaml(sourceText, termid);
       } catch (e) {
         console.warn(`  ${termid}: canonical parse failed (${e.message}); skipping glossarist exports`);
       }
-      if (conceptInstance) {
-        conceptInstances.push(conceptInstance);
-        fs2.writeFileSync(
-          path3.join(conceptsDir, `${termid}.ttl`),
-          conceptToTurtle(conceptInstance, { uriBase: refMaps.uriBase, registerId: register })
-        );
-        const conceptJsonLd = await emitOutputSet([conceptInstance], {
-          registerId: register,
-          uriBase: refMaps.uriBase,
-          formats: ["jsonld"]
-        });
-        fs2.writeFileSync(path3.join(conceptsDir, `${termid}.jsonld`), conceptJsonLd.jsonld);
-        const tbxContent = conceptsToTbx([conceptInstance], { registerId: register });
-        if (tbxContent) {
-          fs2.writeFileSync(path3.join(conceptsDir, `${termid}.tbx`), tbxContent);
+      if (conceptInstance) conceptInstances.push(conceptInstance);
+      if (!unchanged) {
+        const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
+        writeJson(path3.join(conceptsDir, `${termid}.json`), jsonld);
+        fs2.copyFileSync(filePath, path3.join(conceptsDir, `${termid}.yaml`));
+        if (conceptInstance) {
+          emissionJobs.push({ file, termid, sourceText, sourceHash, conceptInstance });
         }
+      } else {
+        emissionCacheFiles[file] = { hash: sourceHash, termid };
+      }
+      producedOutputs.add(`${termid}.json`);
+      producedOutputs.add(`${termid}.yaml`);
+      if (conceptInstance) {
+        producedOutputs.add(`${termid}.ttl`);
+        producedOutputs.add(`${termid}.jsonld`);
+        producedOutputs.add(`${termid}.tbx`);
       }
-      fs2.copyFileSync(path3.join(dir, file), path3.join(conceptsDir, `${termid}.yaml`));
       concepts.push({
         id: termid,
         designations: getPrimaryDesignation(conceptYaml),
@@ -1379,12 +1515,49 @@ async function processDataset(dir, register, opts) {
       console.warn(`  Skipping ${file}: ${e.message}`);
     }
   }
+  if (emissionJobs.length > 0) {
+    const jobsByTermid = new Map(emissionJobs.map((j) => [j.termid, j]));
+    const pool = createEmissionPool(register, refMaps.uriBase);
+    const emissionStart = Date.now();
+    try {
+      const results = await Promise.all(emissionJobs.map((job) => {
+        if (!pool) return inlineEmission(job, register, refMaps.uriBase);
+        return pool.emit(job).catch(() => inlineEmission(job, register, refMaps.uriBase));
+      }));
+      for (const result of results) {
+        if (result.error) {
+          console.warn(`  ${result.termid}: canonical parse failed (${result.error}); skipping glossarist exports`);
+          const job2 = jobsByTermid.get(result.termid);
+          if (job2) delete emissionCacheFiles[job2.file];
+          for (const ext of ["ttl", "jsonld", "tbx"]) producedOutputs.delete(`${result.termid}.${ext}`);
+          continue;
+        }
+        fs2.writeFileSync(path3.join(conceptsDir, `${result.termid}.ttl`), result.ttl);
+        fs2.writeFileSync(path3.join(conceptsDir, `${result.termid}.jsonld`), result.jsonld);
+        if (result.tbx) fs2.writeFileSync(path3.join(conceptsDir, `${result.termid}.tbx`), result.tbx);
+        const job = jobsByTermid.get(result.termid);
+        if (job) emissionCacheFiles[job.file] = { hash: job.sourceHash, termid: job.termid };
+      }
+      console.log(`  Interchange exports: ${((Date.now() - emissionStart) / 1e3).toFixed(1)}s (${emissionJobs.length} concepts, ${pool ? "parallel" : "inline"})`);
+    } finally {
+      if (pool) await pool.close();
+    }
+  }
+  for (const entry of fs2.readdirSync(conceptsDir)) {
+    if (!producedOutputs.has(entry)) fs2.unlinkSync(path3.join(conceptsDir, entry));
+  }
+  fs2.mkdirSync(cacheDir, { recursive: true });
+  fs2.writeFileSync(cachePath, JSON.stringify({ version: EMISSION_CACHE_VERSION, files: emissionCacheFiles }));
+  console.log(`  Per-concept emissions: ${((Date.now() - loopStart) / 1e3).toFixed(1)}s`);
+  const aggregateStart = Date.now();
   const aggregate = await emitOutputSet(conceptInstances, {
     registerId: register,
     uriBase: refMaps.uriBase,
+    formats: ["csv", "jsonld"],
     languageOrder: opts.languageOrder || opts.languages,
     conceptUriPrefix: buildConceptUriPrefix(refMaps.siteOrigin, register)
   });
+  console.log(`  Aggregate csv+jsonld: ${((Date.now() - aggregateStart) / 1e3).toFixed(1)}s`);
   fs2.writeFileSync(path3.join(DATA, register, `${register}.csv`), aggregate.csv);
   fs2.writeFileSync(path3.join(DATA, register, `${register}.jsonld`), aggregate.jsonld);
   const CHUNK_SIZE = 500;
@@ -1393,12 +1566,17 @@ async function processDataset(dir, register, opts) {
     const chunk = concepts.slice(i, i + CHUNK_SIZE);
     const chunkIndex = Math.floor(i / CHUNK_SIZE);
     const chunkFile = `index-${String(chunkIndex).padStart(4, "0")}.json`;
-    writeJson(path3.join(DATA, register, "chunks", chunkFile), {
+    const chunkData = {
       registerId: register,
       chunkIndex,
       concepts: chunk
-    });
-    chunks.push({ file: chunkFile, count: chunk.length });
+    };
+    const chunkContent = JSON.stringify(chunkData);
+    const chunkPath = path3.join(DATA, register, "chunks", chunkFile);
+    fs2.mkdirSync(path3.dirname(chunkPath), { recursive: true });
+    fs2.writeFileSync(chunkPath, chunkContent);
+    const hash = createHash2("sha256").update(chunkContent).digest("hex").slice(0, 12);
+    chunks.push({ file: chunkFile, count: chunk.length, hash });
   }
   const summary = concepts.map((c) => ({
     id: c.id,
@@ -1490,6 +1668,7 @@ async function processDataset(dir, register, opts) {
     conceptIds: s.conceptIds
   })).sort((a, b) => b.conceptCount - a.conceptCount);
   const totalRelationships = Object.values(stats.relTypeCounts).reduce((a, b) => a + b, 0);
+  writeJson(path3.join(DATA, register, "standards.json"), Object.fromEntries(standards));
   writeJson(path3.join(DATA, register, "stats.json"), {
     sourceCount: sourceStats.length,
     sources: sourceStats,
@@ -1532,7 +1711,9 @@ async function processDataset(dir, register, opts) {
   if (opts.status) manifest.editionStatus = opts.status;
   if (opts.ordering) manifest.ordering = opts.ordering;
   if (opts.sections && opts.sections.length > 0) manifest.sections = opts.sections;
+  const rdfStart = Date.now();
   await writeDatasetRdf(register, manifest, concepts, refMaps, opts);
+  console.log(`  Dataset RDF: ${((Date.now() - rdfStart) / 1e3).toFixed(1)}s`);
   addBulkFormat(`${register}.ttl`);
   manifest.bulkFormats = opts.bulkExports?.length ? bulkFormats.filter((f) => opts.bulkExports.includes(f.format)) : bulkFormats;
   writeJson(path3.join(DATA, register, "manifest.json"), manifest);