@glossarist/concept-browser 0.7.139 → 0.7.141

package/scripts/generate-data.ts

@@ -1,5 +1,8 @@
 import fs from 'fs';
+import os from 'os';
 import path from 'path';
+import { Worker as WorkerThread } from 'node:worker_threads';
+import { createHash } from 'node:crypto';
 import yaml from 'js-yaml';
 import {
   naturalSort,
@@ -15,6 +18,8 @@ import {
   type Concept,
 } from 'glossarist';
 import { loadSiteConfig } from './load-site-config';
+import { collectStandardKeys, standardKey, standardLabel } from '../src/utils/source-standard';
+import type { StandardsIndex } from '../src/adapters/standards-index';
 import { getGroups } from './lib/concept-groups';
 import { consumeDatasetEntities } from './lib/build/non-verbal-consumer';
 import { copyImageAssets } from './lib/build/image-assets';
@@ -687,21 +692,129 @@ function getPrimaryDesignation(conceptYaml) {
   return descs;
 }
 
+// Worker pool for glossarist-native per-concept emissions (ttl/jsonld/tbx).
+// Set GCB_WORKERS=0 to force inline emission (same outputs, one thread).
+type EmissionJob = { file: string; termid: string; sourceText: string; sourceHash: string; conceptInstance: Concept };
+type EmissionResult = { termid: string; ttl?: string; jsonld?: string; tbx?: string; error?: string };
+
+async function inlineEmission(job: EmissionJob, register: string, uriBase: string): Promise<EmissionResult> {
+  try {
+    const outputs = await emitOutputSet([job.conceptInstance], {
+      registerId: register,
+      uriBase,
+      formats: ['jsonld'],
+    });
+    return {
+      termid: job.termid,
+      ttl: conceptToTurtle(job.conceptInstance, { uriBase, registerId: register }),
+      jsonld: outputs.jsonld,
+      tbx: conceptsToTbx([job.conceptInstance], { registerId: register }) || null,
+    };
+  } catch (e) {
+    return { termid: job.termid, error: String((e && e.message) || e) };
+  }
+}
+
+function createEmissionPool(register: string, uriBase: string): {
+  emit(job: EmissionJob): Promise<EmissionResult>;
+  close(): Promise<void>;
+} | null {
+  const configured = Number(process.env.GCB_WORKERS);
+  if (Number.isFinite(configured) && configured <= 0) return null;
+  const size = Number.isFinite(configured)
+    ? configured
+    : Math.max(1, Math.min(4, (os.availableParallelism ? os.availableParallelism() : os.cpus().length) - 1));
+  const workerPath = path.join(__dirname, 'lib', 'workers', 'emit-concept-worker.mjs');
+
+  let pool: WorkerThread[];
+  try {
+    pool = Array.from({ length: size }, () => new WorkerThread(workerPath, { workerData: { registerId: register, uriBase } }));
+  } catch {
+    return null;
+  }
+
+  const active = new Map<WorkerThread, { resolve: (r: EmissionResult) => void; reject: (e: Error) => void }>();
+  const pending: { job: EmissionJob; resolve: (r: EmissionResult) => void; reject: (e: Error) => void }[] = [];
+  const idle = new Set<WorkerThread>();
+
+  function dispatch() {
+    while (pending.length > 0) {
+      const worker = idle.values().next();
+      if (worker.done) return;
+      idle.delete(worker.value);
+      const entry = pending.shift()!;
+      active.set(worker.value, entry);
+      worker.value.postMessage({ termid: entry.job.termid, sourceText: entry.job.sourceText });
+    }
+  }
+
+  for (const worker of pool) {
+    idle.add(worker);
+    worker.on('message', (msg: EmissionResult) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.add(worker);
+      entry?.resolve(msg);
+      dispatch();
+    });
+    const fail = (err: Error) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.delete(worker);
+      pool = pool.filter(w => w !== worker);
+      entry?.reject(err);
+      dispatch();
+    };
+    worker.on('error', fail);
+    worker.on('exit', () => fail(new Error('emission worker exited')));
+  }
+
+  return {
+    emit(job: EmissionJob): Promise<EmissionResult> {
+      return new Promise((resolve, reject) => {
+        pending.push({ job, resolve, reject });
+        dispatch();
+      });
+    },
+    async close() {
+      await Promise.allSettled(pool.map(w => w.terminate()));
+    },
+  };
+}
+
 async function processDataset(dir, register, opts) {
   const files = fs.readdirSync(dir).filter(f => f.endsWith('.yaml')).sort((a, b) => naturalSort(a.replace('.yaml', ''), b.replace('.yaml', '')));
 
   console.log(`Processing ${register}: ${files.length} files`);
 
   const conceptsDir = path.join(DATA, register, 'concepts');
-  // Clean previous output to avoid stale files accumulating across runs
-  if (fs.existsSync(conceptsDir)) {
-    for (const f of fs.readdirSync(conceptsDir)) fs.unlinkSync(path.join(conceptsDir, f));
-  }
+  fs.mkdirSync(conceptsDir, { recursive: true });
+
+  // Incremental emission cache: per-source-file content hash → last
+  // emitted termid. Unchanged sources skip per-concept file emission;
+  // orphans (source removed, renamed termid, older cache version) are
+  // swept below. Bump EMISSION_CACHE_VERSION whenever the per-concept
+  // emission logic changes its output shape.
+  const EMISSION_CACHE_VERSION = 1;
+  const EMISSION_EXTENSIONS = ['json', 'ttl', 'jsonld', 'tbx', 'yaml'];
+  const cacheDir = path.join(ROOT, '.gcb-cache');
+  const cachePath = path.join(cacheDir, `${register}.json`);
+  const emissionCache: { version: number; files: Record<string, { hash: string; termid: string }> } = {
+    version: EMISSION_CACHE_VERSION,
+    files: {},
+  };
+  try {
+    const loaded = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
+    if (loaded?.version === EMISSION_CACHE_VERSION && loaded.files) emissionCache.files = loaded.files;
+  } catch {}
+
   const concepts = [];
   const conceptInstances: Concept[] = [];
   const langTermCounts = {};
   const langDefCounts = {};
   const availableFormats = ['ttl', 'jsonld', 'yaml', 'tbx', 'csv'];
+  const producedOutputs = new Set<string>();
+  const emissionCacheFiles: Record<string, { hash: string; termid: string }> = {};
 
   // Pre-scan: build designation → concept ID lookup for same-dataset designation refs
   const designationLookup = new Map();
@@ -740,7 +853,31 @@ async function processDataset(dir, register, opts) {
     },
   };
 
+  const standards = new Map<string, StandardsIndex[string]>();
+
   const STATS_PROCESSORS = [
+    function collectStandards(cy, termid) {
+      // Harmonized metadata uses underscore-prefixed fields; object-valued
+      // language entries include every localization, even if not configured.
+      const concept = {
+        sources: cy._sources || [],
+        languages: Object.keys(cy).filter(lang =>
+          !lang.startsWith('_') && cy[lang] && typeof cy[lang] === 'object' && !Array.isArray(cy[lang])),
+        localization: (lang: string) => cy[lang],
+      };
+      const sources = [
+        ...concept.sources,
+        ...concept.languages.flatMap(lang => concept.localization(lang).sources || []),
+      ];
+      for (const key of collectStandardKeys(concept)) {
+        if (!standards.has(key)) {
+          const source = sources.find(source =>
+            source.type === 'authoritative' && standardKey(source.origin?.ref) === key);
+          standards.set(key, { label: standardLabel(source.origin.ref)!, conceptIds: [] });
+        }
+        standards.get(key)!.conceptIds.push(termid);
+      }
+    },
     function collectSources(cy, termid, s) {
       const refs = new Set();
       const allSources = [
@@ -803,46 +940,54 @@ async function processDataset(dir, register, opts) {
     },
   ];
 
+  const loopStart = Date.now();
+  const emissionJobs: EmissionJob[] = [];
+
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
+    if (i > 0 && i % 250 === 0) console.log(`  ${i}/${files.length} concepts`);
     try {
-      const conceptYaml = loadConceptFile(path.join(dir, file));
+      const filePath = path.join(dir, file);
+      const sourceText = fs.readFileSync(filePath, 'utf8');
+      const sourceHash = createHash('sha256').update(sourceText).digest('hex');
+
+      const conceptYaml = loadConceptFile(filePath);
       if (!conceptYaml || !conceptYaml.termid) continue;
 
       const termid = String(conceptYaml.termid);
-      const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
-      writeJson(path.join(conceptsDir, `${termid}.json`), jsonld);
-      // Canonical interchange exports — routed through glossarist's
-      // output API (one model walk; predicates generated from the
-      // concept-model context, wire shape from schemas/v3). The hand-rolled
-      // gl:-shaped emitters in scripts/lib/concept-formats.ts are retired.
-      // Note: TBX now includes <term> entries — the retired emitter
-      // computed termEntries but never wrote them into the document.
+
+      const cached = emissionCache.files[file];
+      const unchanged = cached?.hash === sourceHash
+        && EMISSION_EXTENSIONS.every(ext => fs.existsSync(path.join(conceptsDir, `${termid}.${ext}`)));
+
+      // Canonical model instance — required for the aggregate exports on
+      // every run, independent of per-file emission caching.
       let conceptInstance: Concept | null = null;
       try {
-        conceptInstance = parseConceptYaml(fs.readFileSync(path.join(dir, file), 'utf8'), termid);
+        conceptInstance = parseConceptYaml(sourceText, termid);
       } catch (e) {
         console.warn(`  ${termid}: canonical parse failed (${e.message}); skipping glossarist exports`);
       }
-      if (conceptInstance) {
-        conceptInstances.push(conceptInstance);
-        fs.writeFileSync(
-          path.join(conceptsDir, `${termid}.ttl`),
-          conceptToTurtle(conceptInstance, { uriBase: refMaps.uriBase, registerId: register }),
-        );
-        const conceptJsonLd = await emitOutputSet([conceptInstance], {
-          registerId: register,
-          uriBase: refMaps.uriBase,
-          formats: ['jsonld'],
-        });
-        fs.writeFileSync(path.join(conceptsDir, `${termid}.jsonld`), conceptJsonLd.jsonld!);
-        const tbxContent = conceptsToTbx([conceptInstance], { registerId: register });
-        if (tbxContent) {
-          fs.writeFileSync(path.join(conceptsDir, `${termid}.tbx`), tbxContent);
+      if (conceptInstance) conceptInstances.push(conceptInstance);
+
+      if (!unchanged) {
+        const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
+        writeJson(path.join(conceptsDir, `${termid}.json`), jsonld);
+        fs.copyFileSync(filePath, path.join(conceptsDir, `${termid}.yaml`));
+        if (conceptInstance) {
+          emissionJobs.push({ file, termid, sourceText, sourceHash, conceptInstance });
         }
+      } else {
+        emissionCacheFiles[file] = { hash: sourceHash, termid };
       }
 
-      fs.copyFileSync(path.join(dir, file), path.join(conceptsDir, `${termid}.yaml`));
+      producedOutputs.add(`${termid}.json`);
+      producedOutputs.add(`${termid}.yaml`);
+      if (conceptInstance) {
+        producedOutputs.add(`${termid}.ttl`);
+        producedOutputs.add(`${termid}.jsonld`);
+        producedOutputs.add(`${termid}.tbx`);
+      }
 
       concepts.push({
         id: termid,
@@ -859,17 +1004,64 @@ async function processDataset(dir, register, opts) {
     }
   }
 
+  // Glossarist-native per-concept exports (ttl/jsonld/tbx), parallelized
+  // on worker threads; falls back to inline emission when workers are
+  // unavailable (GCB_WORKERS=0 or restricted environments).
+  if (emissionJobs.length > 0) {
+    const jobsByTermid = new Map(emissionJobs.map(j => [j.termid, j]));
+    const pool = createEmissionPool(register, refMaps.uriBase);
+    const emissionStart = Date.now();
+    try {
+      const results = await Promise.all(emissionJobs.map(job => {
+        if (!pool) return inlineEmission(job, register, refMaps.uriBase);
+        return pool.emit(job).catch(() => inlineEmission(job, register, refMaps.uriBase));
+      }));
+      for (const result of results) {
+        if (result.error) {
+          console.warn(`  ${result.termid}: canonical parse failed (${result.error}); skipping glossarist exports`);
+          const job = jobsByTermid.get(result.termid);
+          if (job) delete emissionCacheFiles[job.file];
+          for (const ext of ['ttl', 'jsonld', 'tbx']) producedOutputs.delete(`${result.termid}.${ext}`);
+          continue;
+        }
+        fs.writeFileSync(path.join(conceptsDir, `${result.termid}.ttl`), result.ttl);
+        fs.writeFileSync(path.join(conceptsDir, `${result.termid}.jsonld`), result.jsonld);
+        if (result.tbx) fs.writeFileSync(path.join(conceptsDir, `${result.termid}.tbx`), result.tbx);
+        const job = jobsByTermid.get(result.termid);
+        if (job) emissionCacheFiles[job.file] = { hash: job.sourceHash, termid: job.termid };
+      }
+      console.log(`  Interchange exports: ${((Date.now() - emissionStart) / 1000).toFixed(1)}s (${emissionJobs.length} concepts, ${pool ? 'parallel' : 'inline'})`);
+    } finally {
+      if (pool) await pool.close();
+    }
+  }
+
+  // Sweep per-concept outputs this run no longer produces (removed or
+  // renamed sources) — replaces the wholesale pre-loop clean, which the
+  // incremental cache needs gone.
+  for (const entry of fs.readdirSync(conceptsDir)) {
+    if (!producedOutputs.has(entry)) fs.unlinkSync(path.join(conceptsDir, entry));
+  }
+  fs.mkdirSync(cacheDir, { recursive: true });
+  fs.writeFileSync(cachePath, JSON.stringify({ version: EMISSION_CACHE_VERSION, files: emissionCacheFiles }));
+
+  console.log(`  Per-concept emissions: ${((Date.now() - loopStart) / 1000).toFixed(1)}s`);
+
   // Aggregate distributions: single-file exports of the whole register,
   // emitted by glossarist's output-set API from the canonical model walk.
   // The CSV uri column is a link a human opens — it carries the SPA route
   // shape on the deployment's configured host (siteOrigin), not the
-  // RDF-canonical IRI.
+  // RDF-canonical IRI. Only the two formats consumed downstream are
+  // requested — the output-set API computes all six by default.
+  const aggregateStart = Date.now();
   const aggregate = await emitOutputSet(conceptInstances, {
     registerId: register,
     uriBase: refMaps.uriBase,
+    formats: ['csv', 'jsonld'],
     languageOrder: opts.languageOrder || opts.languages,
     conceptUriPrefix: buildConceptUriPrefix(refMaps.siteOrigin, register),
   });
+  console.log(`  Aggregate csv+jsonld: ${((Date.now() - aggregateStart) / 1000).toFixed(1)}s`);
   fs.writeFileSync(path.join(DATA, register, `${register}.csv`), aggregate.csv!);
   fs.writeFileSync(path.join(DATA, register, `${register}.jsonld`), aggregate.jsonld!);
 
@@ -879,12 +1071,17 @@ async function processDataset(dir, register, opts) {
     const chunk = concepts.slice(i, i + CHUNK_SIZE);
     const chunkIndex = Math.floor(i / CHUNK_SIZE);
     const chunkFile = `index-${String(chunkIndex).padStart(4, '0')}.json`;
-    writeJson(path.join(DATA, register, 'chunks', chunkFile), {
+    const chunkData = {
       registerId: register,
       chunkIndex,
       concepts: chunk,
-    });
-    chunks.push({ file: chunkFile, count: chunk.length });
+    };
+    const chunkContent = JSON.stringify(chunkData);
+    const chunkPath = path.join(DATA, register, 'chunks', chunkFile);
+    fs.mkdirSync(path.dirname(chunkPath), { recursive: true });
+    fs.writeFileSync(chunkPath, chunkContent);
+    const hash = createHash('sha256').update(chunkContent).digest('hex').slice(0, 12);
+    chunks.push({ file: chunkFile, count: chunk.length, hash });
   }
 
   const summary = concepts.map(c => ({
@@ -993,6 +1190,8 @@ async function processDataset(dir, register, opts) {
     .sort((a, b) => b.conceptCount - a.conceptCount);
   const totalRelationships = Object.values(stats.relTypeCounts).reduce((a: number, b: number) => a + b, 0);
 
+  writeJson(path.join(DATA, register, 'standards.json'), Object.fromEntries(standards));
+
   writeJson(path.join(DATA, register, 'stats.json'), {
     sourceCount: sourceStats.length,
     sources: sourceStats,
@@ -1040,7 +1239,9 @@ async function processDataset(dir, register, opts) {
   // Dataset-level RDF (WS J2/J5): dcat:Dataset + skos:ConceptScheme + skos:Collection per section.
   // Runs before the manifest write so the ttl distribution is on disk when
   // bulkFormats is assembled.
+  const rdfStart = Date.now();
   await writeDatasetRdf(register, manifest, concepts, refMaps, opts);
+  console.log(`  Dataset RDF: ${((Date.now() - rdfStart) / 1000).toFixed(1)}s`);
   addBulkFormat(`${register}.ttl`);
   manifest.bulkFormats = opts.bulkExports?.length
     ? bulkFormats.filter(f => opts.bulkExports.includes(f.format))

package/scripts/lib/workers/emit-concept-worker.mjs

@@ -0,0 +1,31 @@
+// Worker thread: glossarist-native per-concept interchange emissions
+// (ttl / jsonld / tbx). Plain .mjs — ESM loader hooks like tsx do not
+// propagate into worker threads, so this file must stay dependency-free
+// of workspace TS; everything resolves from node_modules.
+//
+// Concept model instances are class objects and cannot cross the
+// worker boundary, so the worker re-parses the exact source text the
+// main thread read.
+import { parentPort, workerData } from 'node:worker_threads';
+import { parseConceptYaml, conceptToTurtle, emitOutputSet, conceptsToTbx } from 'glossarist';
+
+const { registerId, uriBase } = workerData;
+
+parentPort.on('message', async ({ termid, sourceText }) => {
+  try {
+    const concept = parseConceptYaml(sourceText, termid);
+    const outputs = await emitOutputSet([concept], {
+      registerId,
+      uriBase,
+      formats: ['jsonld'],
+    });
+    parentPort.postMessage({
+      termid,
+      ttl: conceptToTurtle(concept, { uriBase, registerId }),
+      jsonld: outputs.jsonld,
+      tbx: conceptsToTbx([concept], { registerId }) || null,
+    });
+  } catch (e) {
+    parentPort.postMessage({ termid, error: String((e && e.message) || e) });
+  }
+});

package/src/__tests__/app-footer.test.ts

@@ -35,4 +35,40 @@ describe('AppFooter', () => {
     expect(link).toBeDefined();
     expect(link!.attributes('href')).toBe('https://glossarist.org');
   });
+
+  it('omits hosted by when branding.hostedBy is absent', () => {
+    const wrapper = mountFooter();
+    expect(wrapper.text()).not.toContain('Hosted by');
+  });
+
+  it('renders hosted by with logo when branding.hostedBy is configured', async () => {
+    const inline = document.createElement('script');
+    inline.id = 'site-config-json';
+    inline.type = 'application/json';
+    inline.textContent = JSON.stringify({
+      title: 'Test Site',
+      branding: {
+        hostedBy: {
+          name: 'Enosema Foundation',
+          url: 'https://www.enosema.org',
+          logo: 'https://www.enosema.org/logos/logo-text-white.svg',
+        },
+      },
+    });
+    document.head.appendChild(inline);
+
+    const { useSiteConfig } = await import('../config/use-site-config');
+    await useSiteConfig().loadConfig();
+
+    const wrapper = mountFooter();
+    expect(wrapper.text()).toContain('Hosted by');
+    const link = wrapper.findAll('a').find(a => a.attributes('href') === 'https://www.enosema.org');
+    expect(link).toBeDefined();
+    expect(link!.attributes('href')).toBe('https://www.enosema.org');
+    const img = link!.find('img');
+    expect(img.attributes('src')).toBe('https://www.enosema.org/logos/logo-text-white.svg');
+    expect(img.attributes('alt')).toBe('Enosema Foundation');
+    inline.remove();
+  });
+
 });

package/src/__tests__/app-header.test.ts

@@ -3,6 +3,8 @@ import { mount, flushPromises } from '@vue/test-utils';
 import { createPinia, setActivePinia } from 'pinia';
 import { createRouter, createMemoryHistory } from 'vue-router';
 import AppHeader from '../components/AppHeader.vue';
+import { useUiStore } from '../stores/ui';
+import { locale } from '../i18n';
 import { useVocabularyStore } from '../stores/vocabulary';
 import type { Manifest } from '../adapters/types';
 
@@ -45,6 +47,7 @@ describe('AppHeader', () => {
   let router: Awaited<ReturnType<typeof createTestRouter>>;
 
   beforeEach(async () => {
+    locale.value = 'eng';
     pinia = createPinia();
     setActivePinia(pinia);
     router = await createTestRouter();
@@ -66,10 +69,28 @@ describe('AppHeader', () => {
     expect(wrapper.text()).toContain('Glossarist');
   });
 
-  it('renders search input', () => {
+  it('renders quick search on the home route', () => {
     const wrapper = mountHeader();
-    const input = wrapper.find('input[aria-label="Search concepts"]');
+    const input = wrapper.find('input[aria-label="Quick search"]');
     expect(input.exists()).toBe(true);
+    expect(input.attributes('placeholder')).toBe('Quick search...');
+  });
+
+  it('hides quick search on the search route and restores it elsewhere', async () => {
+    await router.push('/search?q=road');
+    const wrapper = mountHeader();
+    expect(wrapper.find('form').exists()).toBe(false);
+
+    await router.push('/');
+    expect(wrapper.find('form').exists()).toBe(true);
+  });
+
+  it('labels quick search in French', () => {
+    locale.value = 'fra';
+    const wrapper = mountHeader();
+    const input = wrapper.get('input');
+    expect(input.attributes('aria-label')).toBe('Recherche rapide');
+    expect(input.attributes('placeholder')).toBe('Recherche rapide…');
   });
 
   it('renders mobile hamburger button', () => {
@@ -89,13 +110,19 @@ describe('AppHeader', () => {
 
   it('navigates to search on form submit with query', async () => {
     const wrapper = mountHeader();
-    const input = wrapper.find('input[aria-label="Search concepts"]');
-    await input.setValue('road');
+    const input = wrapper.find('input[aria-label="Quick search"]');
+    await input.setValue('  road & rail  ');
     const form = wrapper.find('form');
     await form.trigger('submit');
     await flushPromises();
     expect(router.currentRoute.value.name).toBe('search');
-    expect(router.currentRoute.value.query.q).toBe('road');
+    expect(router.currentRoute.value.query).toEqual({ q: 'road & rail' });
+    expect(router.currentRoute.value.path).toBe('/search');
+    expect(useUiStore().searchQuery).toBe('');
+    expect(wrapper.find('form').exists()).toBe(false);
+
+    await router.push('/');
+    expect(wrapper.get('input').element.value).toBe('');
   });
 
   it('does not navigate on empty search', async () => {