@glossarist/concept-browser 0.7.140 → 0.7.141

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glossarist/concept-browser",
-  "version": "0.7.140",
+  "version": "0.7.141",
   "description": "Vue SPA for browsing Glossarist terminology datasets with cross-reference resolution, graph visualization, and multi-language support",
   "type": "module",
   "bin": {

package/scripts/build-edges.js

@@ -3,28 +3,6 @@ function buildConceptUri(uriBase, registerId, conceptId) {
   return `${uriBase}/dataset/${registerId}/concept/${conceptId}`;
 }
 
-// scripts/extract-source-refs.js
-function extractSourceRefs(concept, registerId) {
-  const refs = /* @__PURE__ */ new Set();
-  for (const src of concept["gl:source"] || []) {
-    const origin = src["gl:origin"];
-    if (origin) {
-      const ref = origin["gl:ref"];
-      if (ref?.["gl:source"]) refs.add(ref["gl:source"]);
-    }
-  }
-  for (const lc of Object.values(concept["gl:localizedConcept"] || {})) {
-    for (const src of lc["gl:source"] || []) {
-      const origin = src["gl:origin"];
-      if (origin) {
-        const ref = origin["gl:ref"];
-        if (ref?.["gl:source"]) refs.add(ref["gl:source"]);
-      }
-    }
-  }
-  return [...refs].map((source) => ({ source, registerId }));
-}
-
 // scripts/build-edges.ts
 import { readFileSync, writeFileSync, readdirSync, existsSync, realpathSync } from "fs";
 import { join, dirname } from "path";
@@ -285,7 +263,7 @@ function buildEdgesForDataset(datasetDir, registerId, uriBase, urnMap, manifest)
       warnBinaryPartitiveRedundancy(edges, relations, registerId);
       allEdges.push(...edges);
       allPartitiveRelations.push(...relations);
-      allSourceRefs.push(...extractSourceRefs(data, registerId));
+      allSourceRefs.push(...(void 0)(data, registerId));
       for (const edge of edges) {
         if (edge.type === "domain" || edge.type === "section") {
           domainConceptCount.set(edge.target, (domainConceptCount.get(edge.target) || 0) + 1);

package/scripts/generate-data.js

@@ -1,6 +1,8 @@
 // scripts/generate-data.ts
 import fs2 from "fs";
+import os from "os";
 import path3 from "path";
+import { Worker as WorkerThread } from "node:worker_threads";
 import { createHash as createHash2 } from "node:crypto";
 import yaml4 from "js-yaml";
 import {
@@ -1252,18 +1254,104 @@ function getPrimaryDesignation(conceptYaml) {
   }
   return descs;
 }
+async function inlineEmission(job, register, uriBase) {
+  try {
+    const outputs = await emitOutputSet([job.conceptInstance], {
+      registerId: register,
+      uriBase,
+      formats: ["jsonld"]
+    });
+    return {
+      termid: job.termid,
+      ttl: conceptToTurtle(job.conceptInstance, { uriBase, registerId: register }),
+      jsonld: outputs.jsonld,
+      tbx: conceptsToTbx([job.conceptInstance], { registerId: register }) || null
+    };
+  } catch (e) {
+    return { termid: job.termid, error: String(e && e.message || e) };
+  }
+}
+function createEmissionPool(register, uriBase) {
+  const configured = Number(process.env.GCB_WORKERS);
+  if (Number.isFinite(configured) && configured <= 0) return null;
+  const size = Number.isFinite(configured) ? configured : Math.max(1, Math.min(4, (os.availableParallelism ? os.availableParallelism() : os.cpus().length) - 1));
+  const workerPath = path3.join(__dirname3, "lib", "workers", "emit-concept-worker.mjs");
+  let pool;
+  try {
+    pool = Array.from({ length: size }, () => new WorkerThread(workerPath, { workerData: { registerId: register, uriBase } }));
+  } catch {
+    return null;
+  }
+  const active = /* @__PURE__ */ new Map();
+  const pending = [];
+  const idle = /* @__PURE__ */ new Set();
+  function dispatch() {
+    while (pending.length > 0) {
+      const worker = idle.values().next();
+      if (worker.done) return;
+      idle.delete(worker.value);
+      const entry = pending.shift();
+      active.set(worker.value, entry);
+      worker.value.postMessage({ termid: entry.job.termid, sourceText: entry.job.sourceText });
+    }
+  }
+  for (const worker of pool) {
+    idle.add(worker);
+    worker.on("message", (msg) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.add(worker);
+      entry?.resolve(msg);
+      dispatch();
+    });
+    const fail = (err) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.delete(worker);
+      pool = pool.filter((w) => w !== worker);
+      entry?.reject(err);
+      dispatch();
+    };
+    worker.on("error", fail);
+    worker.on("exit", () => fail(new Error("emission worker exited")));
+  }
+  return {
+    emit(job) {
+      return new Promise((resolve2, reject) => {
+        pending.push({ job, resolve: resolve2, reject });
+        dispatch();
+      });
+    },
+    async close() {
+      await Promise.allSettled(pool.map((w) => w.terminate()));
+    }
+  };
+}
 async function processDataset(dir, register, opts) {
   const files = fs2.readdirSync(dir).filter((f) => f.endsWith(".yaml")).sort((a, b) => naturalSort(a.replace(".yaml", ""), b.replace(".yaml", "")));
   console.log(`Processing ${register}: ${files.length} files`);
   const conceptsDir = path3.join(DATA, register, "concepts");
-  if (fs2.existsSync(conceptsDir)) {
-    for (const f of fs2.readdirSync(conceptsDir)) fs2.unlinkSync(path3.join(conceptsDir, f));
+  fs2.mkdirSync(conceptsDir, { recursive: true });
+  const EMISSION_CACHE_VERSION = 1;
+  const EMISSION_EXTENSIONS = ["json", "ttl", "jsonld", "tbx", "yaml"];
+  const cacheDir = path3.join(ROOT, ".gcb-cache");
+  const cachePath = path3.join(cacheDir, `${register}.json`);
+  const emissionCache = {
+    version: EMISSION_CACHE_VERSION,
+    files: {}
+  };
+  try {
+    const loaded = JSON.parse(fs2.readFileSync(cachePath, "utf8"));
+    if (loaded?.version === EMISSION_CACHE_VERSION && loaded.files) emissionCache.files = loaded.files;
+  } catch {
   }
   const concepts = [];
   const conceptInstances = [];
   const langTermCounts = {};
   const langDefCounts = {};
   const availableFormats = ["ttl", "jsonld", "yaml", "tbx", "csv"];
+  const producedOutputs = /* @__PURE__ */ new Set();
+  const emissionCacheFiles = {};
   const designationLookup = /* @__PURE__ */ new Map();
   for (const file of files) {
     try {
@@ -1376,38 +1464,44 @@ async function processDataset(dir, register, opts) {
       }
     }
   ];
+  const loopStart = Date.now();
+  const emissionJobs = [];
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
+    if (i > 0 && i % 250 === 0) console.log(`  ${i}/${files.length} concepts`);
     try {
-      const conceptYaml = loadConceptFile(path3.join(dir, file));
+      const filePath = path3.join(dir, file);
+      const sourceText = fs2.readFileSync(filePath, "utf8");
+      const sourceHash = createHash2("sha256").update(sourceText).digest("hex");
+      const conceptYaml = loadConceptFile(filePath);
       if (!conceptYaml || !conceptYaml.termid) continue;
       const termid = String(conceptYaml.termid);
-      const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
-      writeJson(path3.join(conceptsDir, `${termid}.json`), jsonld);
+      const cached = emissionCache.files[file];
+      const unchanged = cached?.hash === sourceHash && EMISSION_EXTENSIONS.every((ext) => fs2.existsSync(path3.join(conceptsDir, `${termid}.${ext}`)));
       let conceptInstance = null;
       try {
-        conceptInstance = parseConceptYaml(fs2.readFileSync(path3.join(dir, file), "utf8"), termid);
+        conceptInstance = parseConceptYaml(sourceText, termid);
       } catch (e) {
         console.warn(`  ${termid}: canonical parse failed (${e.message}); skipping glossarist exports`);
       }
-      if (conceptInstance) {
-        conceptInstances.push(conceptInstance);
-        fs2.writeFileSync(
-          path3.join(conceptsDir, `${termid}.ttl`),
-          conceptToTurtle(conceptInstance, { uriBase: refMaps.uriBase, registerId: register })
-        );
-        const conceptJsonLd = await emitOutputSet([conceptInstance], {
-          registerId: register,
-          uriBase: refMaps.uriBase,
-          formats: ["jsonld"]
-        });
-        fs2.writeFileSync(path3.join(conceptsDir, `${termid}.jsonld`), conceptJsonLd.jsonld);
-        const tbxContent = conceptsToTbx([conceptInstance], { registerId: register });
-        if (tbxContent) {
-          fs2.writeFileSync(path3.join(conceptsDir, `${termid}.tbx`), tbxContent);
+      if (conceptInstance) conceptInstances.push(conceptInstance);
+      if (!unchanged) {
+        const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
+        writeJson(path3.join(conceptsDir, `${termid}.json`), jsonld);
+        fs2.copyFileSync(filePath, path3.join(conceptsDir, `${termid}.yaml`));
+        if (conceptInstance) {
+          emissionJobs.push({ file, termid, sourceText, sourceHash, conceptInstance });
         }
+      } else {
+        emissionCacheFiles[file] = { hash: sourceHash, termid };
+      }
+      producedOutputs.add(`${termid}.json`);
+      producedOutputs.add(`${termid}.yaml`);
+      if (conceptInstance) {
+        producedOutputs.add(`${termid}.ttl`);
+        producedOutputs.add(`${termid}.jsonld`);
+        producedOutputs.add(`${termid}.tbx`);
       }
-      fs2.copyFileSync(path3.join(dir, file), path3.join(conceptsDir, `${termid}.yaml`));
       concepts.push({
         id: termid,
         designations: getPrimaryDesignation(conceptYaml),
@@ -1421,12 +1515,49 @@ async function processDataset(dir, register, opts) {
       console.warn(`  Skipping ${file}: ${e.message}`);
     }
   }
+  if (emissionJobs.length > 0) {
+    const jobsByTermid = new Map(emissionJobs.map((j) => [j.termid, j]));
+    const pool = createEmissionPool(register, refMaps.uriBase);
+    const emissionStart = Date.now();
+    try {
+      const results = await Promise.all(emissionJobs.map((job) => {
+        if (!pool) return inlineEmission(job, register, refMaps.uriBase);
+        return pool.emit(job).catch(() => inlineEmission(job, register, refMaps.uriBase));
+      }));
+      for (const result of results) {
+        if (result.error) {
+          console.warn(`  ${result.termid}: canonical parse failed (${result.error}); skipping glossarist exports`);
+          const job2 = jobsByTermid.get(result.termid);
+          if (job2) delete emissionCacheFiles[job2.file];
+          for (const ext of ["ttl", "jsonld", "tbx"]) producedOutputs.delete(`${result.termid}.${ext}`);
+          continue;
+        }
+        fs2.writeFileSync(path3.join(conceptsDir, `${result.termid}.ttl`), result.ttl);
+        fs2.writeFileSync(path3.join(conceptsDir, `${result.termid}.jsonld`), result.jsonld);
+        if (result.tbx) fs2.writeFileSync(path3.join(conceptsDir, `${result.termid}.tbx`), result.tbx);
+        const job = jobsByTermid.get(result.termid);
+        if (job) emissionCacheFiles[job.file] = { hash: job.sourceHash, termid: job.termid };
+      }
+      console.log(`  Interchange exports: ${((Date.now() - emissionStart) / 1e3).toFixed(1)}s (${emissionJobs.length} concepts, ${pool ? "parallel" : "inline"})`);
+    } finally {
+      if (pool) await pool.close();
+    }
+  }
+  for (const entry of fs2.readdirSync(conceptsDir)) {
+    if (!producedOutputs.has(entry)) fs2.unlinkSync(path3.join(conceptsDir, entry));
+  }
+  fs2.mkdirSync(cacheDir, { recursive: true });
+  fs2.writeFileSync(cachePath, JSON.stringify({ version: EMISSION_CACHE_VERSION, files: emissionCacheFiles }));
+  console.log(`  Per-concept emissions: ${((Date.now() - loopStart) / 1e3).toFixed(1)}s`);
+  const aggregateStart = Date.now();
   const aggregate = await emitOutputSet(conceptInstances, {
     registerId: register,
     uriBase: refMaps.uriBase,
+    formats: ["csv", "jsonld"],
     languageOrder: opts.languageOrder || opts.languages,
     conceptUriPrefix: buildConceptUriPrefix(refMaps.siteOrigin, register)
   });
+  console.log(`  Aggregate csv+jsonld: ${((Date.now() - aggregateStart) / 1e3).toFixed(1)}s`);
   fs2.writeFileSync(path3.join(DATA, register, `${register}.csv`), aggregate.csv);
   fs2.writeFileSync(path3.join(DATA, register, `${register}.jsonld`), aggregate.jsonld);
   const CHUNK_SIZE = 500;
@@ -1580,7 +1711,9 @@ async function processDataset(dir, register, opts) {
   if (opts.status) manifest.editionStatus = opts.status;
   if (opts.ordering) manifest.ordering = opts.ordering;
   if (opts.sections && opts.sections.length > 0) manifest.sections = opts.sections;
+  const rdfStart = Date.now();
   await writeDatasetRdf(register, manifest, concepts, refMaps, opts);
+  console.log(`  Dataset RDF: ${((Date.now() - rdfStart) / 1e3).toFixed(1)}s`);
   addBulkFormat(`${register}.ttl`);
   manifest.bulkFormats = opts.bulkExports?.length ? bulkFormats.filter((f) => opts.bulkExports.includes(f.format)) : bulkFormats;
   writeJson(path3.join(DATA, register, "manifest.json"), manifest);

package/scripts/generate-data.ts

@@ -1,5 +1,7 @@
 import fs from 'fs';
+import os from 'os';
 import path from 'path';
+import { Worker as WorkerThread } from 'node:worker_threads';
 import { createHash } from 'node:crypto';
 import yaml from 'js-yaml';
 import {
@@ -690,21 +692,129 @@ function getPrimaryDesignation(conceptYaml) {
   return descs;
 }
 
+// Worker pool for glossarist-native per-concept emissions (ttl/jsonld/tbx).
+// Set GCB_WORKERS=0 to force inline emission (same outputs, one thread).
+type EmissionJob = { file: string; termid: string; sourceText: string; sourceHash: string; conceptInstance: Concept };
+type EmissionResult = { termid: string; ttl?: string; jsonld?: string; tbx?: string; error?: string };
+
+async function inlineEmission(job: EmissionJob, register: string, uriBase: string): Promise<EmissionResult> {
+  try {
+    const outputs = await emitOutputSet([job.conceptInstance], {
+      registerId: register,
+      uriBase,
+      formats: ['jsonld'],
+    });
+    return {
+      termid: job.termid,
+      ttl: conceptToTurtle(job.conceptInstance, { uriBase, registerId: register }),
+      jsonld: outputs.jsonld,
+      tbx: conceptsToTbx([job.conceptInstance], { registerId: register }) || null,
+    };
+  } catch (e) {
+    return { termid: job.termid, error: String((e && e.message) || e) };
+  }
+}
+
+function createEmissionPool(register: string, uriBase: string): {
+  emit(job: EmissionJob): Promise<EmissionResult>;
+  close(): Promise<void>;
+} | null {
+  const configured = Number(process.env.GCB_WORKERS);
+  if (Number.isFinite(configured) && configured <= 0) return null;
+  const size = Number.isFinite(configured)
+    ? configured
+    : Math.max(1, Math.min(4, (os.availableParallelism ? os.availableParallelism() : os.cpus().length) - 1));
+  const workerPath = path.join(__dirname, 'lib', 'workers', 'emit-concept-worker.mjs');
+
+  let pool: WorkerThread[];
+  try {
+    pool = Array.from({ length: size }, () => new WorkerThread(workerPath, { workerData: { registerId: register, uriBase } }));
+  } catch {
+    return null;
+  }
+
+  const active = new Map<WorkerThread, { resolve: (r: EmissionResult) => void; reject: (e: Error) => void }>();
+  const pending: { job: EmissionJob; resolve: (r: EmissionResult) => void; reject: (e: Error) => void }[] = [];
+  const idle = new Set<WorkerThread>();
+
+  function dispatch() {
+    while (pending.length > 0) {
+      const worker = idle.values().next();
+      if (worker.done) return;
+      idle.delete(worker.value);
+      const entry = pending.shift()!;
+      active.set(worker.value, entry);
+      worker.value.postMessage({ termid: entry.job.termid, sourceText: entry.job.sourceText });
+    }
+  }
+
+  for (const worker of pool) {
+    idle.add(worker);
+    worker.on('message', (msg: EmissionResult) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.add(worker);
+      entry?.resolve(msg);
+      dispatch();
+    });
+    const fail = (err: Error) => {
+      const entry = active.get(worker);
+      active.delete(worker);
+      idle.delete(worker);
+      pool = pool.filter(w => w !== worker);
+      entry?.reject(err);
+      dispatch();
+    };
+    worker.on('error', fail);
+    worker.on('exit', () => fail(new Error('emission worker exited')));
+  }
+
+  return {
+    emit(job: EmissionJob): Promise<EmissionResult> {
+      return new Promise((resolve, reject) => {
+        pending.push({ job, resolve, reject });
+        dispatch();
+      });
+    },
+    async close() {
+      await Promise.allSettled(pool.map(w => w.terminate()));
+    },
+  };
+}
+
 async function processDataset(dir, register, opts) {
   const files = fs.readdirSync(dir).filter(f => f.endsWith('.yaml')).sort((a, b) => naturalSort(a.replace('.yaml', ''), b.replace('.yaml', '')));
 
   console.log(`Processing ${register}: ${files.length} files`);
 
   const conceptsDir = path.join(DATA, register, 'concepts');
-  // Clean previous output to avoid stale files accumulating across runs
-  if (fs.existsSync(conceptsDir)) {
-    for (const f of fs.readdirSync(conceptsDir)) fs.unlinkSync(path.join(conceptsDir, f));
-  }
+  fs.mkdirSync(conceptsDir, { recursive: true });
+
+  // Incremental emission cache: per-source-file content hash → last
+  // emitted termid. Unchanged sources skip per-concept file emission;
+  // orphans (source removed, renamed termid, older cache version) are
+  // swept below. Bump EMISSION_CACHE_VERSION whenever the per-concept
+  // emission logic changes its output shape.
+  const EMISSION_CACHE_VERSION = 1;
+  const EMISSION_EXTENSIONS = ['json', 'ttl', 'jsonld', 'tbx', 'yaml'];
+  const cacheDir = path.join(ROOT, '.gcb-cache');
+  const cachePath = path.join(cacheDir, `${register}.json`);
+  const emissionCache: { version: number; files: Record<string, { hash: string; termid: string }> } = {
+    version: EMISSION_CACHE_VERSION,
+    files: {},
+  };
+  try {
+    const loaded = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
+    if (loaded?.version === EMISSION_CACHE_VERSION && loaded.files) emissionCache.files = loaded.files;
+  } catch {}
+
   const concepts = [];
   const conceptInstances: Concept[] = [];
   const langTermCounts = {};
   const langDefCounts = {};
   const availableFormats = ['ttl', 'jsonld', 'yaml', 'tbx', 'csv'];
+  const producedOutputs = new Set<string>();
+  const emissionCacheFiles: Record<string, { hash: string; termid: string }> = {};
 
   // Pre-scan: build designation → concept ID lookup for same-dataset designation refs
   const designationLookup = new Map();
@@ -830,46 +940,54 @@ async function processDataset(dir, register, opts) {
     },
   ];
 
+  const loopStart = Date.now();
+  const emissionJobs: EmissionJob[] = [];
+
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
+    if (i > 0 && i % 250 === 0) console.log(`  ${i}/${files.length} concepts`);
     try {
-      const conceptYaml = loadConceptFile(path.join(dir, file));
+      const filePath = path.join(dir, file);
+      const sourceText = fs.readFileSync(filePath, 'utf8');
+      const sourceHash = createHash('sha256').update(sourceText).digest('hex');
+
+      const conceptYaml = loadConceptFile(filePath);
       if (!conceptYaml || !conceptYaml.termid) continue;
 
       const termid = String(conceptYaml.termid);
-      const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
-      writeJson(path.join(conceptsDir, `${termid}.json`), jsonld);
-      // Canonical interchange exports — routed through glossarist's
-      // output API (one model walk; predicates generated from the
-      // concept-model context, wire shape from schemas/v3). The hand-rolled
-      // gl:-shaped emitters in scripts/lib/concept-formats.ts are retired.
-      // Note: TBX now includes <term> entries — the retired emitter
-      // computed termEntries but never wrote them into the document.
+
+      const cached = emissionCache.files[file];
+      const unchanged = cached?.hash === sourceHash
+        && EMISSION_EXTENSIONS.every(ext => fs.existsSync(path.join(conceptsDir, `${termid}.${ext}`)));
+
+      // Canonical model instance — required for the aggregate exports on
+      // every run, independent of per-file emission caching.
       let conceptInstance: Concept | null = null;
       try {
-        conceptInstance = parseConceptYaml(fs.readFileSync(path.join(dir, file), 'utf8'), termid);
+        conceptInstance = parseConceptYaml(sourceText, termid);
       } catch (e) {
         console.warn(`  ${termid}: canonical parse failed (${e.message}); skipping glossarist exports`);
       }
-      if (conceptInstance) {
-        conceptInstances.push(conceptInstance);
-        fs.writeFileSync(
-          path.join(conceptsDir, `${termid}.ttl`),
-          conceptToTurtle(conceptInstance, { uriBase: refMaps.uriBase, registerId: register }),
-        );
-        const conceptJsonLd = await emitOutputSet([conceptInstance], {
-          registerId: register,
-          uriBase: refMaps.uriBase,
-          formats: ['jsonld'],
-        });
-        fs.writeFileSync(path.join(conceptsDir, `${termid}.jsonld`), conceptJsonLd.jsonld!);
-        const tbxContent = conceptsToTbx([conceptInstance], { registerId: register });
-        if (tbxContent) {
-          fs.writeFileSync(path.join(conceptsDir, `${termid}.tbx`), tbxContent);
+      if (conceptInstance) conceptInstances.push(conceptInstance);
+
+      if (!unchanged) {
+        const jsonld = yamlToJsonLd(conceptYaml, register, dsRefMaps);
+        writeJson(path.join(conceptsDir, `${termid}.json`), jsonld);
+        fs.copyFileSync(filePath, path.join(conceptsDir, `${termid}.yaml`));
+        if (conceptInstance) {
+          emissionJobs.push({ file, termid, sourceText, sourceHash, conceptInstance });
         }
+      } else {
+        emissionCacheFiles[file] = { hash: sourceHash, termid };
       }
 
-      fs.copyFileSync(path.join(dir, file), path.join(conceptsDir, `${termid}.yaml`));
+      producedOutputs.add(`${termid}.json`);
+      producedOutputs.add(`${termid}.yaml`);
+      if (conceptInstance) {
+        producedOutputs.add(`${termid}.ttl`);
+        producedOutputs.add(`${termid}.jsonld`);
+        producedOutputs.add(`${termid}.tbx`);
+      }
 
       concepts.push({
         id: termid,
@@ -886,17 +1004,64 @@ async function processDataset(dir, register, opts) {
     }
   }
 
+  // Glossarist-native per-concept exports (ttl/jsonld/tbx), parallelized
+  // on worker threads; falls back to inline emission when workers are
+  // unavailable (GCB_WORKERS=0 or restricted environments).
+  if (emissionJobs.length > 0) {
+    const jobsByTermid = new Map(emissionJobs.map(j => [j.termid, j]));
+    const pool = createEmissionPool(register, refMaps.uriBase);
+    const emissionStart = Date.now();
+    try {
+      const results = await Promise.all(emissionJobs.map(job => {
+        if (!pool) return inlineEmission(job, register, refMaps.uriBase);
+        return pool.emit(job).catch(() => inlineEmission(job, register, refMaps.uriBase));
+      }));
+      for (const result of results) {
+        if (result.error) {
+          console.warn(`  ${result.termid}: canonical parse failed (${result.error}); skipping glossarist exports`);
+          const job = jobsByTermid.get(result.termid);
+          if (job) delete emissionCacheFiles[job.file];
+          for (const ext of ['ttl', 'jsonld', 'tbx']) producedOutputs.delete(`${result.termid}.${ext}`);
+          continue;
+        }
+        fs.writeFileSync(path.join(conceptsDir, `${result.termid}.ttl`), result.ttl);
+        fs.writeFileSync(path.join(conceptsDir, `${result.termid}.jsonld`), result.jsonld);
+        if (result.tbx) fs.writeFileSync(path.join(conceptsDir, `${result.termid}.tbx`), result.tbx);
+        const job = jobsByTermid.get(result.termid);
+        if (job) emissionCacheFiles[job.file] = { hash: job.sourceHash, termid: job.termid };
+      }
+      console.log(`  Interchange exports: ${((Date.now() - emissionStart) / 1000).toFixed(1)}s (${emissionJobs.length} concepts, ${pool ? 'parallel' : 'inline'})`);
+    } finally {
+      if (pool) await pool.close();
+    }
+  }
+
+  // Sweep per-concept outputs this run no longer produces (removed or
+  // renamed sources) — replaces the wholesale pre-loop clean, which the
+  // incremental cache needs gone.
+  for (const entry of fs.readdirSync(conceptsDir)) {
+    if (!producedOutputs.has(entry)) fs.unlinkSync(path.join(conceptsDir, entry));
+  }
+  fs.mkdirSync(cacheDir, { recursive: true });
+  fs.writeFileSync(cachePath, JSON.stringify({ version: EMISSION_CACHE_VERSION, files: emissionCacheFiles }));
+
+  console.log(`  Per-concept emissions: ${((Date.now() - loopStart) / 1000).toFixed(1)}s`);
+
   // Aggregate distributions: single-file exports of the whole register,
   // emitted by glossarist's output-set API from the canonical model walk.
   // The CSV uri column is a link a human opens — it carries the SPA route
   // shape on the deployment's configured host (siteOrigin), not the
-  // RDF-canonical IRI.
+  // RDF-canonical IRI. Only the two formats consumed downstream are
+  // requested — the output-set API computes all six by default.
+  const aggregateStart = Date.now();
   const aggregate = await emitOutputSet(conceptInstances, {
     registerId: register,
     uriBase: refMaps.uriBase,
+    formats: ['csv', 'jsonld'],
     languageOrder: opts.languageOrder || opts.languages,
     conceptUriPrefix: buildConceptUriPrefix(refMaps.siteOrigin, register),
   });
+  console.log(`  Aggregate csv+jsonld: ${((Date.now() - aggregateStart) / 1000).toFixed(1)}s`);
   fs.writeFileSync(path.join(DATA, register, `${register}.csv`), aggregate.csv!);
   fs.writeFileSync(path.join(DATA, register, `${register}.jsonld`), aggregate.jsonld!);
 
@@ -1074,7 +1239,9 @@ async function processDataset(dir, register, opts) {
   // Dataset-level RDF (WS J2/J5): dcat:Dataset + skos:ConceptScheme + skos:Collection per section.
   // Runs before the manifest write so the ttl distribution is on disk when
   // bulkFormats is assembled.
+  const rdfStart = Date.now();
   await writeDatasetRdf(register, manifest, concepts, refMaps, opts);
+  console.log(`  Dataset RDF: ${((Date.now() - rdfStart) / 1000).toFixed(1)}s`);
   addBulkFormat(`${register}.ttl`);
   manifest.bulkFormats = opts.bulkExports?.length
     ? bulkFormats.filter(f => opts.bulkExports.includes(f.format))

package/scripts/lib/workers/emit-concept-worker.mjs

@@ -0,0 +1,31 @@
+// Worker thread: glossarist-native per-concept interchange emissions
+// (ttl / jsonld / tbx). Plain .mjs — ESM loader hooks like tsx do not
+// propagate into worker threads, so this file must stay dependency-free
+// of workspace TS; everything resolves from node_modules.
+//
+// Concept model instances are class objects and cannot cross the
+// worker boundary, so the worker re-parses the exact source text the
+// main thread read.
+import { parentPort, workerData } from 'node:worker_threads';
+import { parseConceptYaml, conceptToTurtle, emitOutputSet, conceptsToTbx } from 'glossarist';
+
+const { registerId, uriBase } = workerData;
+
+parentPort.on('message', async ({ termid, sourceText }) => {
+  try {
+    const concept = parseConceptYaml(sourceText, termid);
+    const outputs = await emitOutputSet([concept], {
+      registerId,
+      uriBase,
+      formats: ['jsonld'],
+    });
+    parentPort.postMessage({
+      termid,
+      ttl: conceptToTurtle(concept, { uriBase, registerId }),
+      jsonld: outputs.jsonld,
+      tbx: conceptsToTbx([concept], { registerId }) || null,
+    });
+  } catch (e) {
+    parentPort.postMessage({ termid, error: String((e && e.message) || e) });
+  }
+});

package/src/__tests__/scripts/generate-data.test.ts

@@ -32,7 +32,11 @@ describe('generate-data chunk hashes', () => {
         cwd: root,
         env: { ...process.env, SITE_CONFIG: configPath },
         stdio: 'pipe',
-        timeout: 30_000,
+        // Three sequential full generations run in this test; each
+        // subprocess pays tsx startup, worker-pool spin-up, and aggregate
+        // emission. The suite runs tests in parallel, so leave generous
+        // headroom for CPU contention.
+        timeout: 120_000,
       });
       const indexes = ['index.json', 'index-meta.json'].map(file =>
         JSON.parse(readFileSync(join(outputDir, file), 'utf8')),
@@ -70,5 +74,5 @@ describe('generate-data chunk hashes', () => {
     } finally {
       rmSync(root, { recursive: true, force: true });
     }
-  }, 60_000);
+  }, 300_000);
 });