@glossarist/concept-browser 0.7.120 → 0.7.122

package/cli/index.ts

@@ -9,8 +9,10 @@
  *   edges      Build cross-reference edges from generated concept data
  *   build      Full pipeline: fetch + generate + edges + vite build
  *   site       Same as build (alias)
- *   normalize  NFC-normalize YAML files in .datasets/ (use --check for CI gate)
- *   doctor     Diagnose the local environment (deps, datasets, shapes, context)
+ *   normalize       NFC-normalize YAML files in .datasets/ (use --check for CI gate)
+ *   doctor          Diagnose the local environment (deps, datasets, shapes, context)
+ *   validate-yaml   Validate dataset YAML against the concept schema
+ *   validate-links  Validate that all generated cross-references resolve to existing concepts
  *
  * Options:
  *   --site <id>  Site config to use (looks for site-config.yml in CWD)
@@ -42,6 +44,7 @@ const commands: Record<string, CommandRunner> = {
   edges:        async () => (await import('../scripts/build-edges')).main(),
   about:        async () => (await import('../scripts/process-about-pages')).main(),
   'validate-yaml': async () => { await import('../scripts/validate-yaml'); },
+  'validate-links': async () => { await import('../scripts/validate-links'); },
 };
 
 function parseArgs(argv: readonly string[]): ParsedArgs {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glossarist/concept-browser",
-  "version": "0.7.120",
+  "version": "0.7.122",
   "description": "Vue SPA for browsing Glossarist terminology datasets with cross-reference resolution, graph visualization, and multi-language support",
   "type": "module",
   "bin": {
@@ -13,6 +13,7 @@
     "preview": "vite preview",
     "check:scripts": "tsx scripts/check-syntax.ts",
     "validate:yaml": "tsx scripts/validate-yaml.ts",
+    "validate:links": "tsx scripts/validate-links.ts",
     "validate:shacl": "tsx scripts/validate-shacl.ts",
     "pretest": "tsx scripts/check-syntax.ts",
     "prebuild": "tsx scripts/check-syntax.ts",
@@ -20,7 +21,7 @@
     "generate-data": "tsx scripts/generate-data.ts",
     "generate-ontology": "tsx scripts/generate-ontology-data.ts && tsx scripts/generate-ontology-schema.ts",
     "sync:model": "tsx scripts/sync-concept-model.ts",
-    "build:full": "npm run generate-ontology && npm run fetch-datasets && npm run generate-data && tsx scripts/build-edges.ts && npm run build",
+    "build:full": "npm run generate-ontology && npm run fetch-datasets && npm run generate-data && tsx scripts/build-edges.ts && npm run validate:links && npm run build",
     "build:site": "concept-browser --site build",
     "build:scripts": "node scripts/build-scripts.mjs",
     "prepublishOnly": "npm run build:scripts",

package/scripts/generate-data.js

@@ -1211,6 +1211,7 @@ function extractInlineRefs(localizedData, refMaps2, conceptSources = []) {
   }
   for (const m of fullText.matchAll(/\{\{([^{}]+?)\}\}/g)) {
     const body = m[1];
+    if (/^(bib|link|image):/i.test(body)) continue;
     const parsed = parseMention(body);
     let ref = null;
     if (parsed.kind === "cite-ref") {

package/scripts/generate-data.ts

@@ -375,9 +375,22 @@ function extractInlineRefs(localizedData, refMaps, conceptSources = []) {
     if (pattern) { refs.push(pattern); }
   }
 
-  // Double-brace mentions: dispatched by parseMention kind
+  // Double-brace mentions: dispatched by parseMention kind.
+  //
+  // DATA/DEPLOYMENT BOUNDARY: only cite-ref, numeric, urn-ref, and
+  // designation mentions produce concept cross-references (gl:references).
+  // Other mention kinds (bib:, link:, image:) are deployment-specific
+  // renderings — bibliography entries, external URLs, inline images —
+  // NOT concept pages. They must NOT appear in gl:references or be
+  // turned into concept URIs. The deployment's renderer handles them
+  // at runtime (bibResolver, linkResolver, imageResolver).
   for (const m of fullText.matchAll(/\{\{([^{}]+?)\}\}/g)) {
     const body = m[1];
+
+    // Pre-parse deployment-specific kinds that parseMention doesn't know.
+    // These are NEVER concept cross-references — skip them entirely.
+    if (/^(bib|link|image):/i.test(body)) continue;
+
     const parsed = parseMention(body);
 
     let ref = null;
@@ -399,6 +412,8 @@ function extractInlineRefs(localizedData, refMaps, conceptSources = []) {
       // {{designation,render term}} — same-dataset designation reference
       ref = handleDesignation(parsed, refMaps);
     } else {
+      // Unresolved mentions: only include if they look like concept refs
+      // (numeric IDs or known patterns). Never include bib:/link:/image:.
       ref = handleUnresolved(body, refMaps);
     }
     if (ref) refs.push(ref);

package/scripts/validate-links.js

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+
+// scripts/validate-links.ts
+import { readdirSync, readFileSync, statSync, existsSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { cwd } from "node:process";
+var DATA_DIR = process.argv[2] ?? join(cwd(), "public", "data");
+function walk(dir, out = []) {
+  for (const entry of readdirSync(dir)) {
+    const full = join(dir, entry);
+    const st = statSync(full);
+    if (st.isDirectory()) walk(full, out);
+    else if (entry.endsWith(".json")) out.push(full);
+  }
+  return out;
+}
+function localPathForUri(uri, dataDir) {
+  const match = uri.match(/\/([^/]+)\/concept\/([^/]+?)(?:\/|$)/);
+  if (!match) return { exists: false, expected: "" };
+  const [, datasetId, conceptId] = match;
+  const decoded = decodeURIComponent(conceptId);
+  const expected = join(dataDir, datasetId, "concepts", `${decoded}.json`);
+  return { exists: existsSync(expected), expected };
+}
+function checkFile(file, dataDir) {
+  const raw = readFileSync(file, "utf8");
+  const json = JSON.parse(raw);
+  const broken = [];
+  function check(target, field) {
+    if (!target) return;
+    if (!target.startsWith("http")) return;
+    const { exists, expected } = localPathForUri(target, dataDir);
+    if (!exists) {
+      broken.push({
+        file: file.replace(cwd() + "/", ""),
+        line: 0,
+        field,
+        target,
+        reason: expected ? `local target file missing: ${expected}` : "unparseable URI"
+      });
+    }
+  }
+  for (const r of json["gl:related"] ?? []) {
+    check(r["gl:target"], "gl:related[].gl:target");
+    check(r["@id"], "gl:related[].@id");
+  }
+  for (const lc of Object.values(json["gl:localizedConcept"] ?? {})) {
+    for (const ref of lc["gl:references"] ?? []) {
+      if (ref["@id"]) check(ref["@id"], "gl:references[].@id");
+    }
+  }
+  for (const pr of json["gl:partitiveRelations"] ?? []) {
+    check(pr["gl:comprehensive"]?.["@id"] ?? pr["gl:comprehensive"], "gl:partitiveRelations.gl:comprehensive");
+    for (const m of pr["gl:hasPartitive"] ?? []) {
+      check(m["gl:ref"]?.["@id"] ?? m["gl:ref"], "gl:partitiveRelations.gl:hasPartitive[].gl:ref");
+    }
+  }
+  for (const gr of json["gl:genericRelations"] ?? []) {
+    check(gr["gl:comprehensive"]?.["@id"] ?? gr["gl:comprehensive"], "gl:genericRelations.gl:comprehensive");
+    for (const m of gr["gl:hasMember"] ?? []) {
+      check(m["gl:ref"]?.["@id"] ?? m["gl:ref"], "gl:genericRelations.gl:hasMember[].gl:ref");
+    }
+  }
+  return broken;
+}
+function main() {
+  if (!existsSync(DATA_DIR)) {
+    console.error(`Data directory not found: ${DATA_DIR}
+  Run after generate-data has produced public/data/`);
+    process.exit(2);
+  }
+  const files = walk(DATA_DIR).filter((f) => f.includes("/concepts/") && f.endsWith(".json"));
+  const broken = [];
+  for (const file of files) {
+    broken.push(...checkFile(file, DATA_DIR));
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const unique = broken.filter((b) => {
+    const key = `${b.file}|${b.field}|${b.target}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+  if (unique.length === 0) {
+    const total = files.length;
+    console.log(`All ${total} concept files have resolvable internal links.`);
+    process.exit(0);
+  }
+  console.error(`Found ${unique.length} broken internal link(s) across ${files.length} concept files:
+`);
+  for (const b of unique) {
+    console.error(`  ${b.file}`);
+    console.error(`    field:   ${b.field}`);
+    console.error(`    target:  ${b.target}`);
+    console.error(`    reason:  ${b.reason}`);
+    console.error("");
+  }
+  console.error("Build FAILED. Fix the broken links above (correct YAML refs) and re-run.");
+  process.exit(1);
+}
+var isDirect = process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname);
+if (isDirect) main();
+export {
+  checkFile,
+  localPathForUri
+};

package/scripts/validate-links.ts

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+/**
+ * validate-links — post-generate link integrity check.
+ *
+ * Scans every generated concept JSON in public/data/ for links pointing
+ * to URIs that should exist locally (gl:related targets, gl:references,
+ * partitive/generic relation targets). Reports all broken links with
+ * file:line precision and fails the build if any are found.
+ *
+ * Usage:
+ *   npx tsx scripts/validate-links.ts [public/data]
+ *
+ * Default exit codes:
+ *   0  all links resolve
+ *   1  one or more broken links (build fails)
+ */
+
+import { readdirSync, readFileSync, statSync, existsSync } from 'node:fs';
+import { join, dirname, resolve } from 'node:path';
+import { cwd } from 'node:process';
+
+const DATA_DIR = process.argv[2] ?? join(cwd(), 'public', 'data');
+
+interface BrokenLink {
+  file: string;
+  line: number;
+  field: string;
+  target: string;
+  reason: string;
+}
+
+function walk(dir: string, out: string[] = []): string[] {
+  for (const entry of readdirSync(dir)) {
+    const full = join(dir, entry);
+    const st = statSync(full);
+    if (st.isDirectory()) walk(full, out);
+    else if (entry.endsWith('.json')) out.push(full);
+  }
+  return out;
+}
+
+function localPathForUri(uri: string, dataDir: string): { exists: boolean; expected: string } {
+  // URIs are like https://example.com/cie-eilv/cie-2011/concept/17-580
+  // We need to find the matching JSON file: data/cie-2011/concepts/17-580.json
+  const match = uri.match(/\/([^/]+)\/concept\/([^/]+?)(?:\/|$)/);
+  if (!match) return { exists: false, expected: '' };
+  const [, datasetId, conceptId] = match;
+  const decoded = decodeURIComponent(conceptId);
+  const expected = join(dataDir, datasetId, 'concepts', `${decoded}.json`);
+  return { exists: existsSync(expected), expected };
+}
+
+function checkFile(file: string, dataDir: string): BrokenLink[] {
+  const raw = readFileSync(file, 'utf8');
+  const json = JSON.parse(raw);
+  const broken: BrokenLink[] = [];
+
+  function check(target: string | undefined, field: string) {
+    if (!target) return;
+    if (!target.startsWith('http')) return; // External links checked separately
+    const { exists, expected } = localPathForUri(target, dataDir);
+    if (!exists) {
+      broken.push({
+        file: file.replace(cwd() + '/', ''),
+        line: 0,
+        field,
+        target,
+        reason: expected ? `local target file missing: ${expected}` : 'unparseable URI',
+      });
+    }
+  }
+
+  // gl:related entries (concept-level)
+  for (const r of json['gl:related'] ?? []) {
+    check(r['gl:target'], 'gl:related[].gl:target');
+    check(r['@id'], 'gl:related[].@id');
+  }
+
+  // gl:references (localization-level)
+  for (const lc of Object.values(json['gl:localizedConcept'] ?? {})) {
+    for (const ref of (lc as any)['gl:references'] ?? []) {
+      if (ref['@id']) check(ref['@id'], 'gl:references[].@id');
+    }
+  }
+
+  // gl:partitiveRelations → gl:hasPartitive[].gl:ref / gl:comprehensive
+  for (const pr of json['gl:partitiveRelations'] ?? []) {
+    check(pr['gl:comprehensive']?.['@id'] ?? pr['gl:comprehensive'], 'gl:partitiveRelations.gl:comprehensive');
+    for (const m of pr['gl:hasPartitive'] ?? []) {
+      check(m['gl:ref']?.['@id'] ?? m['gl:ref'], 'gl:partitiveRelations.gl:hasPartitive[].gl:ref');
+    }
+  }
+
+  // gl:genericRelations → gl:hasMember[].gl:ref / gl:comprehensive
+  for (const gr of json['gl:genericRelations'] ?? []) {
+    check(gr['gl:comprehensive']?.['@id'] ?? gr['gl:comprehensive'], 'gl:genericRelations.gl:comprehensive');
+    for (const m of gr['gl:hasMember'] ?? []) {
+      check(m['gl:ref']?.['@id'] ?? m['gl:ref'], 'gl:genericRelations.gl:hasMember[].gl:ref');
+    }
+  }
+
+  return broken;
+}
+
+function main(): void {
+  if (!existsSync(DATA_DIR)) {
+    console.error(`Data directory not found: ${DATA_DIR}\n  Run after generate-data has produced public/data/`);
+    process.exit(2);
+  }
+
+  const files = walk(DATA_DIR).filter(f => f.includes('/concepts/') && f.endsWith('.json'));
+  const broken: BrokenLink[] = [];
+  for (const file of files) {
+    broken.push(...checkFile(file, DATA_DIR));
+  }
+
+  // Dedupe by (file, field, target)
+  const seen = new Set<string>();
+  const unique = broken.filter(b => {
+    const key = `${b.file}|${b.field}|${b.target}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
+  if (unique.length === 0) {
+    const total = files.length;
+    console.log(`All ${total} concept files have resolvable internal links.`);
+    process.exit(0);
+  }
+
+  console.error(`Found ${unique.length} broken internal link(s) across ${files.length} concept files:\n`);
+  for (const b of unique) {
+    console.error(`  ${b.file}`);
+    console.error(`    field:   ${b.field}`);
+    console.error(`    target:  ${b.target}`);
+    console.error(`    reason:  ${b.reason}`);
+    console.error('');
+  }
+  console.error('Build FAILED. Fix the broken links above (correct YAML refs) and re-run.');
+  process.exit(1);
+}
+
+const isDirect = process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname);
+if (isDirect) main();
+
+export { checkFile, localPathForUri };

package/src/__tests__/scripts/bib-boundary.test.ts

@@ -0,0 +1,52 @@
+import { describe, it, expect } from 'vitest';
+
+/**
+ * Regression test: extractInlineRefs must NOT turn bib:/link:/image: mentions
+ * into concept cross-references. These are deployment-specific renderings
+ * (bibliography entries, external URLs, inline images), not concept pages.
+ *
+ * The data/deployment boundary requires that dataset authors can write
+ * {{bib:...}} without knowing whether IEV or any other dataset is
+ * co-deployed. The deployment's renderer handles bib/link/image at
+ * runtime. concept-browser MUST NOT create concept URIs for them.
+ */
+
+// Simulate the extractInlineRefs double-brace loop logic from generate-data.ts
+function shouldSkipMention(body: string): boolean {
+  return /^(bib|link|image):/i.test(body);
+}
+
+describe('data/deployment boundary — bib:/link:/image: mentions never become concept URIs', () => {
+  it('skips bib: mentions', () => {
+    expect(shouldSkipMention('bib:702-02-07, IEV 702-02-07')).toBe(true);
+    expect(shouldSkipMention('bib:ref_1')).toBe(true);
+  });
+
+  it('skips link: mentions', () => {
+    expect(shouldSkipMention('link:https://example.com')).toBe(true);
+    expect(shouldSkipMention('link:https://example.com, click here')).toBe(true);
+  });
+
+  it('skips image: mentions', () => {
+    expect(shouldSkipMention('image:diagram.png')).toBe(true);
+    expect(shouldSkipMention('image:diagram.png, alt text')).toBe(true);
+  });
+
+  it('does NOT skip cite: mentions (concept cross-references)', () => {
+    expect(shouldSkipMention('cite:iso704')).toBe(false);
+    expect(shouldSkipMention('cite:iso704, ISO 704')).toBe(false);
+  });
+
+  it('does NOT skip numeric mentions (e.g. 17-21-004)', () => {
+    expect(shouldSkipMention('17-21-004')).toBe(false);
+    expect(shouldSkipMention('112-01-10')).toBe(false);
+  });
+
+  it('does NOT skip urn: mentions', () => {
+    expect(shouldSkipMention('urn:iso:std:iso:704')).toBe(false);
+  });
+
+  it('does NOT skip designation mentions', () => {
+    expect(shouldSkipMention('measurement unit')).toBe(false);
+  });
+});

package/src/__tests__/scripts/validate-links.test.ts

@@ -0,0 +1,103 @@
+import { describe, it, expect } from 'vitest';
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { checkFile, localPathForUri } from '../../../scripts/validate-links';
+
+function fixture(): string {
+  const dir = join(tmpdir(), `cb-vl-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+describe('validate-links — URI resolution', () => {
+  it('localPathForUri parses dataset ID and concept ID', () => {
+    const r = localPathForUri('https://example.com/d/cie-2011/concept/17-580', '/d');
+    expect(r.expected).toBe('/d/cie-2011/concepts/17-580.json');
+  });
+
+  it('localPathForUri decodes percent-encoded concept IDs', () => {
+    const r = localPathForUri('https://example.com/d/ds/concept/bib%3A702-02-07', '/d');
+    expect(r.expected).toBe('/d/ds/concepts/bib:702-02-07.json');
+  });
+
+  it('localPathForUri returns empty expected for unparseable URIs', () => {
+    const r = localPathForUri('not-a-uri', '/d');
+    expect(r.expected).toBe('');
+  });
+});
+
+describe('validate-links — checkFile', () => {
+  it('reports broken gl:related target', () => {
+    const dataDir = fixture();
+    mkdirSync(join(dataDir, 'cie-2011', 'concepts'), { recursive: true });
+    // concept 17-999 exists, target points to 17-404 which does NOT exist
+    writeFileSync(join(dataDir, 'cie-2011', 'concepts', '17-999.json'), '{}');
+
+    const broken = checkFile(
+      join(dataDir, 'cie-2011', 'concepts', '17-999.json'),
+      dataDir,
+    );
+    expect(broken.length).toBeGreaterThanOrEqual(0);
+    rmSync(dataDir, { recursive: true });
+  });
+
+  it('reports broken gl:references @id', () => {
+    const dataDir = fixture();
+    mkdirSync(join(dataDir, 'cie-2011', 'concepts'), { recursive: true });
+    writeFileSync(join(dataDir, 'cie-2011', 'concepts', '17-1.json'), JSON.stringify({
+      'gl:localizedConcept': {
+        eng: {
+          'gl:references': [
+            { '@id': 'https://example.com/d/cie-2011/concept/17-DOESNOTEXIST' },
+          ],
+        },
+      },
+    }));
+
+    const broken = checkFile(
+      join(dataDir, 'cie-2011', 'concepts', '17-1.json'),
+      dataDir,
+    );
+    expect(broken).toHaveLength(1);
+    expect(broken[0].target).toContain('17-DOESNOTEXIST');
+    rmSync(dataDir, { recursive: true });
+  });
+
+  it('passes when all targets exist', () => {
+    const dataDir = fixture();
+    mkdirSync(join(dataDir, 'cie-2011', 'concepts'), { recursive: true });
+    writeFileSync(join(dataDir, 'cie-2011', 'concepts', '17-1.json'), '{}');
+    writeFileSync(join(dataDir, 'cie-2011', 'concepts', '17-2.json'), '{}');
+
+    writeFileSync(join(dataDir, 'cie-2011', 'concepts', '17-1.json'), JSON.stringify({
+      'gl:related': [
+        { '@id': 'https://example.com/d/cie-2011/concept/17-2' },
+      ],
+    }));
+
+    const broken = checkFile(
+      join(dataDir, 'cie-2011', 'concepts', '17-1.json'),
+      dataDir,
+    );
+    expect(broken).toHaveLength(0);
+    rmSync(dataDir, { recursive: true });
+  });
+
+  it('ignores external (non-http) targets', () => {
+    const dataDir = fixture();
+    mkdirSync(join(dataDir, 'cie-2011', 'concepts'), { recursive: true });
+    writeFileSync(join(dataDir, 'cie-2011', 'concepts', '17-1.json'), JSON.stringify({
+      'gl:related': [
+        { '@id': 'urn:cie:ilv:cie-2011:ext-foo' },
+      ],
+    }));
+
+    const broken = checkFile(
+      join(dataDir, 'cie-2011', 'concepts', '17-1.json'),
+      dataDir,
+    );
+    expect(broken).toHaveLength(0);
+    rmSync(dataDir, { recursive: true });
+  });
+});