@glossarist/concept-browser 0.7.43 → 0.7.44

package/cli/index.mjs

@@ -20,7 +20,7 @@
  */
 
 import { loadSiteConfig } from '../scripts/load-site-config.mjs';
-import { execSync } from 'child_process';
+import { existsSync } from 'fs';
 import { resolve, dirname } from 'path';
 import { fileURLToPath } from 'url';
 
@@ -166,22 +166,21 @@ Environment:
       }
     }
 
-    // Run vite build using the package's vite.config.ts
+    // Run vite build using the package's vite.config.ts via programmatic API
     console.log(`\n=== BUILD SPA ===\n`);
     const viteConfig = resolve(pkgRoot, 'vite.config.ts');
-    const viteBin = [resolve(pkgRoot, 'node_modules', '.bin', 'vite'), 'vite'].find(p => {
-      try { execSync(`${p} --version`, { stdio: 'pipe' }); return true; } catch { return false; }
-    });
-    execSync(`${viteBin} build --config ${viteConfig}`, {
-      stdio: 'inherit',
-      env: { ...process.env },
+    const { build: viteBuild } = await import('vite');
+    await viteBuild({
+      configFile: viteConfig,
+      root: pkgRoot,
+      mode: 'production',
     });
 
-    // Run postbuild (404 page)
-    try {
-      const postbuild = resolve(pkgRoot, 'scripts', 'generate-404.js');
-      execSync(`node ${postbuild}`, { stdio: 'inherit' });
-    } catch {}
+    // Run postbuild (404 page) via dynamic import
+    const postbuild = resolve(pkgRoot, 'scripts', 'generate-404.js');
+    if (existsSync(postbuild)) {
+      await import(`file://${postbuild}`);
+    }
 
     return;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glossarist/concept-browser",
-  "version": "0.7.43",
+  "version": "0.7.44",
   "description": "Vue SPA for browsing Glossarist terminology datasets with cross-reference resolution, graph visualization, and multi-language support",
   "type": "module",
   "bin": {
@@ -27,6 +27,7 @@
     "favicons": "^7.2.0",
     "glossarist": "^0.3.7",
     "js-yaml": "^4.1.0",
+    "jszip": "^3.10.1",
     "pinia": "^2.3.1",
     "postcss": "^8.5.3",
     "sharp": "^0.34.5",

package/scripts/__tests__/fetch-datasets.test.mjs

@@ -0,0 +1,105 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { assertLocalPathSafe } from '../lib/local-path-safety.mjs';
+
+function makeTmpTree() {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cb-fetch-'));
+  const datasetsDir = path.join(root, '.datasets');
+  const sourceDir = path.join(root, 'source-data');
+  fs.mkdirSync(datasetsDir);
+  fs.mkdirSync(path.join(sourceDir, 'concepts'), { recursive: true });
+  fs.writeFileSync(path.join(sourceDir, 'concepts', 'a.yaml'), 'termid: 1\n');
+  return { root, datasetsDir, sourceDir };
+}
+
+describe('assertLocalPathSafe', () => {
+  let tree;
+
+  beforeEach(() => { tree = makeTmpTree(); });
+  afterEach(() => {
+    fs.rmSync(tree.root, { recursive: true, force: true });
+  });
+
+  it('returns resolved path for a safe external location', () => {
+    const resolved = assertLocalPathSafe('foo', tree.sourceDir, {
+      root: tree.root,
+      datasetsDir: tree.datasetsDir,
+    });
+    // Returns the realpath (symlinks resolved); on macOS tmpdir resolves
+    // /var → /private/var, so compare against realpath, not path.resolve.
+    expect(resolved).toBe(fs.realpathSync(path.resolve(tree.root, tree.sourceDir)));
+  });
+
+  it('throws when localPath does not exist', () => {
+    expect(() =>
+      assertLocalPathSafe('foo', path.join(tree.root, 'nope'), {
+        root: tree.root,
+        datasetsDir: tree.datasetsDir,
+      })
+    ).toThrow(/does not exist/);
+  });
+
+  it('throws when localPath equals .datasets/<id>', () => {
+    const staged = path.join(tree.datasetsDir, 'foo');
+    fs.mkdirSync(staged, { recursive: true });
+    expect(() =>
+      assertLocalPathSafe('foo', staged, {
+        root: tree.root,
+        datasetsDir: tree.datasetsDir,
+      })
+    ).toThrow(/same physical location/);
+  });
+
+  it('throws when localPath is nested inside .datasets/<id>', () => {
+    const staged = path.join(tree.datasetsDir, 'foo');
+    fs.mkdirSync(path.join(staged, 'subdir'), { recursive: true });
+    expect(() =>
+      assertLocalPathSafe('foo', path.join(staged, 'subdir'), {
+        root: tree.root,
+        datasetsDir: tree.datasetsDir,
+      })
+    ).toThrow(/nested inside/);
+  });
+
+  it('throws when localPath contains .datasets/<id> (parent-of-staging hazard)', () => {
+    // localPath = root itself, datasetsDir = root/.datasets — staging ops
+    // (rm -rf .datasets/<id>) would touch files inside localPath.
+    expect(() =>
+      assertLocalPathSafe('foo', tree.root, {
+        root: tree.root,
+        datasetsDir: tree.datasetsDir,
+      })
+    ).toThrow(/contains .datasets/);
+  });
+
+  it('throws when localPath is a symlink to .datasets/<id> (the reported bug)', () => {
+    const staged = path.join(tree.datasetsDir, 'foo');
+    fs.mkdirSync(staged, { recursive: true });
+    const symlinkPath = path.join(tree.root, 'evil-link');
+    fs.symlinkSync(staged, symlinkPath);
+    expect(() =>
+      assertLocalPathSafe('foo', symlinkPath, {
+        root: tree.root,
+        datasetsDir: tree.datasetsDir,
+      })
+    ).toThrow(/same physical location/);
+  });
+
+  it('does NOT modify the source directory (regression for data-loss bug)', () => {
+    const sentinel = path.join(tree.sourceDir, 'concepts', 'SENTINEL.yaml');
+    fs.writeFileSync(sentinel, 'termid: sentinel\n');
+    const beforeMtime = fs.statSync(sentinel).mtimeMs;
+
+    assertLocalPathSafe('foo', tree.sourceDir, {
+      root: tree.root,
+      datasetsDir: tree.datasetsDir,
+    });
+
+    // Source directory must be completely untouched after the safety check.
+    expect(fs.existsSync(sentinel)).toBe(true);
+    expect(fs.statSync(sentinel).mtimeMs).toBe(beforeMtime);
+    expect(fs.readdirSync(path.join(tree.sourceDir, 'concepts'))).toContain('SENTINEL.yaml');
+  });
+});

package/scripts/fetch-datasets.mjs

@@ -1,23 +1,26 @@
 #!/usr/bin/env node
 /**
- * fetch-datasets.mjs — Load datasets from .gcr files or clone source repos.
+ * fetch-datasets.mjs — Load datasets from .gcr files, local paths, or git repos.
  *
  * Reads site config (via load-site-config.mjs), for each dataset:
  *   1. If .gcr/{id}.gcr exists, extract to .datasets/{id}/
  *   2. Else download from gcrPackage URL and extract
- *   3. Else clone/update source repo into .datasets/{id}/
+ *   3. Else if localPath is set, use it in-place (NO copy, NO staging)
+ *   4. Else clone/update source repo into .datasets/{id}/
  *
  * After fetching, validates that all GCR dependencies are satisfiable
  * (either provided locally or routed externally).
  *
- * Supports localPath field in dataset config for local paths.
- * Supports GITHUB_TOKEN for private repos.
+ * No shell commands. All file ops use Node fs; ZIP uses JSZip; git uses
+ * execFileSync with array args (no shell interpolation).
  */
 import fs from 'fs';
 import path from 'path';
+import JSZip from 'jszip';
 import { loadGcr } from 'glossarist';
-import { execSync } from 'child_process';
+import { execFileSync } from 'child_process';
 import { loadSiteConfig } from './load-site-config.mjs';
+import { assertLocalPathSafe } from './lib/local-path-safety.mjs';
 
 const ROOT = process.cwd();
 const DATASETS_DIR = path.join(ROOT, '.datasets');
@@ -39,23 +42,29 @@ async function downloadGcr(url, destPath) {
   console.log(`  Saved to ${destPath} (${(buf.length / 1024).toFixed(0)} KB)`);
 }
 
-// --- GCR extraction ---
-function extractGcr(gcrPath, targetDir) {
-  if (fs.existsSync(targetDir)) {
-    fs.rmSync(targetDir, { recursive: true, force: true });
+// --- GCR extraction (pure JSZip; no shell, cross-platform) ---
+async function extractGcr(gcrPath, targetDir) {
+  const targetAbs = path.resolve(targetDir);
+  if (fs.existsSync(targetAbs)) {
+    fs.rmSync(targetAbs, { recursive: true, force: true });
   }
-  fs.mkdirSync(targetDir, { recursive: true });
-
-  try {
-    execSync(`unzip -o -q "${gcrPath}" -d "${targetDir}"`, { stdio: 'pipe' });
-  } catch {
-    try {
-      execSync(`python3 -c "import zipfile; zipfile.ZipFile('${gcrPath}').extractall('${targetDir}')"`, { stdio: 'pipe' });
-    } catch (e2) {
-      throw new Error(`Failed to extract ${gcrPath}`);
+  fs.mkdirSync(targetAbs, { recursive: true });
+
+  const buf = fs.readFileSync(gcrPath);
+  const zip = await JSZip.loadAsync(buf);
+  const entries = Object.values(zip.files);
+  for (const entry of entries) {
+    if (entry.dir) continue;
+    // zip-slip guard: refuse entries that escape targetDir
+    const dest = path.resolve(targetAbs, entry.name);
+    if (dest !== targetAbs && !dest.startsWith(targetAbs + path.sep)) {
+      throw new Error(`Refusing to extract entry outside target dir: ${entry.name}`);
     }
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    const content = await entry.async('nodebuffer');
+    fs.writeFileSync(dest, content);
   }
-  console.log(`  Extracted to ${targetDir}`);
+  console.log(`  Extracted to ${targetAbs}`);
 }
 
 // --- Read GCR metadata from ZIP without extraction ---
@@ -96,7 +105,7 @@ function validateDependencies(config, gcrMetadata) {
   return errors;
 }
 
-// --- Git operations ---
+// --- Git operations (execFileSync with array args — no shell) ---
 function cloneOrUpdate(sourceRepo, targetDir) {
   const env = { ...process.env };
   let repoUrl = sourceRepo;
@@ -104,24 +113,28 @@ function cloneOrUpdate(sourceRepo, targetDir) {
     repoUrl = sourceRepo.replace('https://', `https://x-access-token:${env.GITHUB_TOKEN}@`);
   }
 
-  if (fs.existsSync(path.join(targetDir, '.git'))) {
+  const targetAbs = path.resolve(targetDir);
+
+  if (fs.existsSync(path.join(targetAbs, '.git'))) {
     console.log(`  Updating existing clone...`);
     try {
-      execSync('git fetch origin', { cwd: targetDir, stdio: 'pipe', env });
-      execSync('git reset --hard origin/HEAD', { cwd: targetDir, stdio: 'pipe', env });
-      execSync('git clean -fd', { cwd: targetDir, stdio: 'pipe', env });
+      execFileSync('git', ['fetch', 'origin'], { cwd: targetAbs, stdio: 'pipe', env });
+      execFileSync('git', ['reset', '--hard', 'origin/HEAD'], { cwd: targetAbs, stdio: 'pipe', env });
+      execFileSync('git', ['clean', '-fd'], { cwd: targetAbs, stdio: 'pipe', env });
     } catch {
       console.warn(`  git update failed, re-cloning`);
-      fs.rmSync(targetDir, { recursive: true, force: true });
-      execSync(`git clone --depth 1 "${repoUrl}" "${targetDir}"`, { stdio: 'pipe', env });
+      fs.rmSync(targetAbs, { recursive: true, force: true });
+      execFileSync('git', ['clone', '--depth', '1', repoUrl, targetAbs], { stdio: 'pipe', env });
     }
   } else {
-    fs.mkdirSync(targetDir, { recursive: true });
+    fs.mkdirSync(targetAbs, { recursive: true });
     console.log(`  Cloning ${sourceRepo}...`);
-    execSync(`git clone --depth 1 "${repoUrl}" "${targetDir}"`, { stdio: 'pipe', env });
+    execFileSync('git', ['clone', '--depth', '1', repoUrl, targetAbs], { stdio: 'pipe', env });
   }
 }
 
+// --- localPath safety check: see scripts/lib/local-path-safety.mjs ---
+
 // --- Main ---
 console.log('Fetching glossarist datasets...\n');
 
@@ -137,7 +150,7 @@ for (const ds of config.datasets) {
   try {
     if (fs.existsSync(gcrPath)) {
       console.log(`  Using local .gcr/${ds.id}.gcr`);
-      extractGcr(gcrPath, targetDir);
+      await extractGcr(gcrPath, targetDir);
     } else if (ds.gcrPackage) {
       console.log(`  Using GCR package: ${ds.gcrPackage}`);
       try {
@@ -148,29 +161,18 @@ for (const ds of config.datasets) {
         console.log();
         continue;
       }
-      extractGcr(gcrPath, targetDir);
+      await extractGcr(gcrPath, targetDir);
+    } else if (ds.localPath) {
+      // localPath means "data is here, use in-place." No copy, no staging.
+      // generate-data.mjs reads from localPath directly via datasetDir(ds).
+      const localResolved = assertLocalPathSafe(ds.id, ds.localPath);
+      console.log(`  Using localPath in-place: ${localResolved}`);
+    } else if (ds.sourceRepo) {
+      cloneOrUpdate(ds.sourceRepo, targetDir);
     } else {
-      const envOverride = ds.localPath;
-      if (envOverride) {
-        console.log(`  Using local path: ${envOverride}`);
-        if (!fs.existsSync(targetDir)) fs.mkdirSync(targetDir, { recursive: true });
-        const localConcepts = path.join(envOverride, 'concepts');
-        const targetConcepts = path.join(targetDir, 'concepts');
-        if (fs.existsSync(localConcepts)) {
-          if (fs.existsSync(targetConcepts)) fs.rmSync(targetConcepts, { recursive: true, force: true });
-          execSync(`cp -r "${localConcepts}" "${targetConcepts}"`, { stdio: 'pipe' });
-        }
-        const registerYaml = path.join(envOverride, 'register.yaml');
-        if (fs.existsSync(registerYaml)) {
-          fs.copyFileSync(registerYaml, path.join(targetDir, 'register.yaml'));
-        }
-      } else if (ds.sourceRepo) {
-        cloneOrUpdate(ds.sourceRepo, targetDir);
-      } else {
-        console.warn(`  No source configured, skipping`);
-        console.log();
-        continue;
-      }
+      console.warn(`  No source configured, skipping`);
+      console.log();
+      continue;
     }
 
     // Read metadata for dependency validation (from GCR ZIP, not extracted dir)

package/scripts/generate-data.mjs

@@ -9,6 +9,18 @@ const ROOT = process.cwd();
 const PUBLIC = path.join(ROOT, 'public');
 const DATA = path.join(PUBLIC, 'data');
 
+/**
+ * Resolve a dataset's source directory.
+ * - If `ds.localPath` is set, use it in-place (resolved against ROOT).
+ *   No staging, no copy. fetch-datasets.mjs verifies the path is safe.
+ * - Otherwise fall back to the standard .datasets/<id>/ staging dir.
+ */
+function datasetDir(ds) {
+  return ds.localPath
+    ? path.resolve(ROOT, ds.localPath)
+    : path.join(ROOT, '.datasets', ds.id);
+}
+
 const DS_PALETTE = [
   '#3366ff', '#0d9488', '#d97706', '#8b5cf6',
   '#ec4899', '#059669', '#dc2626', '#6366f1',
@@ -934,7 +946,8 @@ function processDataset(dir, register, opts) {
   }
 
   // Copy bulk format files from compiled/ directory (full GCR)
-  const compiledDir = path.join(ROOT, '.datasets', register, 'compiled');
+  const sourceRoot = path.dirname(dir);
+  const compiledDir = path.join(sourceRoot, 'compiled');
   const bulkFormats = [];
   if (fs.existsSync(compiledDir)) {
     for (const file of fs.readdirSync(compiledDir)) {
@@ -993,7 +1006,7 @@ function processDataset(dir, register, opts) {
   writeJson(path.join(DATA, register, 'manifest.json'), manifest);
 
   // Copy bibliography.yaml → bibliography.json
-  const bibPath = path.join(ROOT, '.datasets', register, 'bibliography.yaml');
+  const bibPath = path.join(sourceRoot, 'bibliography.yaml');
   if (fs.existsSync(bibPath)) {
     const bibData = readYaml(bibPath);
     writeJson(path.join(DATA, register, 'bibliography.json'), bibData);
@@ -1001,7 +1014,7 @@ function processDataset(dir, register, opts) {
   }
 
   // Copy images/
-  const imagesSrcDir = path.join(ROOT, '.datasets', register, 'images');
+  const imagesSrcDir = path.join(sourceRoot, 'images');
   if (fs.existsSync(imagesSrcDir) && fs.statSync(imagesSrcDir).isDirectory()) {
     const imagesDestDir = path.join(DATA, register, 'images');
     fs.mkdirSync(imagesDestDir, { recursive: true });
@@ -1030,8 +1043,8 @@ const registerCache = {};
 
 // Pre-load all register.yaml files (needed before buildRefMaps for URI pattern indexing)
 for (const ds of config.datasets) {
-  const registerDir = path.join(ROOT, '.datasets', ds.id);
-  const registerYamlPath = path.join(registerDir, 'register.yaml');
+  const dsDir = datasetDir(ds);
+  const registerYamlPath = path.join(dsDir, 'register.yaml');
   if (fs.existsSync(registerYamlPath)) {
     try {
       const raw = yaml.load(fs.readFileSync(registerYamlPath, 'utf8'));
@@ -1047,7 +1060,7 @@ const refMaps = buildRefMaps(config, registerCache);
 for (let i = 0; i < config.datasets.length; i++) {
   const ds = config.datasets[i];
 
-  const dir = path.join(ROOT, '.datasets', ds.id, 'concepts');
+  const dir = path.join(datasetDir(ds), 'concepts');
   if (!fs.existsSync(dir)) {
     console.warn(`Skipping ${ds.id}: source directory not found (${dir})`);
     console.warn(`  Run: npm run fetch-datasets`);
@@ -1089,8 +1102,8 @@ for (let i = 0; i < config.datasets.length; i++) {
     status: ds.editionStatus || reg?.status,
     ordering: reg?.ordering || null,
     sections: reg?.sections ? reg.sections.map(s => s.toJSON()) : [],
-    hasBibliography: fs.existsSync(path.join(ROOT, '.datasets', ds.id, 'bibliography.yaml')),
-    hasImages: fs.existsSync(path.join(ROOT, '.datasets', ds.id, 'images')),
+    hasBibliography: fs.existsSync(path.join(datasetDir(ds), 'bibliography.yaml')),
+    hasImages: fs.existsSync(path.join(datasetDir(ds), 'images')),
   });
   registry.push({
     id: ds.id,

package/scripts/lib/local-path-safety.mjs

@@ -0,0 +1,68 @@
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Assert that a dataset's `localPath` is safe to use as an in-place source.
+ *
+ * "Safe" means: the resolved physical location of `localPath` is disjoint
+ * from the staging directory (`.datasets/<id>`). If they overlap, staging
+ * operations (rm, extract, clone) would destroy the user's source data —
+ * the data-loss bug reported in v0.7.43.
+ *
+ * Returns the resolved absolute path on success; throws on any hazard.
+ *
+ * @param {string} datasetId
+ * @param {string} localPath - relative to `root` (or absolute)
+ * @param {{ root?: string, datasetsDir?: string }} [opts]
+ * @returns {string} resolved absolute path
+ */
+/**
+ * Compute the canonical physical path of `p`, resolving symlinks on the
+ * existing prefix. If `p` itself exists, this is just `realpathSync(p)`.
+ * If not, we walk up to the nearest existing ancestor, realpath it, and
+ * re-append the non-existent tail. This is needed because macOS tmpdir
+ * (`/var/folders/...`) is a symlink to `/private/var/folders/...`; without
+ * this, prefix comparisons across the symlink boundary silently fail.
+ */
+function physicalPath(p) {
+  if (fs.existsSync(p)) return fs.realpathSync(p);
+  const parent = path.dirname(p);
+  const parentReal = fs.existsSync(parent) ? fs.realpathSync(parent) : physicalPath(parent);
+  return path.join(parentReal, path.basename(p));
+}
+
+export function assertLocalPathSafe(datasetId, localPath, { root = process.cwd(), datasetsDir } = {}) {
+  const datasetsRoot = datasetsDir || path.join(root, '.datasets');
+  const localResolved = path.resolve(root, localPath);
+
+  if (!fs.existsSync(localResolved)) {
+    throw new Error(`localPath for ${datasetId} does not exist: ${localResolved}`);
+  }
+
+  const localReal = fs.realpathSync(localResolved);
+  const stagedAbs = path.join(datasetsRoot, datasetId);
+  const stagedReal = physicalPath(stagedAbs);
+
+  if (localReal === stagedReal) {
+    throw new Error(
+      `localPath for ${datasetId} resolves to the same physical location as .datasets/${datasetId} ` +
+      `(${localReal}). Refusing to operate — source and staging would clobber. ` +
+      `Use a path outside .datasets/.`
+    );
+  }
+  if (localReal.startsWith(stagedReal + path.sep)) {
+    throw new Error(
+      `localPath for ${datasetId} is nested inside .datasets/${datasetId}. ` +
+      `Refusing to operate — staging operations would destroy source data. ` +
+      `Use a path outside .datasets/.`
+    );
+  }
+  if (stagedReal.startsWith(localReal + path.sep)) {
+    throw new Error(
+      `localPath for ${datasetId} contains .datasets/${datasetId}. ` +
+      `Refusing to operate — staging operations would destroy source data. ` +
+      `Use a path outside localPath.`
+    );
+  }
+  return localReal;
+}