@kenjura/ursa 0.96.0 → 0.97.0

package/src/helper/build/templates.js

@@ -1,7 +1,6 @@
 // Template helpers for build
-import { readFile, readdir } from "fs/promises";
+import { readFile, readdir, existsSync } from "./tracedFs.js";
 import { join, basename } from "path";
-import { existsSync } from "fs";
 
 /**
  * Get all templates from meta directory

package/src/helper/build/tracedFs.js

@@ -0,0 +1,247 @@
+/**
+ * Filesystem access that records what it touched.
+ *
+ * The build graph (graph.js) decides what to rebuild purely from recorded
+ * inputs, so every file a node's compute function reads, every path it probes
+ * and every directory it lists has to become an edge. Threading a context
+ * object through every helper that touches the disk — the menu walker, the
+ * label resolver, the breadcrumb builder, the style.css chain finder — would
+ * mean rewriting all of them. Instead those helpers import their `fs` calls
+ * from here. When a node is computing, an AsyncLocalStorage store carries that
+ * node's recorder, and each call below reports the leaf it observed. Outside a
+ * compute (tests, the CLI) the calls are plain `fs`.
+ *
+ * Because AsyncLocalStorage follows the async chain, concurrent computes each
+ * see only their own recorder, and a synchronous read inside a deeply nested
+ * helper is attributed to the right node.
+ *
+ * Fingerprints:
+ *   - file:   md5 of the content (16 hex chars), or "missing"
+ *   - lookup: "exists" | "absent"
+ *   - dir:    md5 of the sorted visible child names, or "missing"
+ *
+ * Names beginning with "." and the deny-listed names (`node_modules`, the
+ * `4913` vim probe, editor scratch files) are not part of a directory's
+ * fingerprint — they are never build inputs, and excluding them keeps an
+ * editor's temp files from dirtying the menu.
+ */
+
+import { AsyncLocalStorage } from "async_hooks";
+import { createHash } from "crypto";
+import fs from "fs";
+import fsp from "fs/promises";
+
+const als = new AsyncLocalStorage();
+
+export const MISSING = "missing";
+export const EXISTS = "exists";
+export const ABSENT = "absent";
+
+export function hashBytes(data) {
+  return createHash("md5").update(data).digest("hex").substring(0, 16);
+}
+
+/** Run `fn` with `recorder` as the active leaf recorder. */
+export function withRecorder(recorder, fn) {
+  return als.run(recorder, fn);
+}
+
+/** The recorder for the currently computing node, or undefined. */
+export function currentRecorder() {
+  return als.getStore();
+}
+
+// ---------------------------------------------------------------------------
+// Directory listings
+// ---------------------------------------------------------------------------
+
+/** Editor scratch and probe files that must never count as inputs. */
+export function isScratchName(name) {
+  return (
+    name.endsWith("~") ||
+    name.endsWith(".swp") ||
+    name.endsWith(".swx") ||
+    name.endsWith(".tmp") ||
+    name.startsWith(".#") ||
+    name === "4913"
+  );
+}
+
+/** Names never observed by a directory leaf. */
+export function isIgnoredDirEntry(name) {
+  return name.startsWith(".") || name === "node_modules" || isScratchName(name);
+}
+
+/**
+ * Fingerprint a directory from its Dirent list (or `null` when it is missing).
+ * Sorted by name so readdir order never leaks into a fingerprint.
+ */
+export function dirFingerprintFromEntries(entries) {
+  if (!entries) return MISSING;
+  const names = [];
+  for (const e of entries) {
+    const name = typeof e === "string" ? e : e.name;
+    if (isIgnoredDirEntry(name)) continue;
+    // A directory and a file of the same name are different inputs
+    names.push(typeof e !== "string" && e.isDirectory() ? name + "/" : name);
+  }
+  names.sort();
+  return hashBytes(names.join("\n"));
+}
+
+// ---------------------------------------------------------------------------
+// Recording helpers (shared by sync and async variants)
+// ---------------------------------------------------------------------------
+
+function recordFile(rec, path, st, buf) {
+  if (!rec) return;
+  if (!st) {
+    rec.file(path, MISSING, null);
+    return;
+  }
+  const stats = { size: st.size, mtimeMs: st.mtimeMs };
+  const fp = rec.knownFileFingerprint?.(path, stats) ?? hashBytes(buf);
+  rec.file(path, fp, stats);
+}
+
+function recordLookup(rec, path, exists) {
+  if (rec) rec.lookup(path, exists ? EXISTS : ABSENT);
+}
+
+function recordDir(rec, path, entries) {
+  if (rec) rec.dir(path, dirFingerprintFromEntries(entries));
+}
+
+// ---------------------------------------------------------------------------
+// Synchronous API
+// ---------------------------------------------------------------------------
+
+export function existsSync(path) {
+  const exists = fs.existsSync(path);
+  recordLookup(currentRecorder(), path, exists);
+  return exists;
+}
+
+export function readFileSync(path, options) {
+  const rec = currentRecorder();
+  let st = null;
+  try {
+    st = fs.statSync(path);
+  } catch {
+    recordFile(rec, path, null, null);
+    throw enoent(path);
+  }
+  const buf = fs.readFileSync(path);
+  recordFile(rec, path, st, buf);
+  return options ? buf.toString(typeof options === "string" ? options : options.encoding ?? "utf8") : buf;
+}
+
+export function readdirSync(path, options) {
+  const rec = currentRecorder();
+  let entries;
+  try {
+    entries = fs.readdirSync(path, { withFileTypes: true });
+  } catch (e) {
+    recordDir(rec, path, null);
+    throw e;
+  }
+  recordDir(rec, path, entries);
+  if (options?.withFileTypes) return entries;
+  return entries.map((e) => e.name);
+}
+
+/** stat as an existence probe plus a content leaf: callers use size/mtime. */
+export function statSync(path) {
+  const rec = currentRecorder();
+  try {
+    const st = fs.statSync(path);
+    recordLookup(rec, path, true);
+    return st;
+  } catch (e) {
+    recordLookup(rec, path, false);
+    throw e;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Promise API
+// ---------------------------------------------------------------------------
+
+export async function readFile(path, options) {
+  const rec = currentRecorder();
+  let st = null;
+  try {
+    st = await fsp.stat(path);
+  } catch {
+    recordFile(rec, path, null, null);
+    throw enoent(path);
+  }
+  const buf = await fsp.readFile(path);
+  recordFile(rec, path, st, buf);
+  return options ? buf.toString(typeof options === "string" ? options : options.encoding ?? "utf8") : buf;
+}
+
+export async function readdir(path, options) {
+  const rec = currentRecorder();
+  let entries;
+  try {
+    entries = await fsp.readdir(path, { withFileTypes: true });
+  } catch (e) {
+    recordDir(rec, path, null);
+    throw e;
+  }
+  recordDir(rec, path, entries);
+  if (options?.withFileTypes) return entries;
+  return entries.map((e) => e.name);
+}
+
+export async function stat(path) {
+  const rec = currentRecorder();
+  try {
+    const st = await fsp.stat(path);
+    recordLookup(rec, path, true);
+    return st;
+  } catch (e) {
+    recordLookup(rec, path, false);
+    throw e;
+  }
+}
+
+export async function exists(path) {
+  try {
+    await fsp.access(path);
+    recordLookup(currentRecorder(), path, true);
+    return true;
+  } catch {
+    recordLookup(currentRecorder(), path, false);
+    return false;
+  }
+}
+
+/**
+ * Directory entries that are build inputs, sorted by name, with `kind`.
+ * Returns [] for a missing directory (the miss is recorded).
+ * @returns {Promise<{name: string, kind: 'file'|'dir'|'other'}[]>}
+ */
+export async function listDir(path) {
+  let entries;
+  try {
+    entries = await readdir(path, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((e) => !isIgnoredDirEntry(e.name))
+    .map((e) => ({
+      name: e.name,
+      kind: e.isDirectory() ? "dir" : e.isFile() ? "file" : "other",
+    }))
+    .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+}
+
+function enoent(path) {
+  const e = new Error(`ENOENT: no such file or directory, open '${path}'`);
+  e.code = "ENOENT";
+  e.path = path;
+  return e;
+}

package/src/helper/contentHash.js

@@ -1,11 +1,9 @@
-import { createHash } from 'crypto';
 import { readFile, writeFile, mkdir, rm } from 'fs/promises';
 import { existsSync } from 'fs';
 import { dirname, join } from 'path';
 import { getUrsaVersion } from './ursaVersion.js';
 
 const URSA_DIR = '.ursa';
-const HASH_CACHE_FILE = 'content-hashes.json';
 const CACHE_STAMP_FILE = 'cache-stamp.json';
 
 /**
@@ -65,79 +63,3 @@ export async function enforceCacheVersion(sourceDir, version = getUrsaVersion())
 
   return { reset: hadCache, previous, version };
 }
-
-/**
- * Generate a short hash of content
- */
-export function hashContent(content) {
-  return createHash('md5').update(content).digest('hex').substring(0, 12);
-}
-
-/**
- * Load the hash cache from disk (.ursa folder in source directory)
- */
-export async function loadHashCache(sourceDir) {
-  const cachePath = join(getUrsaDir(sourceDir), HASH_CACHE_FILE);
-  try {
-    if (existsSync(cachePath)) {
-      const data = await readFile(cachePath, 'utf8');
-      return new Map(Object.entries(JSON.parse(data)));
-    }
-  } catch (e) {
-    console.warn('Could not load hash cache:', e.message);
-  }
-  return new Map();
-}
-
-/**
- * Save the hash cache to disk (.ursa folder in source directory)
- */
-export async function saveHashCache(sourceDir, hashMap) {
-  const ursaDir = getUrsaDir(sourceDir);
-  const cachePath = join(ursaDir, HASH_CACHE_FILE);
-  try {
-    await mkdir(ursaDir, { recursive: true });
-    const obj = Object.fromEntries(hashMap);
-    await writeFile(cachePath, JSON.stringify(obj, null, 2));
-    console.log(`Saved ${hashMap.size} hashes to ${cachePath}`);
-  } catch (e) {
-    console.warn('Could not save hash cache:', e.message);
-  }
-}
-
-/**
- * Check if a file needs regeneration based on content hash
- */
-export function needsRegeneration(filePath, content, hashCache) {
-  const newHash = hashContent(content);
-  const oldHash = hashCache.get(filePath);
-  return newHash !== oldHash;
-}
-
-/**
- * Check whether every expected output file for a source document exists.
- *
- * A matching content hash only proves the *source* is unchanged — it says
- * nothing about whether the output was ever written to this particular output
- * directory. The hash cache lives in the source tree (`<source>/.ursa/`) and is
- * shared by every output directory built from that source, so a hash written
- * during a build to one output dir will hash-skip the same file during a build
- * to another. Deleting (or partially losing) an output dir has the same effect.
- * Callers must combine this with needsRegeneration() so a missing output always
- * forces a rebuild.
- *
- * @param {string[]} outputPaths - Absolute paths to every file the build emits for this document
- * @returns {boolean} True only if all of them are present
- */
-export function outputsExist(outputPaths) {
-  return outputPaths.every((p) => existsSync(p));
-}
-
-/**
- * Update the hash for a file in the cache
- */
-export function updateHash(filePath, content, hashCache) {
-  const hash = hashContent(content);
-  hashCache.set(filePath, hash);
-  return hash;
-}

package/src/helper/customMenu.js

@@ -1,5 +1,5 @@
 // Custom menu support - allows defining custom menus in menu.md, menu.txt, _menu.md, or _menu.txt
-import { existsSync, readFileSync, readdirSync, statSync } from "fs";
+import { existsSync, readFileSync, readdirSync, statSync } from "./build/tracedFs.js";
 import { join, dirname, relative, resolve, basename, extname } from "path";
 import { extractMetadata } from "./metadataExtractor.js";
 

package/src/helper/fileRenderer.js

@@ -1,112 +1,120 @@
-
-import { markdownToHtml } from "./markdownHelper.cjs";
-import { wikiToHtml } from "./wikitextHelper.js";
-import { renderMDX, generateHydrationScript } from "./mdxRenderer.js";
-import { parseWithWorker, terminateParserPool } from "./parserPool.js";
-
-const DEFAULT_WIKITEXT_ARGS = { db: "noDB", noSection: true, noTOC: true };
-
-/**
- * Render a file synchronously (legacy/fallback)
- * Note: .mdx files require async rendering; use renderFileAsync instead.
- * @param {Object} options - Render options
- * @param {string} options.fileContents - Raw file content
- * @param {string} options.type - File extension (.md, .txt)
- * @param {string} options.dirname - Directory name
- * @param {string} options.basename - Base filename
- * @returns {string} Rendered HTML
- */
-export function renderFile({ fileContents, type, dirname, basename }) {
-  switch (type) {
-    case ".md":
-      return markdownToHtml(fileContents);
-    case ".txt":
-      return wikiToHtml({
-        wikitext: fileContents,
-        articleName: basename,
-        args: { ...DEFAULT_WIKITEXT_ARGS, db: dirname },
-      })?.html;
-  }
-}
-
-/**
- * Render a file asynchronously using worker threads for parallel processing
- * Falls back to main thread for wikitext or if workers are unavailable
- * @param {Object} options - Render options
- * @param {string} options.fileContents - Raw file content
- * @param {string} options.type - File extension (.md, .txt, or .mdx)
- * @param {string} options.dirname - Directory name
- * @param {string} options.basename - Base filename
- * @param {string} [options.filePath] - Absolute path to file (required for .mdx)
- * @param {string} [options.sourceRoot] - Source root directory (for .mdx absolute imports)
- * @param {boolean} [options.useWorker=true] - Whether to attempt worker thread parsing
- * @param {boolean} [options.hydrate=false] - Whether to enable client-side hydration (.mdx only)
- * @returns {Promise<string|{html: string, hydrationScript?: string}>} Rendered HTML or object with HTML and hydration script
- */
-export async function renderFileAsync({ fileContents, type, dirname, basename, filePath, sourceRoot, useWorker = true, hydrate = false }) {
-  // Wikitext always runs on main thread due to complex ES module dependencies
-  if (type === ".txt") {
-    return wikiToHtml({
-      wikitext: fileContents,
-      articleName: basename,
-      args: { ...DEFAULT_WIKITEXT_ARGS, db: dirname },
-    })?.html;
-  }
-  
-  // Markdown can use worker threads
-  if (type === ".md") {
-    if (useWorker) {
-      return parseWithWorker(
-        fileContents,
-        type,
-        dirname,
-        basename,
-        () => markdownToHtml(fileContents) // Fallback to main thread
-      );
-    }
-    return markdownToHtml(fileContents);
-  }
-
-  // MDX uses mdx-bundler + React SSR (always async, no worker support)
-  // Falls back to markdown rendering if MDX compilation fails
-  if (type === ".mdx") {
-    try {
-      const result = await renderMDX({ source: fileContents, filePath, sourceRoot, hydrate });
-      
-      // If hydration was requested and we have client code, return object with hydration script
-      if (hydrate && result.clientCode) {
-        return {
-          html: result.html,
-          hydrationScript: generateHydrationScript(result.clientCode),
-        };
-      }
-      
-      return result.html;
-    } catch (mdxError) {
-      // Extract a concise error description for the warning banner
-      const errorMsg = mdxError.message || String(mdxError);
-      const shortPath = filePath?.split('/').slice(-3).join('/') || 'unknown file';
-      
-      // Extract the actionable part of the error (skip the "MDX compilation failed for <path>:" preamble)
-      const lines = errorMsg.split('\n').filter(l => l.trim());
-      const actionableLine = lines.length > 1 ? lines.slice(1).join(' ').trim() : lines[0];
-      const errorDetail = actionableLine.slice(0, 300);
-
-      console.warn(`⚠️  MDX compilation failed for ${shortPath}, falling back to Markdown rendering`);
-      console.warn(`   ${errorDetail}`);
-
-      // Render as markdown instead (handles raw HTML fine, just no custom components)
-      const markdownHtml = markdownToHtml(fileContents);
-      const warningBanner = `<div style="background:#fef3cd;border:1px solid #ffc107;color:#856404;padding:0.75rem 1rem;margin-bottom:1rem;border-radius:4px;font-size:0.875rem;">`
-        + `<strong>⚠️ MDX compilation error</strong> — this page was rendered as Markdown (custom components like &lt;CharacterCard&gt; will not appear).<br>`
-        + `<code style="font-size:0.8rem;word-break:break-all;">${errorDetail.replace(/</g, '&lt;').replace(/>/g, '&gt;')}</code>`
-        + `</div>`;
-      return warningBanner + markdownHtml;
-    }
-  }
-  
-  return undefined;
-}
-
-// Re-export for cleanup on shutdown
+
+import { markdownToHtml } from "./markdownHelper.cjs";
+import { wikiToHtml } from "./wikitextHelper.js";
+import { renderMDX, generateHydrationScript } from "./mdxRenderer.js";
+import { parseWithWorker, terminateParserPool } from "./parserPool.js";
+
+const DEFAULT_WIKITEXT_ARGS = { db: "noDB", noSection: true, noTOC: true };
+
+/**
+ * Render a file synchronously (legacy/fallback)
+ * Note: .mdx files require async rendering; use renderFileAsync instead.
+ * @param {Object} options - Render options
+ * @param {string} options.fileContents - Raw file content
+ * @param {string} options.type - File extension (.md, .txt)
+ * @param {string} options.dirname - Directory name
+ * @param {string} options.basename - Base filename
+ * @returns {string} Rendered HTML
+ */
+export function renderFile({ fileContents, type, dirname, basename }) {
+  switch (type) {
+    case ".md":
+      return markdownToHtml(fileContents);
+    case ".txt":
+      return wikiToHtml({
+        wikitext: fileContents,
+        articleName: basename,
+        args: { ...DEFAULT_WIKITEXT_ARGS, db: dirname },
+      })?.html;
+  }
+}
+
+/**
+ * Render a file asynchronously using worker threads for parallel processing
+ * Falls back to main thread for wikitext or if workers are unavailable
+ * @param {Object} options - Render options
+ * @param {string} options.fileContents - Raw file content
+ * @param {string} options.type - File extension (.md, .txt, or .mdx)
+ * @param {string} options.dirname - Directory name
+ * @param {string} options.basename - Base filename
+ * @param {string} [options.filePath] - Absolute path to file (required for .mdx)
+ * @param {string} [options.sourceRoot] - Source root directory (for .mdx absolute imports)
+ * @param {boolean} [options.useWorker=true] - Whether to attempt worker thread parsing
+ * @param {boolean} [options.hydrate=false] - Whether to enable client-side hydration (.mdx only)
+ * @returns {Promise<string|{html: string, hydrationScript?: string}>} Rendered HTML or object with HTML and hydration script
+ */
+export async function renderFileAsync({ fileContents, type, dirname, basename, filePath, sourceRoot, useWorker = true, hydrate = false }) {
+  // Wikitext always runs on main thread due to complex ES module dependencies
+  if (type === ".txt") {
+    return wikiToHtml({
+      wikitext: fileContents,
+      articleName: basename,
+      args: { ...DEFAULT_WIKITEXT_ARGS, db: dirname },
+    })?.html;
+  }
+  
+  // Markdown can use worker threads
+  if (type === ".md") {
+    if (useWorker) {
+      return parseWithWorker(
+        fileContents,
+        type,
+        dirname,
+        basename,
+        () => markdownToHtml(fileContents) // Fallback to main thread
+      );
+    }
+    return markdownToHtml(fileContents);
+  }
+
+  // MDX uses mdx-bundler + React SSR (always async, no worker support)
+  // Falls back to markdown rendering if MDX compilation fails.
+  // Always returns the object form: { html, hydrationScript, inputs, failed }.
+  // `inputs` lists the module files the bundle loaded; `failed` is set when the
+  // fallback rendered, so callers know the dependency list is incomplete.
+  if (type === ".mdx") {
+    try {
+      const result = await renderMDX({ source: fileContents, filePath, sourceRoot, hydrate });
+      
+      // If hydration was requested and we have client code, include the hydration script
+      return {
+        html: result.html,
+        hydrationScript: hydrate && result.clientCode ? generateHydrationScript(result.clientCode) : '',
+        inputs: result.inputs ?? [],
+        failed: false,
+      };
+    } catch (mdxError) {
+      // Extract a concise error description for the warning banner
+      const errorMsg = mdxError.message || String(mdxError);
+      const shortPath = filePath?.split('/').slice(-3).join('/') || 'unknown file';
+      
+      // Extract the actionable part of the error (skip the "MDX compilation failed for <path>:" preamble)
+      const lines = errorMsg.split('\n').filter(l => l.trim());
+      const actionableLine = lines.length > 1 ? lines.slice(1).join(' ').trim() : lines[0];
+      const errorDetail = actionableLine.slice(0, 300);
+
+      console.warn(`⚠️  MDX compilation failed for ${shortPath}, falling back to Markdown rendering`);
+      console.warn(`   ${errorDetail}`);
+
+      // Render as markdown instead (handles raw HTML fine, just no custom components)
+      const markdownHtml = markdownToHtml(fileContents);
+      const warningBanner = `<div style="background:#fef3cd;border:1px solid #ffc107;color:#856404;padding:0.75rem 1rem;margin-bottom:1rem;border-radius:4px;font-size:0.875rem;">`
+        + `<strong>⚠️ MDX compilation error</strong> — this page was rendered as Markdown (custom components like &lt;CharacterCard&gt; will not appear).<br>`
+        + `<code style="font-size:0.8rem;word-break:break-all;">${errorDetail.replace(/</g, '&lt;').replace(/>/g, '&gt;')}</code>`
+        + `</div>`;
+      return {
+        html: warningBanner + markdownHtml,
+        hydrationScript: '',
+        inputs: mdxError.inputs ?? [],
+        componentDirs: mdxError.componentDirs ?? [],
+        failed: true,
+        error: errorDetail,
+      };
+    }
+  }
+  
+  return undefined;
+}
+
+// Re-export for cleanup on shutdown
 export { terminateParserPool };

package/src/helper/findScriptJs.js

@@ -1,5 +1,5 @@
 import { join, dirname, resolve } from "path";
-import { existsSync } from "fs";
+import { existsSync } from "./build/tracedFs.js";
 
 /**
  * Recursively search for script.js or _script.js up the directory tree.

package/src/helper/findStyleCss.js

@@ -1,5 +1,5 @@
 import { join, dirname, resolve } from "path";
-import { existsSync } from "fs";
+import { existsSync } from "./build/tracedFs.js";
 
 /**
  * Recursively search for style.css or _style.css up the directory tree.

package/src/helper/folderConfig.js

@@ -1,11 +1,8 @@
-import { existsSync, readFileSync } from 'fs';
+import { existsSync, readFileSync } from './build/tracedFs.js';
 import { join, dirname } from 'path';
 
 const CONFIG_FILENAME = 'config.json';
 
-// Cache for folder configs to avoid repeated file reads
-const configCache = new Map();
-
 /**
  * Folder configuration schema:
  * {
@@ -24,11 +21,12 @@ const configCache = new Map();
  */
 
 /**
- * Clear the config cache (useful between generation runs)
+ * Kept for callers that used to reset the per-run cache. There is no cache
+ * any more: every read goes to the (traced) filesystem so that the build
+ * graph records config.json as an input of whatever consulted it. A cached
+ * hit would record nothing, and a `hidden: true` flip would go unnoticed.
  */
-export function clearConfigCache() {
-  configCache.clear();
-}
+export function clearConfigCache() {}
 
 /**
  * Read and parse a folder's config.json if it exists (synchronous)
@@ -36,24 +34,15 @@ export function clearConfigCache() {
  * @returns {object|null} Parsed config object or null if not found
  */
 export function getFolderConfig(folderPath) {
-  // Check cache first
-  if (configCache.has(folderPath)) {
-    return configCache.get(folderPath);
-  }
-  
   const configPath = join(folderPath, CONFIG_FILENAME);
   try {
     if (existsSync(configPath)) {
       const content = readFileSync(configPath, 'utf8');
-      const config = JSON.parse(content);
-      configCache.set(folderPath, config);
-      return config;
+      return JSON.parse(content);
     }
   } catch (e) {
     console.warn(`Could not read folder config at ${configPath}:`, e.message);
   }
-  
-  configCache.set(folderPath, null);
   return null;
 }