@kenjura/ursa 0.86.0 → 0.87.1

package/src/helper/dependencyTracker.js

@@ -17,6 +17,16 @@
  */
 
 import { dirname, join, relative, resolve } from "path";
+import { existsSync } from "fs";
+import { mkdir, readFile, writeFile } from "fs/promises";
+import { getUrsaDir } from "./contentHash.js";
+
+const DEP_GRAPH_FILE = "dependency-graph.json";
+const DEP_GRAPH_VERSION = 1;
+
+// Static assets in meta (copied verbatim to output/public by copyMetaAssets);
+// these are never embedded in document HTML, so no regeneration is needed.
+const META_STATIC_EXTENSIONS = /\.(jpg|jpeg|png|gif|webp|svg|ico|woff|woff2|ttf|eot|pdf|mp3|mp4|webm|ogg)$/i;
 
 export class DependencyTracker {
   constructor() {
@@ -216,7 +226,13 @@ export class DependencyTracker {
 
     // Template file changed → regenerate all documents using that template
     if (fileName.endsWith(".html")) {
-      const templateName = fileName.replace(".html", "");
+      // New structure: templates/{templateName}/index.html → name is the folder;
+      // legacy flat structure: {templateName}.html at the meta root
+      const parts = relativePath.split("/");
+      const templateName =
+        parts[0] === "templates" && parts.length >= 3
+          ? parts[1]
+          : fileName.replace(".html", "");
       const affected = this.getDocumentsUsingTemplate(templateName);
       if (affected.size > 0) {
         return {
@@ -244,6 +260,17 @@ export class DependencyTracker {
       };
     }
 
+    // Static asset in meta (image, font, PDF, media) → copyMetaAssets already
+    // re-copied it to output/public; documents reference it by URL, so no
+    // document regeneration (and no full rebuild) is needed.
+    if (META_STATIC_EXTENSIONS.test(fileName)) {
+      return {
+        affectedDocuments: [],
+        reason: `Meta static asset copied: ${relativePath}`,
+        requiresFullRebuild: false,
+      };
+    }
+
     // Other meta file → full rebuild to be safe
     return {
       affectedDocuments: [],
@@ -263,7 +290,95 @@ export class DependencyTracker {
       uniqueFiles: this.fileToDocuments.size,
     };
   }
+
+  /**
+   * Serialize the tracker for persistence to .ursa/dependency-graph.json.
+   * @returns {{ version: number, sourceDir: string, documents: Object<string, string[]> }}
+   */
+  serialize() {
+    return {
+      version: DEP_GRAPH_VERSION,
+      sourceDir: this.sourceDir,
+      documents: Object.fromEntries(
+        [...this.documentToFiles.entries()].map(([doc, deps]) => [doc, [...deps]])
+      ),
+    };
+  }
+
+  /**
+   * Load persisted registrations, merging with the current run: documents
+   * already registered in this run keep their (fresher) edges; persisted
+   * edges only fill in documents not yet registered (e.g. hash-skipped docs).
+   * Rejects data from a different source directory or schema version.
+   * @param {object} data - Previously serialized tracker
+   * @returns {boolean} Whether the data was loaded
+   */
+  load(data) {
+    if (!data || data.version !== DEP_GRAPH_VERSION) return false;
+    if (data.sourceDir && this.sourceDir && data.sourceDir !== this.sourceDir) return false;
+    for (const [doc, deps] of Object.entries(data.documents || {})) {
+      if (this.documentToFiles.has(doc)) continue; // live registrations win
+      if (!Array.isArray(deps)) continue;
+      for (const dep of deps) {
+        this.addDependency(doc, dep);
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Drop registrations for documents not in the given set (e.g. deleted or
+   * excluded files), so persisted state doesn't accumulate stale entries.
+   * @param {Set<string>} keepDocuments - Document paths that should survive
+   */
+  prune(keepDocuments) {
+    for (const doc of [...this.documentToFiles.keys()]) {
+      if (!keepDocuments.has(doc)) this.clearDocument(doc);
+    }
+  }
 }
 
 // Singleton instance
 export const dependencyTracker = new DependencyTracker();
+
+/** Path to the persisted dependency graph for a source directory. */
+export function getDependencyGraphPath(sourceDir) {
+  return join(getUrsaDir(sourceDir), DEP_GRAPH_FILE);
+}
+
+/**
+ * Load the persisted dependency tracker state from .ursa/dependency-graph.json
+ * and merge it into the tracker (current-run registrations win).
+ * @param {string} sourceDir - Source directory root
+ * @param {DependencyTracker} [tracker] - Defaults to the singleton
+ * @returns {Promise<boolean>} Whether a valid graph was loaded
+ */
+export async function loadDependencyTracker(sourceDir, tracker = dependencyTracker) {
+  const path = getDependencyGraphPath(sourceDir);
+  try {
+    if (!existsSync(path)) return false;
+    const data = JSON.parse(await readFile(path, "utf8"));
+    return tracker.load(data);
+  } catch (e) {
+    console.warn(`Could not load dependency graph: ${e.message}`);
+    return false;
+  }
+}
+
+/**
+ * Persist the dependency tracker to .ursa/dependency-graph.json so that
+ * hash-skipped documents keep their edges across warm starts.
+ * @param {string} sourceDir - Source directory root
+ * @param {DependencyTracker} [tracker] - Defaults to the singleton
+ * @returns {Promise<boolean>} Whether the graph was saved
+ */
+export async function saveDependencyTracker(sourceDir, tracker = dependencyTracker) {
+  try {
+    await mkdir(getUrsaDir(sourceDir), { recursive: true });
+    await writeFile(getDependencyGraphPath(sourceDir), JSON.stringify(tracker.serialize()));
+    return true;
+  } catch (e) {
+    console.warn(`Could not save dependency graph: ${e.message}`);
+    return false;
+  }
+}

package/src/jobs/generate.js

@@ -32,7 +32,7 @@ import { findStyleCss, findAllStyleCss } from "../helper/findStyleCss.js";
 import { findScriptJs, findAllScriptJs } from "../helper/findScriptJs.js";
 import { bundleMetaTemplateAssets, bundleDocumentCss, bundleDocumentJs, clearMetaBundleCache, generateSeparateCssTags, generateSeparateJsTags } from "../helper/assetBundler.js";
 import { buildFullTextIndex, buildIncrementalIndex, loadIndexCache, saveIndexCache } from "../helper/fullTextIndex.js";
-import { dependencyTracker } from "../helper/dependencyTracker.js";
+import { dependencyTracker, loadDependencyTracker, saveDependencyTracker } from "../helper/dependencyTracker.js";
 import { CacheBustHashMap } from "../helper/build/cacheBust.js";
 import { copy as copyDir, emptyDir, outputFile, remove } from "fs-extra";
 import { basename, dirname, extname, join, parse, resolve } from "path";
@@ -74,6 +74,7 @@ import {
   getCustomMenuForFile,
   getTransformedMetadata,
   getFooter,
+  getUrsaMetadata,
   generateAutoIndices,
   generateAutoIndexHtmlFromSource,
   copyMetaAssets,
@@ -165,6 +166,14 @@ export async function generate({
     progress.logTimed(`Clean build: clearing output directory ${output}`);
     await emptyDir(output);
     progress.logTimed(`Clean complete [${progress.stopTimer('Clean')}]`);
+  } else {
+    // Warm start: reload persisted dependency registrations so hash-skipped
+    // documents keep their edges (current-run registrations take precedence)
+    const loaded = await loadDependencyTracker(source);
+    if (loaded) {
+      const stats = dependencyTracker.getStats();
+      progress.logTimed(`Dependency graph loaded: ${stats.totalDocuments} documents, ${stats.uniqueFiles} dependencies`);
+    }
   }
 
   // Phase: Scan source files
@@ -231,6 +240,10 @@ export async function generate({
   progress.logTimed(`Classified: ${allSourceFilenamesThatAreArticles.length} articles, ${allSourceFilenamesThatAreDirectories.length} dirs, ${existingHtmlFiles.size} HTML [${progress.stopTimer('Filter')}]`);
   profiler.endPhase('Filter & classify');
 
+  // Drop persisted dependency registrations for documents that no longer
+  // exist (or are excluded), so stale entries don't accumulate across runs
+  dependencyTracker.prune(new Set(allSourceFilenamesThatAreArticles));
+
   // Phase: Document template reconciliation
   // Must run BEFORE article processing so that any template-driven changes
   // to source .md files are picked up during rendering.
@@ -336,6 +349,9 @@ export async function generate({
   
   profiler.endPhase('Build navigation');
 
+  // Build the _ursa_metadata embedded in every generated JSON file (ursa + doc repo versions)
+  const ursaMetadata = await getUrsaMetadata(_source);
+
   // Phase: Load cache
   profiler.startPhase('Load cache');
   progress.startTimer('Cache');
@@ -899,6 +915,7 @@ export async function generate({
         metadata: fileMeta,
         sections,
         transformedMetadata: jsonTransformedMeta,
+        _ursa_metadata: ursaMetadata,
       };
       
       // Store minimal data for directory indices, including metadata
@@ -1194,6 +1211,10 @@ export async function generate({
     progress.log(`Saved ${contentTimestamps.size} content timestamps`);
   }
 
+  // Persist the dependency tracker so hash-skipped documents keep their
+  // edges on the next warm start (invalidation plans stay accurate)
+  await saveDependencyTracker(source);
+
   // Populate watch mode cache for fast single-file regeneration
   watchModeCache.templates = templates;
   watchModeCache.menu = menu;
@@ -1208,6 +1229,7 @@ export async function generate({
   watchModeCache.allArticlePaths = [...allSourceFilenamesThatAreArticles];
   watchModeCache.imageMap = imageMap;
   watchModeCache.customMenus = customMenus;
+  watchModeCache.ursaMetadata = ursaMetadata;
   watchModeCache.lastFullBuild = Date.now();
   watchModeCache.isInitialized = true;
   const depStats = dependencyTracker.getStats();
@@ -1358,6 +1380,11 @@ export async function regenerateAffectedDocuments(documentPaths, {
     }
   }
 
+  // Persist updated dependency registrations (e.g. a doc switched templates)
+  if (regenerated > 0) {
+    await saveDependencyTracker(resolve(_source) + "/");
+  }
+
   const elapsed = Date.now() - startTime;
   const msg = `Regenerated ${regenerated}/${documentPaths.length} documents in ${elapsed}ms (${reason})${failed > 0 ? `, ${failed} failed` : ""}`;
   console.log(`✅ ${msg}`);
@@ -1399,8 +1426,8 @@ export async function regenerateSingleFile(changedFile, {
   }
   
   try {
-    const { templates, menu, footer, validPaths, hashCache, cacheBustTimestamp, imageMap, customMenus } = watchModeCache;
-    
+    const { templates, menu, footer, validPaths, hashCache, cacheBustTimestamp, imageMap, customMenus, ursaMetadata } = watchModeCache;
+
     const rawBody = await readFile(changedFile, "utf8");
     const type = parse(changedFile).ext;
     const ext = extname(changedFile);
@@ -1499,9 +1526,10 @@ export async function regenerateSingleFile(changedFile, {
     // Find all CSS files up the tree and create separate link tags
     // (regenerateSingleFile is used in serve mode, so separate tags per level for invalidation)
     let styleLink = "";
+    let cssPaths = [];
     try {
       const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-      const cssPaths = await findAllStyleCss(dirKey, _source);
+      cssPaths = await findAllStyleCss(dirKey, _source);
       if (cssPaths.length > 0) {
         // Copy all CSS files to output (always copy in single-file mode to ensure up to date)
         for (const cssPath of cssPaths) {
@@ -1527,9 +1555,10 @@ export async function regenerateSingleFile(changedFile, {
     // Find all script.js files from docroot to current dir and serve as separate external tags
     // (Serve mode: separate tags per level for individual invalidation)
     let customScript = "";
+    let scriptPaths = [];
     try {
       const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-      const scriptPaths = await findAllScriptJs(dirKey, _source);
+      scriptPaths = await findAllScriptJs(dirKey, _source);
       if (scriptPaths.length > 0) {
         // Copy all script files to output so they can be served
         for (const scriptPath of scriptPaths) {
@@ -1543,6 +1572,18 @@ export async function regenerateSingleFile(changedFile, {
       // ignore
     }
 
+    // Register this document's dependencies (template, inherited CSS/JS) so
+    // invalidation plans stay accurate after frontmatter/template changes.
+    // Persisted to .ursa/dependency-graph.json by regenerateAffectedDocuments.
+    const usedTemplateName = (requestedTemplateName && templates[requestedTemplateName])
+      ? requestedTemplateName
+      : DEFAULT_TEMPLATE_NAME;
+    dependencyTracker.registerDocument(changedFile, {
+      templateName: usedTemplateName,
+      cssPaths,
+      scriptPaths,
+    });
+
     // Check if this file has a custom menu
     const customMenuInfo = customMenus ? getCustomMenuForFile(changedFile, source, customMenus) : null;
 
@@ -1610,6 +1651,7 @@ export async function regenerateSingleFile(changedFile, {
       metadata: fileMeta,
       sections,
       transformedMetadata,
+      _ursa_metadata: ursaMetadata,
     };
     const json = JSON.stringify(jsonObject);
     await outputFile(jsonOutputFilename, json);

package/src/serve.js

@@ -271,6 +271,11 @@ const DEBOUNCE_MS = 500; // Wait 500ms of quiet before starting regeneration
 let pendingChanges = [];    // { evt, name, watcher: 'source'|'meta' }
 let debounceTimer = null;
 
+// Changes that arrived while a regeneration pass was in flight.
+// They are processed as the next batch when the current pass finishes —
+// never dropped (passes run sequentially, single-writer).
+let queuedDuringRegeneration = [];
+
 /**
  * Copy a single CSS file to the output directory
  * @param {string} cssPath - Absolute path to the CSS file
@@ -435,8 +440,10 @@ export async function serve({
    */
   async function processChangeBatch(batch, sourceDir, metaDir, outputDir, _whitelist, _exclude) {
     if (isRegenerating) {
-      console.log(`⏳ Debounce batch skipped (regeneration already in progress) — ${batch.length} changes lost`);
-      broadcastMessage({ type: 'update-no-affect', timestamp: Date.now() });
+      // Never drop changes: accumulate them and process when the current
+      // pass finishes (clients keep their update-start indicator until then)
+      queuedDuringRegeneration.push(...batch);
+      console.log(`⏳ Regeneration in progress — queued ${batch.length} change(s) for the next pass`);
       return;
     }
     isRegenerating = true;
@@ -444,6 +451,7 @@ export async function serve({
     try {
       // Categorize changes
       const metaChanges = batch.filter(c => c.watcher === 'meta');
+      const metaStaticChanges = metaChanges.filter(c => c.name && STATIC_FILE_EXTENSIONS.test(c.name));
       const sourceChanges = batch.filter(c => c.watcher === 'source');
 
       const cssChanges = sourceChanges.filter(c => c.name?.endsWith('.css'));
@@ -548,10 +556,6 @@ export async function serve({
       if (menuConfigChanges.length > 0) {
         needsFullRebuild = true;
         fullRebuildReason = `Menu/config change: ${menuConfigChanges.map(c => basename(c.name)).join(', ')}`;
-        // Delete on-disk caches to force full navigation + content rebuild
-        const ursaDir = join(sourceDir, '.ursa');
-        try { await promises.unlink(join(ursaDir, 'content-hashes.json')); } catch {}
-        try { await promises.unlink(join(ursaDir, 'nav-cache.json')); } catch {}
       }
 
       // --- 5.5) Handle document template changes ---
@@ -593,6 +597,13 @@ export async function serve({
       // --- 8) Execute rebuild ---
       if (needsFullRebuild) {
         console.log(`📦 Full rebuild required: ${fullRebuildReason}`);
+        // Delete on-disk caches on EVERY full-rebuild path (not just menu/config):
+        // the content-hash skip only looks at article markdown, so without this a
+        // rebuild after a template/meta change would skip every unchanged article
+        // and leave stale HTML (see docs/changes/serve-logic.md, root cause #2)
+        const ursaDir = join(sourceDir, '.ursa');
+        try { await promises.unlink(join(ursaDir, 'content-hashes.json')); } catch {}
+        try { await promises.unlink(join(ursaDir, 'nav-cache.json')); } catch {}
         clearWatchCache();
         try {
           const result = await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _exclude, _deferImages: true, _deferSearchIndex: true });
@@ -690,7 +701,8 @@ export async function serve({
         }
       } else {
         // No documents affected (e.g. static-only changes) — reload all clients
-        if (staticChanges.length > 0) {
+        // (meta static assets were re-copied to output/public in step 4)
+        if (staticChanges.length > 0 || metaStaticChanges.length > 0) {
           broadcastReload(uniqueNames[0]);
         } else {
           // Nothing to do — clear indicators
@@ -704,11 +716,20 @@ export async function serve({
       broadcastReload();
     } finally {
       isRegenerating = false;
+      // Process changes that arrived during this pass (sequentially, never dropped)
+      if (queuedDuringRegeneration.length > 0) {
+        const nextBatch = queuedDuringRegeneration.splice(0);
+        console.log(`▶️  Processing ${nextBatch.length} change(s) queued during the last pass`);
+        setImmediate(() => processChangeBatch(nextBatch, sourceDir, metaDir, outputDir, _whitelist, _exclude));
+      }
     }
   }
   
-  // Meta changes: queue for debounced batch processing
-  watch(metaDir, { recursive: true, filter: /\.(js|json|css|html|md|txt|yml|yaml)$/ }, (evt, name) => {
+  // Meta changes: queue for debounced batch processing.
+  // Includes static asset extensions (images, fonts, media, PDFs) so that
+  // replacing e.g. a PNG or font in meta/ re-runs copyMetaAssets + re-bundling
+  // instead of being invisible to the watcher.
+  watch(metaDir, { recursive: true, filter: /\.(js|json|css|html|md|txt|yml|yaml|jpg|jpeg|png|gif|webp|svg|ico|woff|woff2|ttf|eot|pdf|mp3|mp4|webm|ogg)$/i }, (evt, name) => {
     queueChange(evt, name, 'meta');
   });
 
@@ -745,6 +766,17 @@ function serveFiles(outputDir, port = 8080) {
     level: 6
   }));
 
+  // Add ursa-version and doc-version headers to all JSON responses
+  // (per-document JSON, directory index arrays, and public/*.json index files)
+  app.use((req, res, next) => {
+    if (req.path.endsWith('.json')) {
+      const meta = watchModeCache.ursaMetadata || {};
+      res.setHeader('X-ursa-version', meta.ursaVersion || 'unknown');
+      res.setHeader('X-doc-version', meta.docVersion || 'unknown');
+    }
+    next();
+  });
+
   // Middleware to inject hot reload script into HTML responses
   app.use(async (req, res, next) => {
     // Only intercept HTML requests