@kenjura/ursa 0.85.0 → 0.87.1

package/src/jobs/generate.js

@@ -32,7 +32,7 @@ import { findStyleCss, findAllStyleCss } from "../helper/findStyleCss.js";
 import { findScriptJs, findAllScriptJs } from "../helper/findScriptJs.js";
 import { bundleMetaTemplateAssets, bundleDocumentCss, bundleDocumentJs, clearMetaBundleCache, generateSeparateCssTags, generateSeparateJsTags } from "../helper/assetBundler.js";
 import { buildFullTextIndex, buildIncrementalIndex, loadIndexCache, saveIndexCache } from "../helper/fullTextIndex.js";
-import { dependencyTracker } from "../helper/dependencyTracker.js";
+import { dependencyTracker, loadDependencyTracker, saveDependencyTracker } from "../helper/dependencyTracker.js";
 import { CacheBustHashMap } from "../helper/build/cacheBust.js";
 import { copy as copyDir, emptyDir, outputFile, remove } from "fs-extra";
 import { basename, dirname, extname, join, parse, resolve } from "path";
@@ -74,11 +74,13 @@ import {
   getCustomMenuForFile,
   getTransformedMetadata,
   getFooter,
+  getUrsaMetadata,
   generateAutoIndices,
   generateAutoIndexHtmlFromSource,
   copyMetaAssets,
 } from "../helper/build/index.js";
 import { getProfiler } from "../helper/build/profiler.js";
+import { reconcileAll, isInsideTemplatesFolder } from "../helper/documentTemplates.js";
 
 // Concurrency limiter for batch processing to avoid memory exhaustion
 const BATCH_SIZE = parseInt(process.env.URSA_BATCH_SIZE || '50', 10);
@@ -164,6 +166,14 @@ export async function generate({
     progress.logTimed(`Clean build: clearing output directory ${output}`);
     await emptyDir(output);
     progress.logTimed(`Clean complete [${progress.stopTimer('Clean')}]`);
+  } else {
+    // Warm start: reload persisted dependency registrations so hash-skipped
+    // documents keep their edges (current-run registrations take precedence)
+    const loaded = await loadDependencyTracker(source);
+    if (loaded) {
+      const stats = dependencyTracker.getStats();
+      progress.logTimed(`Dependency graph loaded: ${stats.totalDocuments} documents, ${stats.uniqueFiles} dependencies`);
+    }
   }
 
   // Phase: Scan source files
@@ -210,7 +220,7 @@ export async function generate({
 
   // read all articles, process them, copy them to build
   const articleExtensions = /\.(md|mdx|txt|yml)$/;
-  const hiddenOrSystemDirs = /[\/\\]\.(?!\.)|[\/\\]node_modules[\/\\]/;  // Matches hidden folders (starting with .) or node_modules
+  const hiddenOrSystemDirs = /[\/\\]\.(?!\.)|[\/\\]node_modules[\/\\]|[\/\\]_templates[\/\\]|[\/\\]_templates$/;  // Matches hidden folders (starting with .), node_modules, or _templates
   const allSourceFilenamesThatAreArticles = allSourceFilenames.filter(
     (filename) => filename.match(articleExtensions) && !filename.match(hiddenOrSystemDirs) && !isInHiddenFolder(filename)
   );
@@ -230,6 +240,44 @@ export async function generate({
   progress.logTimed(`Classified: ${allSourceFilenamesThatAreArticles.length} articles, ${allSourceFilenamesThatAreDirectories.length} dirs, ${existingHtmlFiles.size} HTML [${progress.stopTimer('Filter')}]`);
   profiler.endPhase('Filter & classify');
 
+  // Drop persisted dependency registrations for documents that no longer
+  // exist (or are excluded), so stale entries don't accumulate across runs
+  dependencyTracker.prune(new Set(allSourceFilenamesThatAreArticles));
+
+  // Phase: Document template reconciliation
+  // Must run BEFORE article processing so that any template-driven changes
+  // to source .md files are picked up during rendering.
+  profiler.startPhase('Template reconciliation');
+  progress.startTimer('Templates');
+  const templateReconciliation = await reconcileAll(
+    allSourceFilenamesThatAreArticles,
+    allSourceFilenamesUnfiltered,   // templates live in _templates which is filtered out of articles
+    source
+  );
+  if (templateReconciliation.updated > 0 || templateReconciliation.conflicts > 0 || templateReconciliation.initialized > 0) {
+    progress.logTimed(
+      `📄 Document templates: ${templateReconciliation.initialized} initialized, ` +
+      `${templateReconciliation.updated} auto-merged, ` +
+      `${templateReconciliation.conflicts} conflicts, ` +
+      `${templateReconciliation.unchanged} unchanged, ` +
+      `${templateReconciliation.errors} errors`
+    );
+    if (templateReconciliation.conflicts > 0) {
+      console.warn(`\n⚠️  Template conflicts require manual resolution:`);
+      for (const msg of templateReconciliation.messages) {
+        if (msg.includes('Conflict')) console.warn(`   ${msg}`);
+      }
+      console.warn('');
+    }
+    if (templateReconciliation.errors > 0) {
+      for (const msg of templateReconciliation.messages) {
+        if (msg.includes('Error') || msg.includes('not found')) console.warn(`   ⚠️  ${msg}`);
+      }
+    }
+  }
+  progress.logTimed(`Document templates processed [${progress.stopTimer('Templates')}]`);
+  profiler.endPhase('Template reconciliation');
+
   // Phase: Build navigation and metadata
   profiler.startPhase('Build navigation');
   progress.startTimer('Navigation');
@@ -301,6 +349,9 @@ export async function generate({
   
   profiler.endPhase('Build navigation');
 
+  // Build the _ursa_metadata embedded in every generated JSON file (ursa + doc repo versions)
+  const ursaMetadata = await getUrsaMetadata(_source);
+
   // Phase: Load cache
   profiler.startPhase('Load cache');
   progress.startTimer('Cache');
@@ -864,6 +915,7 @@ export async function generate({
         metadata: fileMeta,
         sections,
         transformedMetadata: jsonTransformedMeta,
+        _ursa_metadata: ursaMetadata,
       };
       
       // Store minimal data for directory indices, including metadata
@@ -1159,6 +1211,10 @@ export async function generate({
     progress.log(`Saved ${contentTimestamps.size} content timestamps`);
   }
 
+  // Persist the dependency tracker so hash-skipped documents keep their
+  // edges on the next warm start (invalidation plans stay accurate)
+  await saveDependencyTracker(source);
+
   // Populate watch mode cache for fast single-file regeneration
   watchModeCache.templates = templates;
   watchModeCache.menu = menu;
@@ -1173,6 +1229,7 @@ export async function generate({
   watchModeCache.allArticlePaths = [...allSourceFilenamesThatAreArticles];
   watchModeCache.imageMap = imageMap;
   watchModeCache.customMenus = customMenus;
+  watchModeCache.ursaMetadata = ursaMetadata;
   watchModeCache.lastFullBuild = Date.now();
   watchModeCache.isInitialized = true;
   const depStats = dependencyTracker.getStats();
@@ -1323,6 +1380,11 @@ export async function regenerateAffectedDocuments(documentPaths, {
     }
   }
 
+  // Persist updated dependency registrations (e.g. a doc switched templates)
+  if (regenerated > 0) {
+    await saveDependencyTracker(resolve(_source) + "/");
+  }
+
   const elapsed = Date.now() - startTime;
   const msg = `Regenerated ${regenerated}/${documentPaths.length} documents in ${elapsed}ms (${reason})${failed > 0 ? `, ${failed} failed` : ""}`;
   console.log(`✅ ${msg}`);
@@ -1364,8 +1426,8 @@ export async function regenerateSingleFile(changedFile, {
   }
   
   try {
-    const { templates, menu, footer, validPaths, hashCache, cacheBustTimestamp, imageMap, customMenus } = watchModeCache;
-    
+    const { templates, menu, footer, validPaths, hashCache, cacheBustTimestamp, imageMap, customMenus, ursaMetadata } = watchModeCache;
+
     const rawBody = await readFile(changedFile, "utf8");
     const type = parse(changedFile).ext;
     const ext = extname(changedFile);
@@ -1464,9 +1526,10 @@ export async function regenerateSingleFile(changedFile, {
     // Find all CSS files up the tree and create separate link tags
     // (regenerateSingleFile is used in serve mode, so separate tags per level for invalidation)
     let styleLink = "";
+    let cssPaths = [];
     try {
       const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-      const cssPaths = await findAllStyleCss(dirKey, _source);
+      cssPaths = await findAllStyleCss(dirKey, _source);
       if (cssPaths.length > 0) {
         // Copy all CSS files to output (always copy in single-file mode to ensure up to date)
         for (const cssPath of cssPaths) {
@@ -1492,9 +1555,10 @@ export async function regenerateSingleFile(changedFile, {
     // Find all script.js files from docroot to current dir and serve as separate external tags
     // (Serve mode: separate tags per level for individual invalidation)
     let customScript = "";
+    let scriptPaths = [];
     try {
       const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-      const scriptPaths = await findAllScriptJs(dirKey, _source);
+      scriptPaths = await findAllScriptJs(dirKey, _source);
       if (scriptPaths.length > 0) {
         // Copy all script files to output so they can be served
         for (const scriptPath of scriptPaths) {
@@ -1508,6 +1572,18 @@ export async function regenerateSingleFile(changedFile, {
       // ignore
     }
 
+    // Register this document's dependencies (template, inherited CSS/JS) so
+    // invalidation plans stay accurate after frontmatter/template changes.
+    // Persisted to .ursa/dependency-graph.json by regenerateAffectedDocuments.
+    const usedTemplateName = (requestedTemplateName && templates[requestedTemplateName])
+      ? requestedTemplateName
+      : DEFAULT_TEMPLATE_NAME;
+    dependencyTracker.registerDocument(changedFile, {
+      templateName: usedTemplateName,
+      cssPaths,
+      scriptPaths,
+    });
+
     // Check if this file has a custom menu
     const customMenuInfo = customMenus ? getCustomMenuForFile(changedFile, source, customMenus) : null;
 
@@ -1575,6 +1651,7 @@ export async function regenerateSingleFile(changedFile, {
       metadata: fileMeta,
       sections,
       transformedMetadata,
+      _ursa_metadata: ursaMetadata,
     };
     const json = JSON.stringify(jsonObject);
     await outputFile(jsonOutputFilename, json);

package/src/serve.js

@@ -11,6 +11,7 @@ import { watchModeCache } from "./helper/build/watchCache.js";
 import { dependencyTracker } from "./helper/dependencyTracker.js";
 import { bundleMetaTemplateAssets, clearMetaBundleCache } from "./helper/assetBundler.js";
 import { getTemplates, copyMetaAssets } from "./helper/build/templates.js";
+import { isInsideTemplatesFolder, reconcileByTemplate } from "./helper/documentTemplates.js";
 import { WebSocketServer } from "ws";
 import { createServer } from "http";
 import { resolvePort } from "./helper/portUtils.js";
@@ -270,6 +271,11 @@ const DEBOUNCE_MS = 500; // Wait 500ms of quiet before starting regeneration
 let pendingChanges = [];    // { evt, name, watcher: 'source'|'meta' }
 let debounceTimer = null;
 
+// Changes that arrived while a regeneration pass was in flight.
+// They are processed as the next batch when the current pass finishes —
+// never dropped (passes run sequentially, single-writer).
+let queuedDuringRegeneration = [];
+
 /**
  * Copy a single CSS file to the output directory
  * @param {string} cssPath - Absolute path to the CSS file
@@ -434,8 +440,10 @@ export async function serve({
    */
   async function processChangeBatch(batch, sourceDir, metaDir, outputDir, _whitelist, _exclude) {
     if (isRegenerating) {
-      console.log(`⏳ Debounce batch skipped (regeneration already in progress) — ${batch.length} changes lost`);
-      broadcastMessage({ type: 'update-no-affect', timestamp: Date.now() });
+      // Never drop changes: accumulate them and process when the current
+      // pass finishes (clients keep their update-start indicator until then)
+      queuedDuringRegeneration.push(...batch);
+      console.log(`⏳ Regeneration in progress — queued ${batch.length} change(s) for the next pass`);
       return;
     }
     isRegenerating = true;
@@ -443,6 +451,7 @@ export async function serve({
     try {
       // Categorize changes
       const metaChanges = batch.filter(c => c.watcher === 'meta');
+      const metaStaticChanges = metaChanges.filter(c => c.name && STATIC_FILE_EXTENSIONS.test(c.name));
       const sourceChanges = batch.filter(c => c.watcher === 'source');
 
       const cssChanges = sourceChanges.filter(c => c.name?.endsWith('.css'));
@@ -547,15 +556,34 @@ export async function serve({
       if (menuConfigChanges.length > 0) {
         needsFullRebuild = true;
         fullRebuildReason = `Menu/config change: ${menuConfigChanges.map(c => basename(c.name)).join(', ')}`;
-        // Delete on-disk caches to force full navigation + content rebuild
-        const ursaDir = join(sourceDir, '.ursa');
-        try { await promises.unlink(join(ursaDir, 'content-hashes.json')); } catch {}
-        try { await promises.unlink(join(ursaDir, 'nav-cache.json')); } catch {}
+      }
+
+      // --- 5.5) Handle document template changes ---
+      // If a _templates/*.md file changed, reconcile all documents using that template
+      // and add the affected instance documents to the regeneration set.
+      const templateChanges = articleChanges.filter(c => c.name && isInsideTemplatesFolder(c.name));
+      if (templateChanges.length > 0 && watchModeCache.isInitialized) {
+        const allArticles = watchModeCache.allArticlePaths || [];
+        for (const change of templateChanges) {
+          console.log(`📄 Document template changed: ${basename(change.name)}`);
+          const reconcileResult = await reconcileByTemplate(change.name, allArticles, sourceDir);
+          if (reconcileResult.updated > 0 || reconcileResult.conflicts > 0) {
+            console.log(`   ${reconcileResult.updated} auto-merged, ${reconcileResult.conflicts} conflicts`);
+            reconcileResult.affectedPaths.forEach(p => affectedDocPaths.add(p));
+          }
+          if (reconcileResult.conflicts > 0) {
+            for (const msg of reconcileResult.messages) {
+              if (msg.includes('Conflict')) console.warn(`   ⚠️  ${msg}`);
+            }
+          }
+        }
       }
 
       // --- 6) Handle article changes via fast single-file regen ---
       // Deduplicate articles (same file may appear multiple times in rapid saves)
-      const uniqueArticles = [...new Set(articleChanges.map(c => c.name))];
+      // Exclude _templates files from direct article regeneration (they aren't rendered)
+      const uniqueArticles = [...new Set(articleChanges.map(c => c.name))]
+        .filter(name => !isInsideTemplatesFolder(name));
       for (const articlePath of uniqueArticles) {
         affectedDocPaths.add(articlePath);
       }
@@ -569,6 +597,13 @@ export async function serve({
       // --- 8) Execute rebuild ---
       if (needsFullRebuild) {
         console.log(`📦 Full rebuild required: ${fullRebuildReason}`);
+        // Delete on-disk caches on EVERY full-rebuild path (not just menu/config):
+        // the content-hash skip only looks at article markdown, so without this a
+        // rebuild after a template/meta change would skip every unchanged article
+        // and leave stale HTML (see docs/changes/serve-logic.md, root cause #2)
+        const ursaDir = join(sourceDir, '.ursa');
+        try { await promises.unlink(join(ursaDir, 'content-hashes.json')); } catch {}
+        try { await promises.unlink(join(ursaDir, 'nav-cache.json')); } catch {}
         clearWatchCache();
         try {
           const result = await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _exclude, _deferImages: true, _deferSearchIndex: true });
@@ -666,7 +701,8 @@ export async function serve({
         }
       } else {
         // No documents affected (e.g. static-only changes) — reload all clients
-        if (staticChanges.length > 0) {
+        // (meta static assets were re-copied to output/public in step 4)
+        if (staticChanges.length > 0 || metaStaticChanges.length > 0) {
           broadcastReload(uniqueNames[0]);
         } else {
           // Nothing to do — clear indicators
@@ -680,11 +716,20 @@ export async function serve({
       broadcastReload();
     } finally {
       isRegenerating = false;
+      // Process changes that arrived during this pass (sequentially, never dropped)
+      if (queuedDuringRegeneration.length > 0) {
+        const nextBatch = queuedDuringRegeneration.splice(0);
+        console.log(`▶️  Processing ${nextBatch.length} change(s) queued during the last pass`);
+        setImmediate(() => processChangeBatch(nextBatch, sourceDir, metaDir, outputDir, _whitelist, _exclude));
+      }
     }
   }
   
-  // Meta changes: queue for debounced batch processing
-  watch(metaDir, { recursive: true, filter: /\.(js|json|css|html|md|txt|yml|yaml)$/ }, (evt, name) => {
+  // Meta changes: queue for debounced batch processing.
+  // Includes static asset extensions (images, fonts, media, PDFs) so that
+  // replacing e.g. a PNG or font in meta/ re-runs copyMetaAssets + re-bundling
+  // instead of being invisible to the watcher.
+  watch(metaDir, { recursive: true, filter: /\.(js|json|css|html|md|txt|yml|yaml|jpg|jpeg|png|gif|webp|svg|ico|woff|woff2|ttf|eot|pdf|mp3|mp4|webm|ogg)$/i }, (evt, name) => {
     queueChange(evt, name, 'meta');
   });
 
@@ -721,6 +766,17 @@ function serveFiles(outputDir, port = 8080) {
     level: 6
   }));
 
+  // Add ursa-version and doc-version headers to all JSON responses
+  // (per-document JSON, directory index arrays, and public/*.json index files)
+  app.use((req, res, next) => {
+    if (req.path.endsWith('.json')) {
+      const meta = watchModeCache.ursaMetadata || {};
+      res.setHeader('X-ursa-version', meta.ursaVersion || 'unknown');
+      res.setHeader('X-doc-version', meta.docVersion || 'unknown');
+    }
+    next();
+  });
+
   // Middleware to inject hot reload script into HTML responses
   app.use(async (req, res, next) => {
     // Only intercept HTML requests