@kenjura/ursa 0.76.0 → 0.77.0

package/src/jobs/generate.js

@@ -26,9 +26,12 @@ import {
 import { getAndIncrementBuildId } from "../helper/ursaConfig.js";
 import { extractSections } from "../helper/sectionExtractor.js";
 import { renderFile, renderFileAsync, terminateParserPool } from "../helper/fileRenderer.js";
-import { findStyleCss } from "../helper/findStyleCss.js";
+import { findStyleCss, findAllStyleCss } from "../helper/findStyleCss.js";
 import { findScriptJs, findAllScriptJs } from "../helper/findScriptJs.js";
+import { bundleMetaTemplateAssets, bundleDocumentCss, bundleDocumentJs, clearMetaBundleCache, generateSeparateCssTags, generateSeparateJsTags } from "../helper/assetBundler.js";
 import { buildFullTextIndex, buildIncrementalIndex, loadIndexCache, saveIndexCache } from "../helper/fullTextIndex.js";
+import { dependencyTracker } from "../helper/dependencyTracker.js";
+import { CacheBustHashMap } from "../helper/build/cacheBust.js";
 import { copy as copyDir, emptyDir, outputFile } from "fs-extra";
 import { basename, dirname, extname, join, parse, resolve } from "path";
 import { URL } from "url";
@@ -71,6 +74,7 @@ import {
   getFooter,
   generateAutoIndices,
   generateAutoIndexHtmlFromSource,
+  copyMetaAssets,
 } from "../helper/build/index.js";
 import { getProfiler } from "../helper/build/profiler.js";
 
@@ -83,10 +87,15 @@ const cssPathCache = new Map();
 // Cache for script path lookups to avoid repeated filesystem walks
 const scriptPathCache = new Map();
 
+// Cache for document-level CSS/JS bundle paths to avoid re-bundling identical sets
+const docBundleCache = new Map();
+
 // Wrapper for clearWatchCache that passes cssPathCache and scriptPathCache
 export function clearWatchCache() {
   clearWatchCacheBase(cssPathCache);
   scriptPathCache.clear();
+  docBundleCache.clear();
+  clearMetaBundleCache();
 }
 
 const progress = new ProgressReporter();
@@ -116,8 +125,12 @@ export async function generate({
 
   // Generate cache-busting timestamp for this build
   const cacheBustTimestamp = generateCacheBustTimestamp();
+  const cacheBustHashes = new CacheBustHashMap();
   progress.logTimed(`Cache-bust timestamp: ${cacheBustTimestamp}`);
 
+  // Initialize dependency tracker for this build
+  dependencyTracker.init(source);
+
   // Clear output directory when --clean is specified
   if (_clean) {
     progress.startTimer('Clean');
@@ -283,7 +296,27 @@ export async function generate({
   // create public folder
   const pub = join(output, "public");
   await mkdir(pub, { recursive: true });
-  await copyDir(meta, pub);
+  
+  // Copy meta assets with new template folder structure
+  const { copiedFiles, orphanedFiles } = await copyMetaAssets(meta, pub);
+  
+  // Warn about orphaned files in meta that aren't part of any template
+  if (orphanedFiles.length > 0) {
+    console.warn(`\n⚠️  Warning: Found ${orphanedFiles.length} orphaned file(s) in meta directory:`);
+    console.warn(`   These files are not in meta/templates/ or meta/shared/ and won't be included:`);
+    for (const file of orphanedFiles.slice(0, 10)) {
+      console.warn(`   - ${file}`);
+    }
+    if (orphanedFiles.length > 10) {
+      console.warn(`   ... and ${orphanedFiles.length - 10} more`);
+    }
+    console.warn(`   Move them to meta/templates/{templateName}/ or meta/shared/ to include them.\n`);
+  }
+
+  // Bundle meta template assets (CSS + JS) into single files per template
+  // This must happen after copying meta to public but before cache-busting
+  templates = await bundleMetaTemplateAssets(templates, meta, pub, { minify: true, sourcemap: false });
+  progress.logTimed(`Meta template assets bundled`);
 
   // Process all CSS files in the entire output directory tree for cache-busting
   const allOutputFiles = await recurse(output, [() => false]);
@@ -596,65 +629,97 @@ export async function generate({
         }
       }
 
-      // Find nearest style.css or _style.css up the tree and copy to output
-      // Use cache to avoid repeated filesystem walks for same directory
+      // Find all style.css files up the tree and bundle them into a single CSS file per folder path
+      // (Generate mode: one CSS bundle per unique folder, minimizing requests per page load)
       let styleLink = "";
       try {
-        // For root-level files, dir may be "/" which would resolve to filesystem root
-        // Use source directory directly in that case
         const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-        let cssPath = cssPathCache.get(dirKey);
-        if (cssPath === undefined) {
-          cssPath = await findStyleCss(dirKey);
-          cssPathCache.set(dirKey, cssPath); // Cache null results too
-        }
-        if (cssPath) {
-          // Calculate output path for the CSS file (mirrors source structure)
-          const cssOutputPath = cssPath.replace(source, output);
-          const cssUrlPath = '/' + cssPath.replace(source, '');
-          
-          // Copy CSS file if not already copied
-          if (!copiedCssFiles.has(cssPath)) {
-            const cssContent = await readFile(cssPath, 'utf8');
-            await outputFile(cssOutputPath, cssContent);
-            copiedCssFiles.add(cssPath);
+        const folderRelative = (dir === "/" || dir === "") ? "" : dir;
+
+        // Check bundle cache first (dirs with same CSS ancestry share the same bundle)
+        let cachedBundleUrl = docBundleCache.get(`css:${dirKey}`);
+        if (cachedBundleUrl !== undefined) {
+          if (cachedBundleUrl) {
+            styleLink = `<link rel="stylesheet" href="${cachedBundleUrl}" />`;
+          }
+        } else {
+          let cssPaths = cssPathCache.get(dirKey);
+          if (cssPaths === undefined) {
+            cssPaths = await findAllStyleCss(dirKey, _source);
+            cssPathCache.set(dirKey, cssPaths);
+          }
+          if (cssPaths.length > 0) {
+            // Copy all source CSS files to output (still needed for serve mode fallback)
+            for (const cssPath of cssPaths) {
+              if (!copiedCssFiles.has(cssPath)) {
+                const cssOutputPath = cssPath.replace(source, output);
+                const cssContent = await readFile(cssPath, 'utf8');
+                await outputFile(cssOutputPath, cssContent);
+                copiedCssFiles.add(cssPath);
+              }
+            }
+            // Bundle into a single file
+            const bundleUrl = await bundleDocumentCss(cssPaths, output, source, folderRelative, { minify: true });
+            docBundleCache.set(`css:${dirKey}`, bundleUrl);
+            styleLink = `<link rel="stylesheet" href="${bundleUrl}" />`;
+          } else {
+            docBundleCache.set(`css:${dirKey}`, null);
           }
-          
-          // Generate link tag
-          styleLink = `<link rel="stylesheet" href="${cssUrlPath}" />`;
         }
       } catch (e) {
         // ignore
         console.error(e);
       }
 
-      // Find all script.js or _script.js files from docroot to current dir and inline their contents
-      // Use cache to avoid repeated filesystem walks for same directory
+      // Find all script.js files from docroot to current dir and bundle them
+      // (Generate mode: one JS bundle per unique folder, external not inlined)
       let customScript = "";
       try {
         const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-        let scriptPaths = scriptPathCache.get(dirKey);
-        if (scriptPaths === undefined) {
-          scriptPaths = await findAllScriptJs(dirKey, _source);
-          scriptPathCache.set(dirKey, scriptPaths); // Cache empty arrays too
-        }
-        const scriptTags = [];
-        for (const scriptPath of scriptPaths) {
-          const scriptContent = await readFile(scriptPath, 'utf8');
-          scriptTags.push(`<script>\n${scriptContent}\n</script>`);
+        const folderRelative = (dir === "/" || dir === "") ? "" : dir;
+
+        let cachedBundleUrl = docBundleCache.get(`js:${dirKey}`);
+        if (cachedBundleUrl !== undefined) {
+          if (cachedBundleUrl) {
+            customScript = `<script src="${cachedBundleUrl}"></script>`;
+          }
+        } else {
+          let scriptPaths = scriptPathCache.get(dirKey);
+          if (scriptPaths === undefined) {
+            scriptPaths = await findAllScriptJs(dirKey, _source);
+            scriptPathCache.set(dirKey, scriptPaths);
+          }
+          if (scriptPaths.length > 0) {
+            const bundleUrl = await bundleDocumentJs(scriptPaths, output, source, folderRelative, { minify: true });
+            docBundleCache.set(`js:${dirKey}`, bundleUrl);
+            customScript = `<script src="${bundleUrl}"></script>`;
+          } else {
+            docBundleCache.set(`js:${dirKey}`, null);
+          }
         }
-        customScript = scriptTags.join('\n');
       } catch (e) {
         // ignore
         console.error(e);
       }
 
       const requestedTemplateName = fileMeta && fileMeta.template;
-      const template =
-        templates[requestedTemplateName] || templates[DEFAULT_TEMPLATE_NAME];
+      const templateName = requestedTemplateName || DEFAULT_TEMPLATE_NAME;
+      const template = templates[templateName];
 
       if (!template) {
-        throw new Error(`Template not found. Requested: "${requestedTemplateName || DEFAULT_TEMPLATE_NAME}". Available templates: ${Object.keys(templates).join(', ') || 'none'}`);
+        throw new Error(`Template not found. Requested: "${templateName}". Available templates: ${Object.keys(templates).join(', ') || 'none'}`);
+      }
+
+      // Register this document's dependencies for invalidation tracking
+      {
+        const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
+        const cssDeps = cssPathCache.get(dirKey) || [];
+        const jsDeps = scriptPathCache.get(dirKey) || [];
+        dependencyTracker.registerDocument(file, {
+          templateName,
+          cssPaths: cssDeps,
+          scriptPaths: jsDeps,
+        });
       }
 
       // Check if this file has a custom menu
@@ -1026,11 +1091,15 @@ export async function generate({
   watchModeCache.meta = meta;
   watchModeCache.output = output;
   watchModeCache.hashCache = hashCache;
+  watchModeCache.cacheBustTimestamp = cacheBustTimestamp;
+  watchModeCache.cacheBustHashes = cacheBustHashes;
+  watchModeCache.allArticlePaths = [...allSourceFilenamesThatAreArticles];
   watchModeCache.imageMap = imageMap;
   watchModeCache.customMenus = customMenus;
   watchModeCache.lastFullBuild = Date.now();
   watchModeCache.isInitialized = true;
-  progress.log(`Watch cache initialized for fast single-file regeneration`);
+  const depStats = dependencyTracker.getStats();
+  progress.log(`Watch cache initialized (${depStats.totalDocuments} documents, ${depStats.uniqueFiles} dependencies tracked)`);
 
   // Write error report if there were any errors
   if (errors.length > 0) {
@@ -1085,6 +1154,104 @@ export async function generate({
   };
 }
 
+/**
+ * Regenerate multiple documents affected by a dependency change (e.g., style.css, script.js, template).
+ * Uses the watchModeCache and dependency tracker to efficiently re-render affected documents
+ * with updated cache-bust timestamps.
+ *
+ * @param {string[]} documentPaths - Absolute paths to documents to regenerate
+ * @param {Object} options
+ * @param {string} options._source - Source directory
+ * @param {string} options._meta - Meta directory
+ * @param {string} options._output - Output directory
+ * @param {string} [options.reason] - Reason for regeneration (for logging)
+ * @param {string[]} [options.priorityPaths] - Document paths to regenerate first (e.g. client-viewed docs)
+ * @param {function} [options.onPriorityComplete] - Callback after priority paths are done (receives { regenerated, failed })
+ * @returns {Promise<{success: boolean, message: string, regenerated: number, failed: number}>}
+ */
+export async function regenerateAffectedDocuments(documentPaths, {
+  _source,
+  _meta,
+  _output,
+  reason = "dependency change",
+  priorityPaths = [],
+  onPriorityComplete = null,
+} = {}) {
+  const startTime = Date.now();
+
+  if (!watchModeCache.isInitialized) {
+    return { success: false, message: "Cache not initialized - need full build first", regenerated: 0, failed: 0 };
+  }
+
+  if (documentPaths.length === 0) {
+    return { success: true, message: "No documents to regenerate", regenerated: 0, failed: 0 };
+  }
+
+  // Generate a fresh cache-bust timestamp for this invalidation pass
+  const newTimestamp = generateCacheBustTimestamp();
+  watchModeCache.cacheBustTimestamp = newTimestamp;
+
+  let regenerated = 0;
+  let failed = 0;
+
+  // Separate priority paths from the rest
+  const prioritySet = new Set(priorityPaths.map(p => resolve(p)));
+  const priorityDocs = documentPaths.filter(p => prioritySet.has(resolve(p)));
+  const remainingDocs = documentPaths.filter(p => !prioritySet.has(resolve(p)));
+
+  if (priorityDocs.length > 0) {
+    console.log(`🔄 Regenerating ${priorityDocs.length} priority documents first, then ${remainingDocs.length} remaining (${reason})`);
+  } else {
+    console.log(`🔄 Regenerating ${documentPaths.length} documents (${reason})`);
+  }
+
+  // Process priority documents first
+  for (const docPath of priorityDocs) {
+    try {
+      const result = await regenerateSingleFile(docPath, { _source, _meta, _output });
+      if (result.success) {
+        regenerated++;
+      } else {
+        console.warn(`  ⚠️  ${docPath}: ${result.message}`);
+        failed++;
+      }
+    } catch (e) {
+      console.error(`  ❌ ${docPath}: ${e.message}`);
+      failed++;
+    }
+  }
+
+  // Notify caller that priority docs are done (so server can reload those clients immediately)
+  if (priorityDocs.length > 0 && onPriorityComplete) {
+    try {
+      onPriorityComplete({ regenerated, failed, priorityDocs });
+    } catch (e) {
+      console.error(`  ⚠️ onPriorityComplete callback error: ${e.message}`);
+    }
+  }
+
+  // Process remaining documents
+  for (const docPath of remainingDocs) {
+    try {
+      const result = await regenerateSingleFile(docPath, { _source, _meta, _output });
+      if (result.success) {
+        regenerated++;
+      } else {
+        console.warn(`  ⚠️  ${docPath}: ${result.message}`);
+        failed++;
+      }
+    } catch (e) {
+      console.error(`  ❌ ${docPath}: ${e.message}`);
+      failed++;
+    }
+  }
+
+  const elapsed = Date.now() - startTime;
+  const msg = `Regenerated ${regenerated}/${documentPaths.length} documents in ${elapsed}ms (${reason})${failed > 0 ? `, ${failed} failed` : ""}`;
+  console.log(`✅ ${msg}`);
+  return { success: failed === 0, message: msg, regenerated, failed };
+}
+
 /**
  * Regenerate a single file without scanning the entire source directory.
  * This is much faster for watch mode - only regenerate what changed.
@@ -1204,23 +1371,20 @@ export async function regenerateSingleFile(changedFile, {
       }
     }
 
-    // Find CSS and copy to output
+    // Find all CSS files up the tree and create separate link tags
+    // (regenerateSingleFile is used in serve mode, so separate tags per level for invalidation)
     let styleLink = "";
     try {
-      // For root-level files, dir may be "/" which would resolve to filesystem root
       const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-      const cssPath = await findStyleCss(dirKey);
-      if (cssPath) {
-        // Calculate output path for the CSS file
-        const cssOutputPath = cssPath.replace(source, output);
-        const cssUrlPath = '/' + cssPath.replace(source, '');
-        
-        // Copy CSS file (always copy in single-file mode to ensure it's up to date)
-        const cssContent = await readFile(cssPath, 'utf8');
-        await outputFile(cssOutputPath, cssContent);
-        
-        // Generate link tag
-        styleLink = `<link rel="stylesheet" href="${cssUrlPath}" />`;
+      const cssPaths = await findAllStyleCss(dirKey, _source);
+      if (cssPaths.length > 0) {
+        // Copy all CSS files to output (always copy in single-file mode to ensure up to date)
+        for (const cssPath of cssPaths) {
+          const cssOutputPath = cssPath.replace(source, output);
+          const cssContent = await readFile(cssPath, 'utf8');
+          await outputFile(cssOutputPath, cssContent);
+        }
+        styleLink = generateSeparateCssTags(cssPaths, source);
       }
     } catch (e) {
       // ignore
@@ -1235,17 +1399,21 @@ export async function regenerateSingleFile(changedFile, {
       return { success: false, message: `Template not found: ${requestedTemplateName || DEFAULT_TEMPLATE_NAME}` };
     }
     
-    // Find all script.js or _script.js files from docroot to current dir and inline their contents
+    // Find all script.js files from docroot to current dir and serve as separate external tags
+    // (Serve mode: separate tags per level for individual invalidation)
     let customScript = "";
     try {
       const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
       const scriptPaths = await findAllScriptJs(dirKey, _source);
-      const scriptTags = [];
-      for (const scriptPath of scriptPaths) {
-        const scriptContent = await readFile(scriptPath, 'utf8');
-        scriptTags.push(`<script>\n${scriptContent}\n</script>`);
+      if (scriptPaths.length > 0) {
+        // Copy all script files to output so they can be served
+        for (const scriptPath of scriptPaths) {
+          const scriptOutputPath = scriptPath.replace(source, output);
+          const scriptContent = await readFile(scriptPath, 'utf8');
+          await outputFile(scriptOutputPath, scriptContent);
+        }
+        customScript = generateSeparateJsTags(scriptPaths, source);
       }
-      customScript = scriptTags.join('\n');
     } catch (e) {
       // ignore
     }