@kenjura/ursa 0.87.1 → 0.89.1

package/src/jobs/generate.js

@@ -1,1715 +1,1772 @@
-import { recurse } from "../helper/recursive-readdir.js";
-import { copyFile, mkdir, readdir, readFile, stat } from "fs/promises";
-import { getAutomenu } from "../helper/automenu.js";
-import { filterAsync } from "../helper/filterAsync.js";
-import { isDirectory } from "../helper/isDirectory.js";
-import { isFolderHidden, clearConfigCache } from "../helper/folderConfig.js";
-import {
-  extractMetadata,
-  extractRawMetadata,
-  isMetadataOnly,
-  getAutoIndexConfig,
-} from "../helper/metadataExtractor.js";
-import { injectFrontmatterTable } from "../helper/frontmatterTable.js";
-import {
-  hashContent,
-  loadHashCache,
-  saveHashCache,
-  needsRegeneration,
-  updateHash,
-  getUrsaDir,
-} from "../helper/contentHash.js";
-import {
-  buildValidPaths,
-  markInactiveLinks,
-  resolveRelativeUrls,
-} from "../helper/linkValidator.js";
-import { getAndIncrementBuildId, loadContentTimestamps, saveContentTimestamps, updateContentTimestamp } from "../helper/ursaConfig.js";
-import { extractSections } from "../helper/sectionExtractor.js";
-import { renderFile, renderFileAsync, terminateParserPool } from "../helper/fileRenderer.js";
-import { buildReactRuntime } from "../helper/mdxRenderer.js";
-import { findStyleCss, findAllStyleCss } from "../helper/findStyleCss.js";
-import { findScriptJs, findAllScriptJs } from "../helper/findScriptJs.js";
-import { bundleMetaTemplateAssets, bundleDocumentCss, bundleDocumentJs, clearMetaBundleCache, generateSeparateCssTags, generateSeparateJsTags } from "../helper/assetBundler.js";
-import { buildFullTextIndex, buildIncrementalIndex, loadIndexCache, saveIndexCache } from "../helper/fullTextIndex.js";
-import { dependencyTracker, loadDependencyTracker, saveDependencyTracker } from "../helper/dependencyTracker.js";
-import { CacheBustHashMap } from "../helper/build/cacheBust.js";
-import { copy as copyDir, emptyDir, outputFile, remove } from "fs-extra";
-import { basename, dirname, extname, join, parse, resolve } from "path";
-import { URL } from "url";
-import o2x from "object-to-xml";
-import { existsSync } from "fs";
-import { fileExists } from "../helper/fileExists.js";
-import { createWhitelistFilter } from "../helper/whitelistFilter.js";
-import { processAllImages, transformImageTags, clearImageCache, copyAllImagesFast } from "../helper/imageProcessor.js";
-import { extractImageReferences } from "../helper/imageExtractor.js";
-import { checkFileSize, readFileStreaming, formatFileSize } from "../helper/streamingReader.js";
-import { generateBreadcrumbs } from "../helper/breadcrumbs.js";
-import {
-  loadNavCache,
-  saveNavCache,
-  hashFileList,
-  hashFileStats,
-  isNavCacheValid,
-  createNavCacheEntry,
-  restoreMap,
-} from "../helper/build/navCache.js";
-
-// Import build helpers from organized modules
-import {
-  generateCacheBustTimestamp,
-  addTimestampToCssUrls,
-  addTimestampToHtmlStaticRefs,
-  processBatched,
-  ProgressReporter,
-  watchModeCache,
-  clearWatchCache as clearWatchCacheBase,
-  toTitleCase,
-  parseExcludeOption,
-  createExcludeFilter,
-  addTrailingSlash,
-  getTemplates,
-  getMenu,
-  findAllCustomMenus,
-  getCustomMenuForFile,
-  getTransformedMetadata,
-  getFooter,
-  getUrsaMetadata,
-  generateAutoIndices,
-  generateAutoIndexHtmlFromSource,
-  copyMetaAssets,
-} from "../helper/build/index.js";
-import { getProfiler } from "../helper/build/profiler.js";
-import { reconcileAll, isInsideTemplatesFolder } from "../helper/documentTemplates.js";
-
-// Concurrency limiter for batch processing to avoid memory exhaustion
-const BATCH_SIZE = parseInt(process.env.URSA_BATCH_SIZE || '50', 10);
-
-// Cache for CSS path lookups to avoid repeated filesystem walks
-const cssPathCache = new Map();
-
-// Cache for script path lookups to avoid repeated filesystem walks
-const scriptPathCache = new Map();
-
-// Cache for document-level CSS/JS bundle paths to avoid re-bundling identical sets
-const docBundleCache = new Map();
-
-// Wrapper for clearWatchCache that passes cssPathCache and scriptPathCache
-export function clearWatchCache() {
-  clearWatchCacheBase(cssPathCache);
-  scriptPathCache.clear();
-  docBundleCache.clear();
-  clearMetaBundleCache();
-}
-
-// Clear just the script-related caches (for when script.js changes)
-export function clearScriptCache() {
-  scriptPathCache.clear();
-  // Clear all JS bundle cache entries
-  for (const key of docBundleCache.keys()) {
-    if (key.startsWith('js:')) {
-      docBundleCache.delete(key);
-    }
-  }
-}
-
-// Clear just the CSS-related caches (for when style.css changes)
-export function clearStyleCache() {
-  cssPathCache.clear();
-  // Clear all CSS bundle cache entries
-  for (const key of docBundleCache.keys()) {
-    if (key.startsWith('css:')) {
-      docBundleCache.delete(key);
-    }
-  }
-}
-
-const progress = new ProgressReporter();
-
-const DEFAULT_TEMPLATE_NAME =
-  process.env.DEFAULT_TEMPLATE_NAME ?? "default-template";
-
-export async function generate({
-  _source = join(process.cwd(), "."),
-  _meta = join(process.cwd(), "meta"),
-  _output = join(process.cwd(), "build"),
-  _whitelist = null,
-  _exclude = null,
-  _incremental = false,  // Legacy flag, now ignored (always incremental)
-  _clean = false,  // When true, ignore cache and regenerate all files
-  _deferImages = false,  // When true, copy images without processing, return promise for background processing
-  _deferSearchIndex = false,  // When true, return promise for search index building (for faster startup)
-} = {}) {
-  // Initialize profiler for this build
-  const profiler = getProfiler(true);
-  
-  console.log({ _source, _meta, _output, _whitelist, _exclude, _clean, _deferImages, _deferSearchIndex });
-  const source = resolve(_source) + "/";
-  const meta = resolve(_meta);
-  const output = resolve(_output) + "/";
-  console.log({ source, meta, output });
-
-  // Generate cache-busting timestamp for this build
-  const cacheBustTimestamp = generateCacheBustTimestamp();
-  const cacheBustHashes = new CacheBustHashMap();
-  progress.logTimed(`Cache-bust timestamp: ${cacheBustTimestamp}`);
-
-  // Initialize dependency tracker for this build
-  dependencyTracker.init(source);
-
-  // Clear output directory and cache when --clean is specified
-  if (_clean) {
-    progress.startTimer('Clean');
-    const ursaDir = getUrsaDir(source);
-    progress.logTimed(`Clean build: deleting cache folder ${ursaDir}`);
-    await remove(ursaDir);
-    progress.logTimed(`Clean build: clearing output directory ${output}`);
-    await emptyDir(output);
-    progress.logTimed(`Clean complete [${progress.stopTimer('Clean')}]`);
-  } else {
-    // Warm start: reload persisted dependency registrations so hash-skipped
-    // documents keep their edges (current-run registrations take precedence)
-    const loaded = await loadDependencyTracker(source);
-    if (loaded) {
-      const stats = dependencyTracker.getStats();
-      progress.logTimed(`Dependency graph loaded: ${stats.totalDocuments} documents, ${stats.uniqueFiles} dependencies`);
-    }
-  }
-
-  // Phase: Scan source files
-  profiler.startPhase('Scan source files');
-  progress.startTimer('Scan');
-  const allSourceFilenamesUnfiltered = await recurse(source, [() => false]);
-  progress.logTimed(`Scanned ${allSourceFilenamesUnfiltered.length} files [${progress.stopTimer('Scan')}]`);
-  profiler.endPhase('Scan source files');
-  
-  // Phase: Filter and classify files
-  profiler.startPhase('Filter & classify');
-  progress.startTimer('Filter');
-  
-  // Apply include filter (existing functionality)
-  const includeFilter = process.env.INCLUDE_FILTER
-    ? (fileName) => fileName.match(process.env.INCLUDE_FILTER)
-    : Boolean;
-  let allSourceFilenames = allSourceFilenamesUnfiltered.filter(includeFilter);
-  
-  // Apply exclude filter if specified
-  if (_exclude) {
-    const excludedPaths = await parseExcludeOption(_exclude, source);
-    const excludeFilter = createExcludeFilter(excludedPaths, source);
-    const beforeCount = allSourceFilenames.length;
-    allSourceFilenames = allSourceFilenames.filter(excludeFilter);
-    progress.logTimed(`Exclude filter applied: ${beforeCount - allSourceFilenames.length} files excluded`);
-  }
-  
-  // Apply whitelist filter if specified
-  if (_whitelist) {
-    const whitelistFilter = await createWhitelistFilter(_whitelist, source);
-    allSourceFilenames = allSourceFilenames.filter(whitelistFilter);
-    progress.logTimed(`Whitelist applied: ${allSourceFilenames.length} files after filtering`);
-  }
-
-  // Clear config cache at start of generation to pick up any changes
-  clearConfigCache();
-
-  // Helper to check if a path is inside a config-hidden folder
-  const isInHiddenFolder = (filePath) => {
-    const dir = dirname(filePath);
-    return isFolderHidden(dir, source);
-  };
-
-  // read all articles, process them, copy them to build
-  const articleExtensions = /\.(md|mdx|txt|yml)$/;
-  const hiddenOrSystemDirs = /[\/\\]\.(?!\.)|[\/\\]node_modules[\/\\]|[\/\\]_templates[\/\\]|[\/\\]_templates$/;  // Matches hidden folders (starting with .), node_modules, or _templates
-  const allSourceFilenamesThatAreArticles = allSourceFilenames.filter(
-    (filename) => filename.match(articleExtensions) && !filename.match(hiddenOrSystemDirs) && !isInHiddenFolder(filename)
-  );
-  const allSourceFilenamesThatAreDirectories = (await filterAsync(
-    allSourceFilenames,
-    (filename) => isDirectory(filename)
-  )).filter((filename) => !filename.match(hiddenOrSystemDirs) && !isFolderHidden(filename, source));
-
-  // Build set of existing HTML files in source directory (these should not be overwritten)
-  const htmlExtensions = /\.html$/;
-  const existingHtmlFiles = new Set(
-    allSourceFilenames
-      .filter(f => f.match(htmlExtensions) && !f.match(hiddenOrSystemDirs))
-      .map(f => f.replace(source, '')) // Store relative paths for easy lookup
-  );
-  
-  progress.logTimed(`Classified: ${allSourceFilenamesThatAreArticles.length} articles, ${allSourceFilenamesThatAreDirectories.length} dirs, ${existingHtmlFiles.size} HTML [${progress.stopTimer('Filter')}]`);
-  profiler.endPhase('Filter & classify');
-
-  // Drop persisted dependency registrations for documents that no longer
-  // exist (or are excluded), so stale entries don't accumulate across runs
-  dependencyTracker.prune(new Set(allSourceFilenamesThatAreArticles));
-
-  // Phase: Document template reconciliation
-  // Must run BEFORE article processing so that any template-driven changes
-  // to source .md files are picked up during rendering.
-  profiler.startPhase('Template reconciliation');
-  progress.startTimer('Templates');
-  const templateReconciliation = await reconcileAll(
-    allSourceFilenamesThatAreArticles,
-    allSourceFilenamesUnfiltered,   // templates live in _templates which is filtered out of articles
-    source
-  );
-  if (templateReconciliation.updated > 0 || templateReconciliation.conflicts > 0 || templateReconciliation.initialized > 0) {
-    progress.logTimed(
-      `📄 Document templates: ${templateReconciliation.initialized} initialized, ` +
-      `${templateReconciliation.updated} auto-merged, ` +
-      `${templateReconciliation.conflicts} conflicts, ` +
-      `${templateReconciliation.unchanged} unchanged, ` +
-      `${templateReconciliation.errors} errors`
-    );
-    if (templateReconciliation.conflicts > 0) {
-      console.warn(`\n⚠️  Template conflicts require manual resolution:`);
-      for (const msg of templateReconciliation.messages) {
-        if (msg.includes('Conflict')) console.warn(`   ${msg}`);
-      }
-      console.warn('');
-    }
-    if (templateReconciliation.errors > 0) {
-      for (const msg of templateReconciliation.messages) {
-        if (msg.includes('Error') || msg.includes('not found')) console.warn(`   ⚠️  ${msg}`);
-      }
-    }
-  }
-  progress.logTimed(`Document templates processed [${progress.stopTimer('Templates')}]`);
-  profiler.endPhase('Template reconciliation');
-
-  // Phase: Build navigation and metadata
-  profiler.startPhase('Build navigation');
-  progress.startTimer('Navigation');
-  
-  // Check if we can use cached navigation
-  let validPaths, templates, menu, menuData, customMenus, footer, buildId;
-  let navCacheUsed = false;
-  
-  if (!_clean) {
-    const fileListHash = hashFileList(allSourceFilenames);
-    const fileStatsHash = await hashFileStats(allSourceFilenames);
-    const navCache = await loadNavCache(source);
-    
-    if (isNavCacheValid(navCache, fileListHash, fileStatsHash)) {
-      // Use cached navigation data
-      navCacheUsed = true;
-      validPaths = restoreMap(navCache.validPaths);
-      menuData = navCache.menuData;
-      menu = navCache.menuHtml;
-      customMenus = restoreMap(navCache.customMenus);
-      
-      // Templates and footer still need to be loaded (they depend on meta directory)
-      templates = await getTemplates(meta);
-      buildId = getAndIncrementBuildId(resolve(_source));
-      footer = await getFooter(source, _source, buildId);
-      
-      progress.logTimed(`Navigation loaded from cache: ${validPaths.size} paths, ${customMenus.size} custom menus [${progress.stopTimer('Navigation')}]`);
-    } else {
-      // Cache miss - build navigation from scratch
-      validPaths = buildValidPaths(allSourceFilenamesThatAreArticles, source, allSourceFilenamesThatAreDirectories);
-      templates = await getTemplates(meta);
-      
-      const menuResult = await getMenu(allSourceFilenames, source, validPaths);
-      menu = menuResult.html;
-      menuData = menuResult.menuData;
-      
-      customMenus = findAllCustomMenus(allSourceFilenames, source);
-      buildId = getAndIncrementBuildId(resolve(_source));
-      footer = await getFooter(source, _source, buildId);
-      
-      // Save to cache for next run
-      const cacheEntry = createNavCacheEntry(
-        fileListHash,
-        fileStatsHash,
-        menuData,
-        menu,
-        Array.from(validPaths.entries()),
-        Array.from(customMenus.entries())
-      );
-      await saveNavCache(source, cacheEntry);
-      
-      progress.logTimed(`Navigation built: ${validPaths.size} paths, ${customMenus.size} custom menus [${progress.stopTimer('Navigation')}]`);
-    }
-  } else {
-    // Clean build - ignore cache
-    validPaths = buildValidPaths(allSourceFilenamesThatAreArticles, source, allSourceFilenamesThatAreDirectories);
-    templates = await getTemplates(meta);
-    
-    const menuResult = await getMenu(allSourceFilenames, source, validPaths);
-    menu = menuResult.html;
-    menuData = menuResult.menuData;
-    
-    customMenus = findAllCustomMenus(allSourceFilenames, source);
-    buildId = getAndIncrementBuildId(resolve(_source));
-    footer = await getFooter(source, _source, buildId);
-    
-    progress.logTimed(`Navigation built (clean): ${validPaths.size} paths, ${customMenus.size} custom menus [${progress.stopTimer('Navigation')}]`);
-  }
-  
-  profiler.endPhase('Build navigation');
-
-  // Build the _ursa_metadata embedded in every generated JSON file (ursa + doc repo versions)
-  const ursaMetadata = await getUrsaMetadata(_source);
-
-  // Phase: Load cache
-  profiler.startPhase('Load cache');
-  progress.startTimer('Cache');
-  
-  // Load content hash cache from .ursa folder in source directory
-  let hashCache = new Map();
-  if (!_clean) {
-    hashCache = await loadHashCache(source);
-    progress.logTimed(`Loaded ${hashCache.size} cached hashes [${progress.stopTimer('Cache')}]`);
-  } else {
-    progress.logTimed(`Clean build: ignoring cached hashes`);
-    progress.stopTimer('Cache');
-  }
-  
-  // Load content timestamps from .ursa.json (survives --clean)
-  // These track when content actually changed, not filesystem mtime
-  const contentTimestamps = loadContentTimestamps(source);
-  const buildTimestamp = Date.now();
-  progress.logTimed(`Loaded ${contentTimestamps.size} content timestamps`);
-  profiler.endPhase('Load cache');
-
-  // Phase: Copy meta/public files
-  profiler.startPhase('Copy meta files');
-  progress.startTimer('Meta');
-  
-  // create public folder
-  const pub = join(output, "public");
-  await mkdir(pub, { recursive: true });
-  
-  // Copy meta assets with new template folder structure
-  const { copiedFiles, orphanedFiles } = await copyMetaAssets(meta, pub);
-  
-  // Warn about orphaned files in meta that aren't part of any template
-  if (orphanedFiles.length > 0) {
-    console.warn(`\n⚠️  Warning: Found ${orphanedFiles.length} orphaned file(s) in meta directory:`);
-    console.warn(`   These files are not in meta/templates/ or meta/shared/ and won't be included:`);
-    for (const file of orphanedFiles.slice(0, 10)) {
-      console.warn(`   - ${file}`);
-    }
-    if (orphanedFiles.length > 10) {
-      console.warn(`   ... and ${orphanedFiles.length - 10} more`);
-    }
-    console.warn(`   Move them to meta/templates/{templateName}/ or meta/shared/ to include them.\n`);
-  }
-
-  // Bundle meta template assets (CSS + JS) into single files per template
-  // This must happen after copying meta to public but before cache-busting
-  templates = await bundleMetaTemplateAssets(templates, meta, pub, { minify: true, sourcemap: false });
-  progress.logTimed(`Meta template assets bundled`);
-
-  // Build React runtime for MDX hydration (React 19 has no UMD, so we bundle locally)
-  await buildReactRuntime(pub);
-
-  // Process all CSS files in the entire output directory tree for cache-busting
-  const allOutputFiles = await recurse(output, [() => false]);
-  for (const cssFile of allOutputFiles.filter(f => f.endsWith('.css'))) {
-    const cssContent = await readFile(cssFile, 'utf8');
-    const processedCss = addTimestampToCssUrls(cssContent, cacheBustTimestamp);
-    await outputFile(cssFile, processedCss);
-  }
-
-  // Process JS files in output for cache-busting fetch URLs
-  for (const jsFile of allOutputFiles.filter(f => f.endsWith('.js'))) {
-    let jsContent = await readFile(jsFile, 'utf8');
-    jsContent = jsContent.replace(
-      /fetch\(['"]([^'"\)]+\.(json))['"](?!\s*\+)/g,
-      `fetch('$1?v=${cacheBustTimestamp}'`
-    );
-    await outputFile(jsFile, jsContent);
-  }
-  
-  progress.logTimed(`Meta files copied and processed [${progress.stopTimer('Meta')}]`);
-  profiler.endPhase('Copy meta files');
-
-  // Track errors for error report
-  const errors = [];
-
-  // Search index: built incrementally during article processing (lighter memory footprint)
-  const searchIndex = [];
-  // Full-text index: collect documents for word-to-document mapping
-  const fullTextDocs = [];
-  // Recent activity: collect {title, url, mtime} for all articles, keep top 10 by mtime
-  const recentActivity = [];
-  // Track paths of documents that were regenerated (for incremental index updates)
-  const changedPaths = new Set();
-  // Directory index cache: only stores minimal data needed for directory indices
-  // Uses WeakRef-style approach - store only what's needed, clear as we go
-  const dirIndexCache = new Map();
-  
-  // Track CSS files that have been copied to avoid duplicates
-  const copiedCssFiles = new Set();
-
-  // Identify all image files from the filtered source list
-  const imageExtensions = /\.(jpg|jpeg|png|gif|webp|svg|ico)/;
-  let allSourceFilenamesThatAreImages = allSourceFilenames.filter(
-    (filename) => filename.match(imageExtensions) && !filename.match(hiddenOrSystemDirs)
-  );
-  
-  // When using a whitelist, also include images referenced by whitelisted documents
-  // This ensures that images used in whitelisted articles are processed even if not explicitly whitelisted
-  if (_whitelist) {
-    progress.logTimed('Scanning whitelisted articles for image references...');
-    const referencedImages = new Set();
-    
-    for (const articlePath of allSourceFilenamesThatAreArticles) {
-      try {
-        const content = await readFile(articlePath, 'utf8');
-        const imageRefs = extractImageReferences(content, articlePath, source);
-        imageRefs.forEach(img => referencedImages.add(img));
-      } catch (e) {
-        // Ignore read errors - file might not exist or be unreadable
-      }
-    }
-    
-    // Get all images from the unfiltered source list that are referenced
-    const allImagesUnfiltered = allSourceFilenamesUnfiltered.filter(
-      (filename) => filename.match(imageExtensions) && !filename.match(hiddenOrSystemDirs)
-    );
-    
-    // Add referenced images that aren't already in the list
-    const additionalImages = allImagesUnfiltered.filter(
-      img => referencedImages.has(img) && !allSourceFilenamesThatAreImages.includes(img)
-    );
-    
-    if (additionalImages.length > 0) {
-      progress.logTimed(`Found ${additionalImages.length} additional images referenced by whitelisted documents`);
-      allSourceFilenamesThatAreImages = [...allSourceFilenamesThatAreImages, ...additionalImages];
-    }
-  }
-  
-  // Phase: Process images
-  profiler.startPhase('Process images');
-  progress.startTimer('Images');
-  
-  // Handle images based on deferred mode
-  let imageMap = new Map();
-  let deferredImageProcessingPromise = null;
-  
-  if (_deferImages) {
-    // Fast mode: just copy images without processing, defer preview generation
-    progress.logTimed(`Copying ${allSourceFilenamesThatAreImages.length} images (preview generation deferred)...`);
-    await copyAllImagesFast(
-      allSourceFilenamesThatAreImages,
-      source,
-      output,
-      (current, total, path) => {
-        progress.status('Images (copy)', `${current}/${total} ${path}`);
-      }
-    );
-    progress.done('Images (copy)', `${allSourceFilenamesThatAreImages.length} copied (previews deferred)`);
-    profiler.endPhase('Process images');
-    
-    // Create promise for background image processing (will be returned to caller)
-    deferredImageProcessingPromise = (async () => {
-      progress.logTimed(`\n🖼️  Starting deferred image preview generation...`);
-      const startTime = Date.now();
-      const processedImageMap = await processAllImages(
-        allSourceFilenamesThatAreImages,
-        source,
-        output,
-        (current, total, path) => {
-          progress.status('Images (previews)', `${current}/${total} ${path}`);
-        }
-      );
-      const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
-      progress.done('Images (previews)', `${allSourceFilenamesThatAreImages.length} done (${processedImageMap.size} with previews) in ${elapsed}s`);
-      
-      // Update the watch cache with the processed image map
-      watchModeCache.imageMap = processedImageMap;
-      
-      return processedImageMap;
-    })();
-  } else {
-    // Normal mode: process all images FIRST to build the preview image map
-    // This is done before articles so we can transform img tags in the HTML
-    progress.logTimed(`Processing ${allSourceFilenamesThatAreImages.length} images for preview generation...`);
-    imageMap = await processAllImages(
-      allSourceFilenamesThatAreImages,
-      source,
-      output,
-      (current, total, path) => {
-        progress.status('Images', `${current}/${total} ${path}`);
-      }
-    );
-    progress.done('Images', `${allSourceFilenamesThatAreImages.length} done (${imageMap.size} with previews) [${progress.stopTimer('Images')}]`);
-    profiler.endPhase('Process images');
-  }
-
-  // Phase: Process articles
-  profiler.startPhase('Process articles');
-  progress.startTimer('Articles');
-  
-  // Track files that were regenerated (for incremental mode stats)
-  let regeneratedCount = 0;
-  let skippedCount = 0;
-  let processedCount = 0;
-  const totalArticles = allSourceFilenamesThatAreArticles.length;
-
-  progress.logTimed(`Processing ${totalArticles} articles in batches of ${BATCH_SIZE}...`);
-
-  // Single pass: process all articles with batched concurrency to limit memory usage
-  await processBatched(allSourceFilenamesThatAreArticles, async (file) => {
-    try {
-      processedCount++;
-      const shortFile = file.replace(source, '');
-      progress.status('Articles', `${processedCount}/${totalArticles} ${shortFile}`);
-      
-      // Use streaming for large files to reduce memory pressure
-      const { size: fileSize, useStreaming } = await checkFileSize(file);
-      let rawBody;
-      if (useStreaming) {
-        rawBody = await readFileStreaming(file, 'utf8');
-        progress.log(`📄 Streaming large file ${shortFile} (${formatFileSize(fileSize)})`);
-      } else {
-        rawBody = await readFile(file, "utf8");
-      }
-      
-      const type = parse(file).ext;
-      const ext = extname(file);
-      const base = basename(file, ext);
-      const dir = addTrailingSlash(dirname(file)).replace(source, "");
-      
-      // Calculate output paths for this file
-      const outputFilename = file
-        .replace(source, output)
-        .replace(parse(file).ext, ".html");
-      const url = '/' + outputFilename.replace(output, '');
-      
-      // Generate URL path relative to output (for search index)
-      const relativePath = file.replace(source, '').replace(/\.(md|mdx|txt|yml)$/, '.html');
-      const searchUrl = relativePath.startsWith('/') ? relativePath : '/' + relativePath;
-      
-      // Generate title from filename (in title case)
-      // For index/home files, use parent folder name instead
-      const titleBase = (base === 'index' || base === 'home') ? basename(dirname(file)) : base;
-      const title = toTitleCase(titleBase || base);
-      
-      // Always add to search index (lightweight: title + path only, content added lazily)
-      searchIndex.push({
-        title: title,
-        path: relativePath,
-        url: searchUrl,
-        content: '' // Content excerpts built lazily to save memory
-      });
-
-      // Add document to full-text index (uses raw markdown content)
-      fullTextDocs.push({
-        path: relativePath,
-        title: title,
-        content: rawBody
-      });
-
-      // Collect timestamp for recent activity tracking
-      // Use stored content timestamp if available, otherwise fall back to file mtime
-      // Content timestamps track when content actually changed, not filesystem mtime
-      const storedTimestamp = contentTimestamps.get(relativePath);
-      let activityTimestamp = storedTimestamp;
-      if (!activityTimestamp) {
-        // No stored timestamp - use file mtime as initial value
-        try {
-          const fileStat = await stat(file);
-          activityTimestamp = fileStat.mtimeMs;
-        } catch (e) {
-          activityTimestamp = 0;
-        }
-      }
-      recentActivity.push({
-        title: title,
-        url: searchUrl,
-        mtime: activityTimestamp
-      });
-      
-      // Check if a corresponding .html file already exists in source directory
-      const outputHtmlRelative = relativePath.startsWith('/') ? relativePath.slice(1) : relativePath;
-      if (existingHtmlFiles.has(outputHtmlRelative)) {
-        progress.log(`⚠️  Warning: Skipping ${shortFile} - would overwrite existing ${outputHtmlRelative} in source`);
-        skippedCount++;
-        return;
-      }
-
-      // Skip metadata-only index files - they exist only to provide folder metadata
-      // The auto-index system will generate the actual index.html for these folders
-      if (base === 'index' && (type === '.md' || type === '.mdx') && isMetadataOnly(rawBody)) {
-        progress.log(`ℹ️  Skipping metadata-only ${shortFile} - auto-index will generate listing`);
-        skippedCount++;
-        return;
-      }
-
-      // Check if file needs regeneration
-      const needsRegen = _clean || needsRegeneration(file, rawBody, hashCache);
-      
-      if (!needsRegen) {
-        skippedCount++;
-        // For directory indices, store minimal data (not full bodyHtml)
-        // But include metadata for directory JSON files
-        const skippedMeta = extractMetadata(rawBody);
-        dirIndexCache.set(file, {
-          name: base,
-          url,
-          metadata: skippedMeta,
-        });
-        return; // Skip regenerating this file
-      }
-      
-      regeneratedCount++;
-      // Track this path for incremental search index updates
-      changedPaths.add(relativePath);
-
-      const fileMeta = extractMetadata(rawBody);
-      const rawMeta = extractRawMetadata(rawBody);
-      
-      // Lazy metadata transform - only compute if template actually uses it
-      // This defers the potentially expensive custom transform function load
-      let transformedMetadata = null;
-      const getTransformedMeta = async () => {
-        if (transformedMetadata === null) {
-          transformedMetadata = await getTransformedMetadata(dirname(file), fileMeta);
-        }
-        return transformedMetadata;
-      };
-      
-      // Calculate the document's URL path (e.g., "/character/index.html")
-      const docUrlPath = '/' + dir + base + '.html';
-
-      // Use async rendering with worker threads for parallel markdown parsing
-      // Wikitext (.txt) files will fall back to main thread
-      // For MDX files, enable hydration if frontmatter has `hydrate: true`
-      const shouldHydrate = type === '.mdx' && fileMeta?.hydrate === true;
-      
-      let renderResult = await renderFileAsync({
-        fileContents: rawBody,
-        type,
-        dirname: dir,
-        basename: base,
-        filePath: file,
-        sourceRoot: source,
-        useWorker: true,
-        hydrate: shouldHydrate,
-      });
-
-      // Handle the result - can be string or { html, hydrationScript }
-      let body;
-      let hydrationScript = '';
-      if (typeof renderResult === 'object' && renderResult.html) {
-        body = renderResult.html;
-        hydrationScript = renderResult.hydrationScript || '';
-      } else {
-        body = renderResult;
-      }
-
-      // Inject default H1 if body doesn't start with one
-      if (!body || !body.trimStart().startsWith('<h1')) {
-        const h1Title = fileMeta?.title || title;
-        body = `<h1>${h1Title}</h1>\n` + (body || '');
-      }
-
-      // Inject breadcrumbs before the H1
-      const breadcrumbs = generateBreadcrumbs(dir, base, fileMeta);
-      if (breadcrumbs) {
-        body = breadcrumbs + body;
-      }
-
-      // Inject frontmatter table after first H1 (for markdown files with metadata)
-      if ((type === '.md' || type === '.mdx') && fileMeta) {
-        body = injectFrontmatterTable(body, fileMeta);
-      }
-
-      // Handle auto-index generation for index files with generate-auto-index: true
-      if (base === 'index' && fileMeta) {
-        const autoIndexConfig = getAutoIndexConfig(fileMeta);
-        if (autoIndexConfig.enabled) {
-          // Generate auto-index HTML for this directory from source
-          // Using source avoids race conditions with concurrent file generation
-          const sourceDir = dirname(file);
-          const autoIndexHtml = await generateAutoIndexHtmlFromSource(sourceDir, autoIndexConfig.depth);
-          
-          if (autoIndexHtml) {
-            if (autoIndexConfig.position === 'bottom') {
-              body = body + '\n' + autoIndexHtml;
-            } else {
-              body = autoIndexHtml + '\n' + body;
-            }
-          }
-        }
-      }
-
-      // Find all style.css files up the tree and bundle them into a single CSS file per folder path
-      // (Generate mode: one CSS bundle per unique folder, minimizing requests per page load)
-      let styleLink = "";
-      try {
-        const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-        const folderRelative = (dir === "/" || dir === "") ? "" : dir;
-
-        // Check bundle cache first (dirs with same CSS ancestry share the same bundle)
-        let cachedBundleUrl = docBundleCache.get(`css:${dirKey}`);
-        if (cachedBundleUrl !== undefined) {
-          if (cachedBundleUrl) {
-            styleLink = `<link rel="stylesheet" href="${cachedBundleUrl}" />`;
-          }
-        } else {
-          let cssPaths = cssPathCache.get(dirKey);
-          if (cssPaths === undefined) {
-            cssPaths = await findAllStyleCss(dirKey, _source);
-            cssPathCache.set(dirKey, cssPaths);
-          }
-          if (cssPaths.length > 0) {
-            // Copy all source CSS files to output (still needed for serve mode fallback)
-            for (const cssPath of cssPaths) {
-              if (!copiedCssFiles.has(cssPath)) {
-                const cssOutputPath = cssPath.replace(source, output);
-                const cssContent = await readFile(cssPath, 'utf8');
-                await outputFile(cssOutputPath, cssContent);
-                copiedCssFiles.add(cssPath);
-              }
-            }
-            // Bundle into a single file
-            const bundleUrl = await bundleDocumentCss(cssPaths, output, source, folderRelative, { minify: true });
-            docBundleCache.set(`css:${dirKey}`, bundleUrl);
-            styleLink = `<link rel="stylesheet" href="${bundleUrl}" />`;
-          } else {
-            docBundleCache.set(`css:${dirKey}`, null);
-          }
-        }
-      } catch (e) {
-        // ignore
-        console.error(e);
-      }
-
-      // Find all script.js files from docroot to current dir and bundle them
-      // (Generate mode: one JS bundle per unique folder, external not inlined)
-      let customScript = "";
-      try {
-        const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-        const folderRelative = (dir === "/" || dir === "") ? "" : dir;
-
-        let cachedBundleUrl = docBundleCache.get(`js:${dirKey}`);
-        if (cachedBundleUrl !== undefined) {
-          if (cachedBundleUrl) {
-            customScript = `<script src="${cachedBundleUrl}"></script>`;
-          }
-        } else {
-          let scriptPaths = scriptPathCache.get(dirKey);
-          if (scriptPaths === undefined) {
-            scriptPaths = await findAllScriptJs(dirKey, _source);
-            scriptPathCache.set(dirKey, scriptPaths);
-          }
-          if (scriptPaths.length > 0) {
-            const bundleUrl = await bundleDocumentJs(scriptPaths, output, source, folderRelative, { minify: true });
-            docBundleCache.set(`js:${dirKey}`, bundleUrl);
-            customScript = `<script src="${bundleUrl}"></script>`;
-          } else {
-            docBundleCache.set(`js:${dirKey}`, null);
-          }
-        }
-      } catch (e) {
-        // ignore
-        console.error(e);
-      }
-
-      const requestedTemplateName = fileMeta && fileMeta.template;
-      const templateName = requestedTemplateName || DEFAULT_TEMPLATE_NAME;
-      const template = templates[templateName];
-
-      if (!template) {
-        throw new Error(`Template not found. Requested: "${templateName}". Available templates: ${Object.keys(templates).join(', ') || 'none'}`);
-      }
-
-      // Register this document's dependencies for invalidation tracking
-      {
-        const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-        const cssDeps = cssPathCache.get(dirKey) || [];
-        const jsDeps = scriptPathCache.get(dirKey) || [];
-        dependencyTracker.registerDocument(file, {
-          templateName,
-          cssPaths: cssDeps,
-          scriptPaths: jsDeps,
-        });
-      }
-
-      // Check if this file has a custom menu
-      const customMenuInfo = getCustomMenuForFile(file, source, customMenus);
-
-      // Lazy evaluation of transformed metadata - only compute if template uses it
-      // This defers expensive custom transform function loading until actually needed
-      const templateUsesTransformedMeta = template.includes('${transformedMetadata}');
-      const lazyTransformedMeta = templateUsesTransformedMeta 
-        ? await getTransformedMeta() 
-        : '';
-
-      // Build final HTML with all replacements in a single regex pass
-      // This avoids creating 8 intermediate strings
-      // Append hydration script to customScript if present (for MDX with hydrate: true)
-      const finalCustomScript = hydrationScript 
-        ? customScript + '\n' + hydrationScript 
-        : customScript;
-      
-      const replacements = {
-        "${title}": fileMeta?.title || title,
-        "${menu}": menu,
-        "${meta}": JSON.stringify(fileMeta),
-        "${transformedMetadata}": lazyTransformedMeta,
-        "${body}": body,
-        "${styleLink}": styleLink,
-        "${customScript}": finalCustomScript,
-        "${searchIndex}": "[]", // Placeholder - search index written separately as JSON file
-        "${footer}": footer
-      };
-      // Single-pass replacement using regex alternation
-      const pattern = /\$\{(title|menu|meta|transformedMetadata|body|styleLink|customScript|searchIndex|footer)\}/g;
-      let finalHtml = template.replace(pattern, (match) => replacements[match] ?? match);
-
-      // Add menu data attributes to body
-      if (customMenuInfo) {
-        const menuPosition = customMenuInfo.menuPosition || 'top';
-        finalHtml = finalHtml.replace(
-          /<body([^>]*)>/,
-          `<body$1 data-custom-menu="${customMenuInfo.menuJsonPath}" data-menu-position="${menuPosition}">`
-        );
-      } else {
-        // No custom menu — default to top menu
-        finalHtml = finalHtml.replace(
-          /<body([^>]*)>/,
-          `<body$1 data-menu-position="top">`
-        );
-      }
-
-      // Resolve relative URLs in raw HTML elements (img src, etc.)
-      finalHtml = resolveRelativeUrls(finalHtml, docUrlPath);
-      
-      // Resolve links and mark broken internal links as inactive
-      finalHtml = markInactiveLinks(finalHtml, validPaths, docUrlPath, false);
-
-      // Transform image tags to use preview images with data-fullsrc for originals
-      // Skip in deferred mode - images will use original paths until preview generation completes
-      if (!_deferImages) {
-        finalHtml = transformImageTags(finalHtml, imageMap, docUrlPath);
-      }
-
-      // Add cache-busting timestamps to static file references
-      finalHtml = addTimestampToHtmlStaticRefs(finalHtml, cacheBustTimestamp);
-
-      await outputFile(outputFilename, finalHtml);
-      
-      // Clear finalHtml reference to allow GC
-      finalHtml = null;
-
-      // JSON output
-      const jsonOutputFilename = outputFilename.replace(".html", ".json");
-      
-      // Extract sections for markdown files
-      const sections = (type === '.md' || type === '.mdx') ? extractSections(rawBody) : [];
-      
-      // Use lazy metadata for JSON output - may have been computed above for HTML
-      const jsonTransformedMeta = await getTransformedMeta();
-      
-      const jsonObject = {
-        name: base,
-        url,
-        contents: rawBody,
-        bodyHtml: body,
-        metadata: fileMeta,
-        sections,
-        transformedMetadata: jsonTransformedMeta,
-        _ursa_metadata: ursaMetadata,
-      };
-      
-      // Store minimal data for directory indices, including metadata
-      dirIndexCache.set(file, {
-        name: base,
-        url,
-        metadata: fileMeta,
-      });
-      
-      const json = JSON.stringify(jsonObject);
-      await outputFile(jsonOutputFilename, json);
-
-      // XML output
-      const xmlOutputFilename = outputFilename.replace(".html", ".xml");
-      const xml = `<article>${o2x(jsonObject)}</article>`;
-      await outputFile(xmlOutputFilename, xml);
-      
-      // Update the content hash for this file
-      updateHash(file, rawBody, hashCache);
-      
-      // Update content timestamp since this file was regenerated (content changed)
-      contentTimestamps.set(relativePath, buildTimestamp);
-      // Also update the recentActivity entry we pushed earlier with the new timestamp
-      const activityEntry = recentActivity.find(e => e.url === searchUrl);
-      if (activityEntry) {
-        activityEntry.mtime = buildTimestamp;
-      }
-    } catch (e) {
-      progress.log(`Error processing ${file}: ${e.message}`);
-      errors.push({ file, phase: 'article-generation', error: e });
-    }
-  });
-
-  // Complete the articles status line
-  progress.done('Articles', `${totalArticles} done (${regeneratedCount} regenerated, ${skippedCount} unchanged) [${progress.stopTimer('Articles')}]`);
-  profiler.endPhase('Process articles');
-
-  // Phase: Write search index
-  // Can be deferred in serve mode for faster startup
-  let deferredSearchIndexPromise = null;
-  
-  const buildSearchIndex = async () => {
-    const indexStartTime = Date.now();
-    progress.logTimed('Building search index...');
-    
-    // Write search index as a separate JSON file (not embedded in each page)
-    const searchIndexPath = join(output, 'public', 'search-index.json');
-    progress.log(`Writing search index with ${searchIndex.length} entries`);
-    await outputFile(searchIndexPath, JSON.stringify(searchIndex));
-
-    // Build full-text index - use incremental mode when possible
-    let fullTextIndex;
-    if (!_clean && changedPaths.size > 0 && changedPaths.size < fullTextDocs.length) {
-      // Incremental update: only re-index changed documents
-      progress.log(`Incremental full-text index: ${changedPaths.size} changed, ${fullTextDocs.length - changedPaths.size} cached`);
-      fullTextIndex = buildIncrementalIndex(fullTextDocs, changedPaths, source);
-    } else {
-      // Full rebuild: clean build or all documents changed
-      progress.log(`Building full-text index from ${fullTextDocs.length} documents...`);
-      fullTextIndex = buildFullTextIndex(fullTextDocs);
-      // Save to cache for future incremental updates
-      saveIndexCache(source, fullTextIndex);
-    }
-    
-    const fullTextIndexPath = join(output, 'public', 'fulltext-index.json');
-    const fullTextIndexJson = JSON.stringify(fullTextIndex);
-    const wordCount = Object.keys(fullTextIndex).length;
-    progress.log(`Writing full-text index (${wordCount} unique words, ${(fullTextIndexJson.length / 1024).toFixed(1)} KB)`);
-    await outputFile(fullTextIndexPath, fullTextIndexJson);
-    
-    const elapsed = ((Date.now() - indexStartTime) / 1000).toFixed(1);
-    return { entries: searchIndex.length, words: wordCount, elapsed };
-  };
-  
-  if (_deferSearchIndex) {
-    // Deferred mode: start building in background, return promise
-    profiler.startPhase('Write search index (deferred)');
-    progress.startTimer('Search index');
-    progress.log('Search index building deferred for faster startup...');
-    deferredSearchIndexPromise = buildSearchIndex().then(result => {
-      progress.done('Search index (background)', `${result.entries} entries, ${result.words} words in ${result.elapsed}s`);
-      return result;
-    });
-    progress.done('Search index', 'deferred [0ms]');
-    profiler.endPhase('Write search index (deferred)');
-  } else {
-    // Normal mode: build search index now
-    profiler.startPhase('Write search index');
-    progress.startTimer('Search index');
-    const result = await buildSearchIndex();
-    progress.done('Search index', `${result.entries} entries, ${result.words} words [${progress.stopTimer('Search index')}]`);
-    profiler.endPhase('Write search index');
-  }
-
-  // Phase: Write recent activity data
-  profiler.startPhase('Write recent activity');
-  progress.startTimer('Recent activity');
-  // Sort by mtime descending, keep top 10
-  recentActivity.sort((a, b) => b.mtime - a.mtime);
-  const top10 = recentActivity.slice(0, 10);
-  const recentActivityPath = join(output, 'public', 'recent-activity.json');
-  await outputFile(recentActivityPath, JSON.stringify(top10));
-  progress.done('Recent activity', `${top10.length} entries [${progress.stopTimer('Recent activity')}]`);
-  profiler.endPhase('Write recent activity');
-
-  // Phase: Write menu data
-  profiler.startPhase('Write menu data');
-  progress.startTimer('Menu data');
-  // Write menu data as a separate JSON file (not embedded in each page)
-  // This dramatically reduces HTML file sizes for large sites
-  const menuDataPath = join(output, 'public', 'menu-data.json');
-  const menuDataJson = JSON.stringify(menuData);
-  progress.log(`Writing menu data (${(menuDataJson.length / 1024).toFixed(1)} KB)`);
-  await outputFile(menuDataPath, menuDataJson);
-
-  // Write custom menu JSON files
-  for (const [menuDir, menuInfo] of customMenus) {
-    const customMenuPath = join(output, menuInfo.menuJsonPath);
-    // Include menuPosition in the JSON so client knows how to render
-    const customMenuJson = JSON.stringify({
-      menuData: menuInfo.menuData,
-      menuPosition: menuInfo.menuPosition || 'top',
-    });
-    progress.log(`Writing custom menu: ${menuInfo.menuJsonPath}`);
-    await outputFile(customMenuPath, customMenuJson);
-  }
-  progress.done('Menu data', `${customMenus.size + 1} files [${progress.stopTimer('Menu data')}]`);
-  profiler.endPhase('Write menu data');
-
-  // Phase: Process directory indices
-  profiler.startPhase('Process directories');
-  progress.startTimer('Directories');
-  // Process directory indices with batched concurrency
-  const totalDirs = allSourceFilenamesThatAreDirectories.length;
-  let processedDirs = 0;
-  progress.log(`Processing ${totalDirs} directories...`);
-  await processBatched(allSourceFilenamesThatAreDirectories, async (dirPath) => {
-    try {
-      processedDirs++;
-      const shortDir = dirPath.replace(source, '');
-      progress.status('Directories', `${processedDirs}/${totalDirs} ${shortDir}`);
-
-      const pathsInThisDirectory = allSourceFilenames.filter((filename) =>
-        filename.match(new RegExp(`${dirPath}.+`))
-      );
-
-      // Use minimal directory index cache instead of full jsonCache
-      const jsonObjects = pathsInThisDirectory
-        .map((path) => {
-          const object = dirIndexCache.get(path);
-          return typeof object === "object" ? object : null;
-        })
-        .filter((a) => a);
-
-      const json = JSON.stringify(jsonObjects);
-
-      const outputFilename = dirPath.replace(source, output) + ".json";
-      await outputFile(outputFilename, json);
-
-      // html
-      const htmlOutputFilename = dirPath.replace(source, output) + ".html";
-      const indexAlreadyExists = await fileExists(htmlOutputFilename);
-      if (!indexAlreadyExists) {
-        const template = templates["default-template"];
-        const indexHtml = `<ul>${pathsInThisDirectory
-          .map((path) => {
-            const partialPath = path
-              .replace(source, "")
-              .replace(parse(path).ext, ".html");
-            const name = basename(path, parse(path).ext);
-            return `<li><a href="${partialPath}">${name}</a></li>`;
-          })
-          .join("")}</ul>`;
-        let finalHtml = template;
-        const replacements = {
-          "${menu}": menu,
-          "${body}": indexHtml,
-          "${searchIndex}": "[]", // Search index now in separate file
-          "${title}": "Index",
-          "${meta}": "{}",
-          "${transformedMetadata}": "",
-          "${styleLink}": "",
-          "${footer}": footer
-        };
-        for (const [key, value] of Object.entries(replacements)) {
-          finalHtml = finalHtml.replace(key, value);
-        }
-        // Add cache-busting timestamps to static file references
-        finalHtml = addTimestampToHtmlStaticRefs(finalHtml, cacheBustTimestamp);
-        await outputFile(htmlOutputFilename, finalHtml);
-      }
-    } catch (e) {
-      progress.log(`Error processing directory ${dirPath}: ${e.message}`);
-      errors.push({ file: dirPath, phase: 'directory-index', error: e });
-    }
-  });
-  
-  progress.done('Directories', `${totalDirs} done [${progress.stopTimer('Directories')}]`);
-  profiler.endPhase('Process directories');
-
-  // Clear directory index cache to free memory before processing static files
-  dirIndexCache.clear();
-
-  // Phase: Process static files
-  profiler.startPhase('Process static files');
-  progress.startTimer('Static files');
-  // Copy static HTML files (images were already processed above with preview generation)
-  // Note: Images are processed before articles to enable preview transformation in HTML
-  
-  // Also copy existing HTML files from source to output (they're treated as static)
-  const allSourceFilenamesThatAreHtml = allSourceFilenames.filter(
-    (filename) => filename.match(/\.html$/) && !filename.match(hiddenOrSystemDirs)
-  );
-  
-  const allStaticFiles = allSourceFilenamesThatAreHtml;
-  const totalStatic = allStaticFiles.length;
-  let processedStatic = 0;
-  let copiedStatic = 0;
-  progress.log(`Processing ${totalStatic} static HTML files...`);
-  await processBatched(allStaticFiles, async (file) => {
-    try {
-      processedStatic++;
-      const shortFile = file.replace(source, '');
-      progress.status('Static files', `${processedStatic}/${totalStatic} ${shortFile}`);
-
-      // Check if file has changed using file stat as a quick check
-      const fileStat = await stat(file);
-      const statKey = `${file}:stat`;
-      const newStatHash = `${fileStat.size}:${fileStat.mtimeMs}`;
-      if (hashCache.get(statKey) === newStatHash) {
-        return; // Skip unchanged static file
-      }
-      hashCache.set(statKey, newStatHash);
-      copiedStatic++;
-
-      const outputFilename = file.replace(source, output);
-      await mkdir(dirname(outputFilename), { recursive: true });
-
-      if (file.endsWith('.css')) {
-        // Process CSS for cache busting
-        const cssContent = await readFile(file, 'utf8');
-        const processedCss = addTimestampToCssUrls(cssContent, cacheBustTimestamp);
-        await outputFile(outputFilename, processedCss);
-      } else if (file.endsWith('.html')) {
-        // Process HTML files for link resolution
-        let htmlContent = await readFile(file, 'utf8');
-        // Calculate the document's URL path for relative link resolution
-        const docUrlPath = '/' + file.replace(source, '').replace(/^\//, '');
-        // Resolve relative URLs in raw HTML elements (img src, etc.)
-        htmlContent = resolveRelativeUrls(htmlContent, docUrlPath);
-        // Resolve internal links to have proper .html extensions
-        htmlContent = markInactiveLinks(htmlContent, validPaths, docUrlPath, false);
-        // Transform image tags to use preview images with data-fullsrc for originals
-        // Skip in deferred mode - images will use original paths until preview generation completes
-        if (!_deferImages) {
-          htmlContent = transformImageTags(htmlContent, imageMap, docUrlPath);
-        }
-        // Add cache-busting timestamps
-        htmlContent = addTimestampToHtmlStaticRefs(htmlContent, cacheBustTimestamp);
-        await outputFile(outputFilename, htmlContent);
-      } else {
-        await copyFile(file, outputFilename);
-      }
-    } catch (e) {
-      progress.log(`Error processing static file ${file}: ${e.message}`);
-      errors.push({ file, phase: 'static-file', error: e });
-    }
-  });
-  
-  progress.done('Static files', `${totalStatic} done (${copiedStatic} copied) [${progress.stopTimer('Static files')}]`);
-  profiler.endPhase('Process static files');
-
-  // Phase: Auto-index generation
-  profiler.startPhase('Auto-index generation');
-  progress.startTimer('Auto-index');
-  // Automatic index generation for folders without index.html
-  progress.log(`Checking for missing index files...`);
-  await generateAutoIndices(output, allSourceFilenamesThatAreDirectories, source, templates, menu, footer, allSourceFilenamesThatAreArticles, copiedCssFiles, existingHtmlFiles, cacheBustTimestamp, progress, customMenus);
-  progress.done('Auto-index', `checked ${allSourceFilenamesThatAreDirectories.length} directories [${progress.stopTimer('Auto-index')}]`);
-  profiler.endPhase('Auto-index generation');
-
-  // Phase: Finalization
-  profiler.startPhase('Finalization');
-  progress.startTimer('Finalization');
-  // Save the hash cache to .ursa folder in source directory
-  if (hashCache.size > 0) {
-    await saveHashCache(source, hashCache);
-  }
-  
-  // Save content timestamps to .ursa.json (tracks when content actually changed)
-  if (contentTimestamps.size > 0) {
-    saveContentTimestamps(source, contentTimestamps);
-    progress.log(`Saved ${contentTimestamps.size} content timestamps`);
-  }
-
-  // Persist the dependency tracker so hash-skipped documents keep their
-  // edges on the next warm start (invalidation plans stay accurate)
-  await saveDependencyTracker(source);
-
-  // Populate watch mode cache for fast single-file regeneration
-  watchModeCache.templates = templates;
-  watchModeCache.menu = menu;
-  watchModeCache.footer = footer;
-  watchModeCache.validPaths = validPaths;
-  watchModeCache.source = source;
-  watchModeCache.meta = meta;
-  watchModeCache.output = output;
-  watchModeCache.hashCache = hashCache;
-  watchModeCache.cacheBustTimestamp = cacheBustTimestamp;
-  watchModeCache.cacheBustHashes = cacheBustHashes;
-  watchModeCache.allArticlePaths = [...allSourceFilenamesThatAreArticles];
-  watchModeCache.imageMap = imageMap;
-  watchModeCache.customMenus = customMenus;
-  watchModeCache.ursaMetadata = ursaMetadata;
-  watchModeCache.lastFullBuild = Date.now();
-  watchModeCache.isInitialized = true;
-  const depStats = dependencyTracker.getStats();
-  progress.log(`Watch cache initialized (${depStats.totalDocuments} documents, ${depStats.uniqueFiles} dependencies tracked)`);
-
-  // Write error report if there were any errors
-  if (errors.length > 0) {
-    const errorReportPath = join(output, '_errors.log');
-    const failedFiles = errors.map(e => e.file);
-    
-    let report = `URSA GENERATION ERROR REPORT\n`;
-    report += `Generated: ${new Date().toISOString()}\n`;
-    report += `Total errors: ${errors.length}\n\n`;
-    report += `${'='.repeat(60)}\n`;
-    report += `FAILED FILES:\n`;
-    report += `${'='.repeat(60)}\n\n`;
-    failedFiles.forEach(f => {
-      report += `  - ${f}\n`;
-    });
-    report += `\n${'='.repeat(60)}\n`;
-    report += `ERROR DETAILS:\n`;
-    report += `${'='.repeat(60)}\n\n`;
-    
-    errors.forEach(({ file, phase, error }) => {
-      report += `${'─'.repeat(60)}\n`;
-      report += `File: ${file}\n`;
-      report += `Phase: ${phase}\n`;
-      report += `Error: ${error.message}\n`;
-      if (error.stack) {
-        report += `Stack:\n${error.stack}\n`;
-      }
-      report += `\n`;
-    });
-    
-    await outputFile(errorReportPath, report);
-    progress.log(`\n⚠️  ${errors.length} error(s) occurred during generation.`);
-    progress.log(`   Error report written to: ${errorReportPath}\n`);
-  } else {
-    progress.log(`\n✅ Generation complete with no errors.\n`);
-  }
-  
-  progress.done('Finalization', `complete [${progress.stopTimer('Finalization')}]`);
-  profiler.endPhase('Finalization');
-  
-  // Print profiler report
-  progress.log(profiler.report());
-  
-  // Terminate worker pool so threads don't keep the process alive
-  await terminateParserPool();
-
-  // Return deferred processing promises if in deferred mode
-  // Caller can await these to know when background processing is complete
-  return {
-    deferredImageProcessing: deferredImageProcessingPromise,
-    deferredSearchIndex: deferredSearchIndexPromise
-  };
-}
-
-/**
- * Regenerate multiple documents affected by a dependency change (e.g., style.css, script.js, template).
- * Uses the watchModeCache and dependency tracker to efficiently re-render affected documents
- * with updated cache-bust timestamps.
- *
- * @param {string[]} documentPaths - Absolute paths to documents to regenerate
- * @param {Object} options
- * @param {string} options._source - Source directory
- * @param {string} options._meta - Meta directory
- * @param {string} options._output - Output directory
- * @param {string} [options.reason] - Reason for regeneration (for logging)
- * @param {string[]} [options.priorityPaths] - Document paths to regenerate first (e.g. client-viewed docs)
- * @param {function} [options.onPriorityComplete] - Callback after priority paths are done (receives { regenerated, failed })
- * @returns {Promise<{success: boolean, message: string, regenerated: number, failed: number}>}
- */
-export async function regenerateAffectedDocuments(documentPaths, {
-  _source,
-  _meta,
-  _output,
-  reason = "dependency change",
-  priorityPaths = [],
-  onPriorityComplete = null,
-} = {}) {
-  const startTime = Date.now();
-
-  if (!watchModeCache.isInitialized) {
-    return { success: false, message: "Cache not initialized - need full build first", regenerated: 0, failed: 0 };
-  }
-
-  if (documentPaths.length === 0) {
-    return { success: true, message: "No documents to regenerate", regenerated: 0, failed: 0 };
-  }
-
-  // Generate a fresh cache-bust timestamp for this invalidation pass
-  const newTimestamp = generateCacheBustTimestamp();
-  watchModeCache.cacheBustTimestamp = newTimestamp;
-
-  let regenerated = 0;
-  let failed = 0;
-
-  // Separate priority paths from the rest
-  const prioritySet = new Set(priorityPaths.map(p => resolve(p)));
-  const priorityDocs = documentPaths.filter(p => prioritySet.has(resolve(p)));
-  const remainingDocs = documentPaths.filter(p => !prioritySet.has(resolve(p)));
-
-  if (priorityDocs.length > 0) {
-    console.log(`🔄 Regenerating ${priorityDocs.length} priority documents first, then ${remainingDocs.length} remaining (${reason})`);
-  } else {
-    console.log(`🔄 Regenerating ${documentPaths.length} documents (${reason})`);
-  }
-
-  // Process priority documents first
-  for (const docPath of priorityDocs) {
-    try {
-      const result = await regenerateSingleFile(docPath, { _source, _meta, _output });
-      if (result.success) {
-        regenerated++;
-      } else {
-        console.warn(`  ⚠️  ${docPath}: ${result.message}`);
-        failed++;
-      }
-    } catch (e) {
-      console.error(`  ❌ ${docPath}: ${e.message}`);
-      failed++;
-    }
-  }
-
-  // Notify caller that priority docs are done (so server can reload those clients immediately)
-  if (priorityDocs.length > 0 && onPriorityComplete) {
-    try {
-      onPriorityComplete({ regenerated, failed, priorityDocs });
-    } catch (e) {
-      console.error(`  ⚠️ onPriorityComplete callback error: ${e.message}`);
-    }
-  }
-
-  // Process remaining documents
-  for (const docPath of remainingDocs) {
-    try {
-      const result = await regenerateSingleFile(docPath, { _source, _meta, _output });
-      if (result.success) {
-        regenerated++;
-      } else {
-        console.warn(`  ⚠️  ${docPath}: ${result.message}`);
-        failed++;
-      }
-    } catch (e) {
-      console.error(`  ❌ ${docPath}: ${e.message}`);
-      failed++;
-    }
-  }
-
-  // Persist updated dependency registrations (e.g. a doc switched templates)
-  if (regenerated > 0) {
-    await saveDependencyTracker(resolve(_source) + "/");
-  }
-
-  const elapsed = Date.now() - startTime;
-  const msg = `Regenerated ${regenerated}/${documentPaths.length} documents in ${elapsed}ms (${reason})${failed > 0 ? `, ${failed} failed` : ""}`;
-  console.log(`✅ ${msg}`);
-  return { success: failed === 0, message: msg, regenerated, failed };
-}
-
-/**
- * Regenerate a single file without scanning the entire source directory.
- * This is much faster for watch mode - only regenerate what changed.
- * 
- * @param {string} changedFile - Absolute path to the file that changed
- * @param {Object} options - Same options as generate()
- * @returns {Promise<{success: boolean, message: string}>}
- */
-export async function regenerateSingleFile(changedFile, {
-  _source,
-  _meta,
-  _output,
-} = {}) {
-  const startTime = Date.now();
-  const source = resolve(_source) + "/";
-  const meta = resolve(_meta);
-  const output = resolve(_output) + "/";
-  
-  // Check if this is an article file we can regenerate
-  const articleExtensions = /\.(md|mdx|txt|yml)$/;
-  if (!changedFile.match(articleExtensions)) {
-    return { success: false, message: `Not an article file: ${changedFile}` };
-  }
-  
-  // Check if cache is initialized
-  if (!watchModeCache.isInitialized) {
-    return { success: false, message: 'Cache not initialized - need full build first' };
-  }
-  
-  // Verify paths match cached paths
-  if (watchModeCache.source !== source || watchModeCache.output !== output) {
-    return { success: false, message: 'Paths changed - need full rebuild' };
-  }
-  
-  try {
-    const { templates, menu, footer, validPaths, hashCache, cacheBustTimestamp, imageMap, customMenus, ursaMetadata } = watchModeCache;
-
-    const rawBody = await readFile(changedFile, "utf8");
-    const type = parse(changedFile).ext;
-    const ext = extname(changedFile);
-    const base = basename(changedFile, ext);
-    const dir = addTrailingSlash(dirname(changedFile)).replace(source, "");
-    
-    // Calculate output paths
-    const outputFilename = changedFile
-      .replace(source, output)
-      .replace(parse(changedFile).ext, ".html");
-    const url = '/' + outputFilename.replace(output, '');
-    
-    // Title from filename (for index/home, use parent folder name)
-    const titleBase = (base === 'index' || base === 'home') ? basename(dirname(changedFile)) : base;
-    const title = toTitleCase(titleBase || base);
-    
-    // Extract metadata
-    const fileMeta = extractMetadata(rawBody);
-    const transformedMetadata = await getTransformedMetadata(
-      dirname(changedFile),
-      fileMeta
-    );
-    
-    // Calculate the document's URL path
-    const docUrlPath = '/' + dir + base + '.html';
-    
-    // Check if hydration should be enabled for MDX files
-    const shouldHydrate = type === '.mdx' && fileMeta?.hydrate === true;
-    
-    // Render body (use async for .mdx, sync for .md/.txt)
-    let body;
-    let hydrationScript = '';
-    if (type === '.mdx') {
-      const renderResult = await renderFileAsync({
-        fileContents: rawBody,
-        type,
-        dirname: dir,
-        basename: base,
-        filePath: changedFile,
-        sourceRoot: source,
-        useWorker: false,
-        hydrate: shouldHydrate,
-      });
-      
-      // Handle the result - can be string or { html, hydrationScript }
-      if (typeof renderResult === 'object' && renderResult.html) {
-        body = renderResult.html;
-        hydrationScript = renderResult.hydrationScript || '';
-      } else {
-        body = renderResult;
-      }
-    } else {
-      body = renderFile({
-        fileContents: rawBody,
-        type,
-        dirname: dir,
-        basename: base,
-      });
-    }
-
-    // Inject default H1 if body doesn't start with one
-    if (!body || !body.trimStart().startsWith('<h1')) {
-      const h1Title = fileMeta?.title || title;
-      body = `<h1>${h1Title}</h1>\n` + (body || '');
-    }
-
-    // Inject breadcrumbs before the H1
-    const breadcrumbs = generateBreadcrumbs(dir, base, fileMeta);
-    if (breadcrumbs) {
-      body = breadcrumbs + body;
-    }
-    
-    // Inject frontmatter table for markdown/mdx files
-    if ((type === '.md' || type === '.mdx') && fileMeta) {
-      body = injectFrontmatterTable(body, fileMeta);
-    }
-
-    // Handle auto-index generation for index files with generate-auto-index: true
-    if (base === 'index' && fileMeta) {
-      const autoIndexConfig = getAutoIndexConfig(fileMeta);
-      if (autoIndexConfig.enabled) {
-        // Generate auto-index HTML for this directory from source
-        const sourceDir = dirname(changedFile);
-        const autoIndexHtml = await generateAutoIndexHtmlFromSource(sourceDir, autoIndexConfig.depth);
-        
-        if (autoIndexHtml) {
-          if (autoIndexConfig.position === 'bottom') {
-            body = body + '\n' + autoIndexHtml;
-          } else {
-            body = autoIndexHtml + '\n' + body;
-          }
-        }
-      }
-    }
-
-    // Find all CSS files up the tree and create separate link tags
-    // (regenerateSingleFile is used in serve mode, so separate tags per level for invalidation)
-    let styleLink = "";
-    let cssPaths = [];
-    try {
-      const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-      cssPaths = await findAllStyleCss(dirKey, _source);
-      if (cssPaths.length > 0) {
-        // Copy all CSS files to output (always copy in single-file mode to ensure up to date)
-        for (const cssPath of cssPaths) {
-          const cssOutputPath = cssPath.replace(source, output);
-          const cssContent = await readFile(cssPath, 'utf8');
-          await outputFile(cssOutputPath, cssContent);
-        }
-        styleLink = generateSeparateCssTags(cssPaths, source);
-      }
-    } catch (e) {
-      // ignore
-    }
-    
-    // Get template
-    const requestedTemplateName = fileMeta && fileMeta.template;
-    const template =
-      templates[requestedTemplateName] || templates[DEFAULT_TEMPLATE_NAME];
-    
-    if (!template) {
-      return { success: false, message: `Template not found: ${requestedTemplateName || DEFAULT_TEMPLATE_NAME}` };
-    }
-    
-    // Find all script.js files from docroot to current dir and serve as separate external tags
-    // (Serve mode: separate tags per level for individual invalidation)
-    let customScript = "";
-    let scriptPaths = [];
-    try {
-      const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
-      scriptPaths = await findAllScriptJs(dirKey, _source);
-      if (scriptPaths.length > 0) {
-        // Copy all script files to output so they can be served
-        for (const scriptPath of scriptPaths) {
-          const scriptOutputPath = scriptPath.replace(source, output);
-          const scriptContent = await readFile(scriptPath, 'utf8');
-          await outputFile(scriptOutputPath, scriptContent);
-        }
-        customScript = generateSeparateJsTags(scriptPaths, source);
-      }
-    } catch (e) {
-      // ignore
-    }
-
-    // Register this document's dependencies (template, inherited CSS/JS) so
-    // invalidation plans stay accurate after frontmatter/template changes.
-    // Persisted to .ursa/dependency-graph.json by regenerateAffectedDocuments.
-    const usedTemplateName = (requestedTemplateName && templates[requestedTemplateName])
-      ? requestedTemplateName
-      : DEFAULT_TEMPLATE_NAME;
-    dependencyTracker.registerDocument(changedFile, {
-      templateName: usedTemplateName,
-      cssPaths,
-      scriptPaths,
-    });
-
-    // Check if this file has a custom menu
-    const customMenuInfo = customMenus ? getCustomMenuForFile(changedFile, source, customMenus) : null;
-
-    // Append hydration script to customScript if present (for MDX with hydrate: true)
-    const finalCustomScript = hydrationScript 
-      ? customScript + '\n' + hydrationScript 
-      : customScript;
-
-    // Build final HTML
-    let finalHtml = template;
-    const replacements = {
-      "${title}": fileMeta?.title || title,
-      "${menu}": menu,
-      "${meta}": JSON.stringify(fileMeta),
-      "${transformedMetadata}": transformedMetadata,
-      "${body}": body,
-      "${styleLink}": styleLink,
-      "${customScript}": finalCustomScript,
-      "${searchIndex}": "[]",
-      "${footer}": footer
-    };
-    for (const [key, value] of Object.entries(replacements)) {
-      finalHtml = finalHtml.replace(key, value);
-    }
-
-    // If this page has a custom menu, add data attributes to body
-    if (customMenuInfo) {
-      const menuPosition = customMenuInfo.menuPosition || 'top';
-      finalHtml = finalHtml.replace(
-        /<body([^>]*)>/,
-        `<body$1 data-custom-menu="${customMenuInfo.menuJsonPath}" data-menu-position="${menuPosition}">`
-      );
-    } else {
-      // No custom menu — default to top menu
-      finalHtml = finalHtml.replace(
-        /<body([^>]*)>/,
-        `<body$1 data-menu-position="top">`
-      );
-    }
-    
-    // Resolve relative URLs in raw HTML elements (img src, etc.)
-    finalHtml = resolveRelativeUrls(finalHtml, docUrlPath);
-    
-    // Mark broken links
-    finalHtml = markInactiveLinks(finalHtml, validPaths, docUrlPath, false);
-    
-    // Transform image tags to use preview images with data-fullsrc for originals
-    if (imageMap) {
-      finalHtml = transformImageTags(finalHtml, imageMap, docUrlPath);
-    }
-    
-    // Add cache-busting timestamps to static file references
-    finalHtml = addTimestampToHtmlStaticRefs(finalHtml, cacheBustTimestamp);
-    
-    await outputFile(outputFilename, finalHtml);
-    
-    // JSON output
-    const jsonOutputFilename = outputFilename.replace(".html", ".json");
-    const sections = (type === '.md' || type === '.mdx') ? extractSections(rawBody) : [];
-    const jsonObject = {
-      name: base,
-      url,
-      contents: rawBody,
-      bodyHtml: body,
-      metadata: fileMeta,
-      sections,
-      transformedMetadata,
-      _ursa_metadata: ursaMetadata,
-    };
-    const json = JSON.stringify(jsonObject);
-    await outputFile(jsonOutputFilename, json);
-    
-    // XML output
-    const xmlOutputFilename = outputFilename.replace(".html", ".xml");
-    const xml = `<article>${o2x(jsonObject)}</article>`;
-    await outputFile(xmlOutputFilename, xml);
-
-    // Folder-named index promotion (mirrors autoIndex behavior on full builds):
-    // If the file is named like its parent folder (e.g. aletheia/aletheia.md) and
-    // no explicit index.{md,mdx,txt,yml,html} exists alongside it, also write the
-    // same outputs to <dir>/index.html|json|xml so the canonical URL stays fresh.
-    const sourceDirOfFile = dirname(changedFile);
-    const parentFolderName = basename(sourceDirOfFile);
-    if (base && parentFolderName && base === parentFolderName) {
-      const hasExplicitIndex = ['index.md', 'index.mdx', 'index.txt', 'index.yml', 'index.html']
-        .some(name => existsSync(join(sourceDirOfFile, name)));
-      if (!hasExplicitIndex) {
-        const outDirOfFile = dirname(outputFilename);
-        await outputFile(join(outDirOfFile, 'index.html'), finalHtml);
-        await outputFile(join(outDirOfFile, 'index.json'), json);
-        await outputFile(join(outDirOfFile, 'index.xml'), xml);
-      }
-    }
-
-    // Update hash cache
-    updateHash(changedFile, rawBody, hashCache);
-    
-    // Update recent-activity.json with this file's new content timestamp
-    try {
-      const now = Date.now();
-      const recentActivityPath = join(output, 'public', 'recent-activity.json');
-      let recentActivity = [];
-      try {
-        const existing = await readFile(recentActivityPath, 'utf8');
-        recentActivity = JSON.parse(existing);
-      } catch (e) { /* no existing file, start fresh */ }
-      // Remove old entry for this URL if present
-      recentActivity = recentActivity.filter(r => r.url !== url);
-      // Add updated entry with current timestamp (content changed now)
-      recentActivity.push({ title, url, mtime: now });
-      // Sort by mtime descending, keep top 10
-      recentActivity.sort((a, b) => b.mtime - a.mtime);
-      recentActivity = recentActivity.slice(0, 10);
-      await outputFile(recentActivityPath, JSON.stringify(recentActivity));
-      
-      // Also update content timestamp in .ursa.json for persistence
-      const relativePath = '/' + changedFile.replace(source, '').replace(/\.(md|mdx|txt|yml)$/, '.html');
-      updateContentTimestamp(source, relativePath, now);
-    } catch (e) {
-      // ignore recent activity update errors
-    }
-    
-    const elapsed = Date.now() - startTime;
-    const shortFile = changedFile.replace(source, '');
-    return { success: true, message: `Regenerated ${shortFile} in ${elapsed}ms` };
-  } catch (e) {
-    return { success: false, message: `Error: ${e.message}` };
-  }
+import { recurse } from "../helper/recursive-readdir.js";
+import { copyFile, mkdir, readdir, readFile, stat } from "fs/promises";
+import { getAutomenu } from "../helper/automenu.js";
+import { filterAsync } from "../helper/filterAsync.js";
+import { isDirectory } from "../helper/isDirectory.js";
+import { isFolderHidden, clearConfigCache } from "../helper/folderConfig.js";
+import { isHiddenOrSystemPath } from "../helper/hiddenPaths.js";
+import { IMAGE_EXTENSIONS, isMedia } from "../helper/staticAssets.js";
+import {
+  extractMetadata,
+  extractRawMetadata,
+  isMetadataOnly,
+  getAutoIndexConfig,
+} from "../helper/metadataExtractor.js";
+import { injectFrontmatterTable } from "../helper/frontmatterTable.js";
+import {
+  hashContent,
+  loadHashCache,
+  saveHashCache,
+  needsRegeneration,
+  outputsExist,
+  updateHash,
+  getUrsaDir,
+} from "../helper/contentHash.js";
+import {
+  buildValidPaths,
+  markInactiveLinks,
+  resolveRelativeUrls,
+} from "../helper/linkValidator.js";
+import { getAndIncrementBuildId, loadContentTimestamps, saveContentTimestamps, updateContentTimestamp } from "../helper/ursaConfig.js";
+import { extractSections } from "../helper/sectionExtractor.js";
+import { renderFile, renderFileAsync, terminateParserPool } from "../helper/fileRenderer.js";
+import { buildReactRuntime } from "../helper/mdxRenderer.js";
+import { findStyleCss, findAllStyleCss } from "../helper/findStyleCss.js";
+import { findScriptJs, findAllScriptJs } from "../helper/findScriptJs.js";
+import { bundleMetaTemplateAssets, bundleDocumentCss, bundleDocumentJs, clearMetaBundleCache, generateSeparateCssTags, generateSeparateJsTags } from "../helper/assetBundler.js";
+import { buildFullTextIndex, buildIncrementalIndex, loadIndexCache, saveIndexCache } from "../helper/fullTextIndex.js";
+import { dependencyTracker, loadDependencyTracker, saveDependencyTracker } from "../helper/dependencyTracker.js";
+import { CacheBustHashMap } from "../helper/build/cacheBust.js";
+import { copy as copyDir, emptyDir, outputFile, remove } from "fs-extra";
+import { basename, dirname, extname, join, parse, resolve } from "path";
+import { URL } from "url";
+import o2x from "object-to-xml";
+import { existsSync } from "fs";
+import { createWhitelistFilter } from "../helper/whitelistFilter.js";
+import { processAllImages, transformImageTags, clearImageCache, copyAllImagesFast } from "../helper/imageProcessor.js";
+import { extractImageReferences } from "../helper/imageExtractor.js";
+import { checkFileSize, readFileStreaming, formatFileSize } from "../helper/streamingReader.js";
+import { generateBreadcrumbs } from "../helper/breadcrumbs.js";
+import {
+  loadNavCache,
+  saveNavCache,
+  hashFileList,
+  hashFileStats,
+  isNavCacheValid,
+  createNavCacheEntry,
+  restoreMap,
+} from "../helper/build/navCache.js";
+
+// Import build helpers from organized modules
+import {
+  generateCacheBustTimestamp,
+  addTimestampToCssUrls,
+  addTimestampToHtmlStaticRefs,
+  processBatched,
+  ProgressReporter,
+  watchModeCache,
+  clearWatchCache as clearWatchCacheBase,
+  toTitleCase,
+  parseExcludeOption,
+  createExcludeFilter,
+  addTrailingSlash,
+  getTemplates,
+  getMenu,
+  findAllCustomMenus,
+  getCustomMenuForFile,
+  getTransformedMetadata,
+  getFooter,
+  getUrsaMetadata,
+  generateAutoIndices,
+  generateAutoIndexHtmlFromSource,
+  copyMetaAssets,
+} from "../helper/build/index.js";
+import { getProfiler } from "../helper/build/profiler.js";
+import { reconcileAll, isInsideTemplatesFolder } from "../helper/documentTemplates.js";
+
+// Concurrency limiter for batch processing to avoid memory exhaustion
+const BATCH_SIZE = parseInt(process.env.URSA_BATCH_SIZE || '50', 10);
+
+// Cache for CSS path lookups to avoid repeated filesystem walks
+const cssPathCache = new Map();
+
+// Cache for script path lookups to avoid repeated filesystem walks
+const scriptPathCache = new Map();
+
+// Cache for document-level CSS/JS bundle paths to avoid re-bundling identical sets
+const docBundleCache = new Map();
+
+// Wrapper for clearWatchCache that passes cssPathCache and scriptPathCache
+export function clearWatchCache() {
+  clearWatchCacheBase(cssPathCache);
+  scriptPathCache.clear();
+  docBundleCache.clear();
+  clearMetaBundleCache();
+}
+
+// Clear just the script-related caches (for when script.js changes)
+export function clearScriptCache() {
+  scriptPathCache.clear();
+  // Clear all JS bundle cache entries
+  for (const key of docBundleCache.keys()) {
+    if (key.startsWith('js:')) {
+      docBundleCache.delete(key);
+    }
+  }
+}
+
+// Clear just the CSS-related caches (for when style.css changes)
+export function clearStyleCache() {
+  cssPathCache.clear();
+  // Clear all CSS bundle cache entries
+  for (const key of docBundleCache.keys()) {
+    if (key.startsWith('css:')) {
+      docBundleCache.delete(key);
+    }
+  }
+}
+
+const progress = new ProgressReporter();
+
+const DEFAULT_TEMPLATE_NAME =
+  process.env.DEFAULT_TEMPLATE_NAME ?? "default-template";
+
+export async function generate({
+  _source = join(process.cwd(), "."),
+  _meta = join(process.cwd(), "meta"),
+  _output = join(process.cwd(), "build"),
+  _whitelist = null,
+  _exclude = null,
+  _incremental = false,  // Legacy flag, now ignored (always incremental)
+  _clean = false,  // When true, ignore cache and regenerate all files
+  _deferImages = false,  // When true, copy images without processing, return promise for background processing
+  _deferSearchIndex = false,  // When true, return promise for search index building (for faster startup)
+} = {}) {
+  // Initialize profiler for this build
+  const profiler = getProfiler(true);
+  
+  console.log({ _source, _meta, _output, _whitelist, _exclude, _clean, _deferImages, _deferSearchIndex });
+  const source = resolve(_source) + "/";
+  const meta = resolve(_meta);
+  const output = resolve(_output) + "/";
+  console.log({ source, meta, output });
+
+  // Generate cache-busting timestamp for this build
+  const cacheBustTimestamp = generateCacheBustTimestamp();
+  const cacheBustHashes = new CacheBustHashMap();
+  progress.logTimed(`Cache-bust timestamp: ${cacheBustTimestamp}`);
+
+  // Initialize dependency tracker for this build
+  dependencyTracker.init(source);
+
+  // Clear output directory and cache when --clean is specified
+  if (_clean) {
+    progress.startTimer('Clean');
+    const ursaDir = getUrsaDir(source);
+    progress.logTimed(`Clean build: deleting cache folder ${ursaDir}`);
+    await remove(ursaDir);
+    progress.logTimed(`Clean build: clearing output directory ${output}`);
+    await emptyDir(output);
+    progress.logTimed(`Clean complete [${progress.stopTimer('Clean')}]`);
+  } else {
+    // Warm start: reload persisted dependency registrations so hash-skipped
+    // documents keep their edges (current-run registrations take precedence)
+    const loaded = await loadDependencyTracker(source);
+    if (loaded) {
+      const stats = dependencyTracker.getStats();
+      progress.logTimed(`Dependency graph loaded: ${stats.totalDocuments} documents, ${stats.uniqueFiles} dependencies`);
+    }
+  }
+
+  // Phase: Scan source files
+  profiler.startPhase('Scan source files');
+  progress.startTimer('Scan');
+  const allSourceFilenamesUnfiltered = await recurse(source, [() => false]);
+  progress.logTimed(`Scanned ${allSourceFilenamesUnfiltered.length} files [${progress.stopTimer('Scan')}]`);
+  profiler.endPhase('Scan source files');
+  
+  // Phase: Filter and classify files
+  profiler.startPhase('Filter & classify');
+  progress.startTimer('Filter');
+  
+  // Apply include filter (existing functionality)
+  const includeFilter = process.env.INCLUDE_FILTER
+    ? (fileName) => fileName.match(process.env.INCLUDE_FILTER)
+    : Boolean;
+  let allSourceFilenames = allSourceFilenamesUnfiltered.filter(includeFilter);
+  
+  // Apply exclude filter if specified
+  if (_exclude) {
+    const excludedPaths = await parseExcludeOption(_exclude, source);
+    const excludeFilter = createExcludeFilter(excludedPaths, source);
+    const beforeCount = allSourceFilenames.length;
+    allSourceFilenames = allSourceFilenames.filter(excludeFilter);
+    progress.logTimed(`Exclude filter applied: ${beforeCount - allSourceFilenames.length} files excluded`);
+  }
+  
+  // Apply whitelist filter if specified
+  if (_whitelist) {
+    const whitelistFilter = await createWhitelistFilter(_whitelist, source);
+    allSourceFilenames = allSourceFilenames.filter(whitelistFilter);
+    progress.logTimed(`Whitelist applied: ${allSourceFilenames.length} files after filtering`);
+  }
+
+  // Clear config cache at start of generation to pick up any changes
+  clearConfigCache();
+
+  // Helper to check if a path is inside a config-hidden folder
+  const isInHiddenFolder = (filePath) => {
+    const dir = dirname(filePath);
+    return isFolderHidden(dir, source);
+  };
+
+  // read all articles, process them, copy them to build
+  const articleExtensions = /\.(md|mdx|txt|yml)$/;
+  // Hidden/system folders are judged RELATIVE to the docroot — see
+  // helper/hiddenPaths.js for why testing the absolute path silently produces
+  // an empty site when the checkout lives under a dot-directory.
+  const isHiddenOrSystem = (filename) => isHiddenOrSystemPath(filename, source);
+  const allSourceFilenamesThatAreArticles = allSourceFilenames.filter(
+    (filename) => filename.match(articleExtensions) && !isHiddenOrSystem(filename) && !isInHiddenFolder(filename)
+  );
+  const allSourceFilenamesThatAreDirectories = (await filterAsync(
+    allSourceFilenames,
+    (filename) => isDirectory(filename)
+  )).filter((filename) => !isHiddenOrSystem(filename) && !isFolderHidden(filename, source));
+
+  // Build set of existing HTML files in source directory (these should not be overwritten)
+  const htmlExtensions = /\.html$/;
+  const existingHtmlFiles = new Set(
+    allSourceFilenames
+      .filter(f => f.match(htmlExtensions) && !isHiddenOrSystem(f))
+      .map(f => f.replace(source, '')) // Store relative paths for easy lookup
+  );
+  
+  progress.logTimed(`Classified: ${allSourceFilenamesThatAreArticles.length} articles, ${allSourceFilenamesThatAreDirectories.length} dirs, ${existingHtmlFiles.size} HTML [${progress.stopTimer('Filter')}]`);
+  profiler.endPhase('Filter & classify');
+
+  // Drop persisted dependency registrations for documents that no longer
+  // exist (or are excluded), so stale entries don't accumulate across runs
+  dependencyTracker.prune(new Set(allSourceFilenamesThatAreArticles));
+
+  // Phase: Document template reconciliation
+  // Must run BEFORE article processing so that any template-driven changes
+  // to source .md files are picked up during rendering.
+  profiler.startPhase('Template reconciliation');
+  progress.startTimer('Templates');
+  const templateReconciliation = await reconcileAll(
+    allSourceFilenamesThatAreArticles,
+    allSourceFilenamesUnfiltered,   // templates live in _templates which is filtered out of articles
+    source
+  );
+  if (templateReconciliation.updated > 0 || templateReconciliation.conflicts > 0 || templateReconciliation.initialized > 0) {
+    progress.logTimed(
+      `📄 Document templates: ${templateReconciliation.initialized} initialized, ` +
+      `${templateReconciliation.updated} auto-merged, ` +
+      `${templateReconciliation.conflicts} conflicts, ` +
+      `${templateReconciliation.unchanged} unchanged, ` +
+      `${templateReconciliation.errors} errors`
+    );
+    if (templateReconciliation.conflicts > 0) {
+      console.warn(`\n⚠️  Template conflicts require manual resolution:`);
+      for (const msg of templateReconciliation.messages) {
+        if (msg.includes('Conflict')) console.warn(`   ${msg}`);
+      }
+      console.warn('');
+    }
+    if (templateReconciliation.errors > 0) {
+      for (const msg of templateReconciliation.messages) {
+        if (msg.includes('Error') || msg.includes('not found')) console.warn(`   ⚠️  ${msg}`);
+      }
+    }
+  }
+  progress.logTimed(`Document templates processed [${progress.stopTimer('Templates')}]`);
+  profiler.endPhase('Template reconciliation');
+
+  // Phase: Build navigation and metadata
+  profiler.startPhase('Build navigation');
+  progress.startTimer('Navigation');
+  
+  // Check if we can use cached navigation
+  let validPaths, templates, menu, menuData, customMenus, footer, buildId;
+  let navCacheUsed = false;
+  
+  if (!_clean) {
+    const fileListHash = hashFileList(allSourceFilenames);
+    const fileStatsHash = await hashFileStats(allSourceFilenames);
+    const navCache = await loadNavCache(source);
+    
+    if (isNavCacheValid(navCache, fileListHash, fileStatsHash)) {
+      // Use cached navigation data
+      navCacheUsed = true;
+      validPaths = restoreMap(navCache.validPaths);
+      menuData = navCache.menuData;
+      menu = navCache.menuHtml;
+      customMenus = restoreMap(navCache.customMenus);
+      
+      // Templates and footer still need to be loaded (they depend on meta directory)
+      templates = await getTemplates(meta);
+      buildId = getAndIncrementBuildId(resolve(_source));
+      footer = await getFooter(source, _source, buildId);
+      
+      progress.logTimed(`Navigation loaded from cache: ${validPaths.size} paths, ${customMenus.size} custom menus [${progress.stopTimer('Navigation')}]`);
+    } else {
+      // Cache miss - build navigation from scratch
+      validPaths = buildValidPaths(allSourceFilenamesThatAreArticles, source, allSourceFilenamesThatAreDirectories);
+      templates = await getTemplates(meta);
+      
+      const menuResult = await getMenu(allSourceFilenames, source, validPaths);
+      menu = menuResult.html;
+      menuData = menuResult.menuData;
+      
+      customMenus = findAllCustomMenus(allSourceFilenames, source);
+      buildId = getAndIncrementBuildId(resolve(_source));
+      footer = await getFooter(source, _source, buildId);
+      
+      // Save to cache for next run
+      const cacheEntry = createNavCacheEntry(
+        fileListHash,
+        fileStatsHash,
+        menuData,
+        menu,
+        Array.from(validPaths.entries()),
+        Array.from(customMenus.entries())
+      );
+      await saveNavCache(source, cacheEntry);
+      
+      progress.logTimed(`Navigation built: ${validPaths.size} paths, ${customMenus.size} custom menus [${progress.stopTimer('Navigation')}]`);
+    }
+  } else {
+    // Clean build - ignore cache
+    validPaths = buildValidPaths(allSourceFilenamesThatAreArticles, source, allSourceFilenamesThatAreDirectories);
+    templates = await getTemplates(meta);
+    
+    const menuResult = await getMenu(allSourceFilenames, source, validPaths);
+    menu = menuResult.html;
+    menuData = menuResult.menuData;
+    
+    customMenus = findAllCustomMenus(allSourceFilenames, source);
+    buildId = getAndIncrementBuildId(resolve(_source));
+    footer = await getFooter(source, _source, buildId);
+    
+    progress.logTimed(`Navigation built (clean): ${validPaths.size} paths, ${customMenus.size} custom menus [${progress.stopTimer('Navigation')}]`);
+  }
+  
+  profiler.endPhase('Build navigation');
+
+  // Build the _ursa_metadata embedded in every generated JSON file (ursa + doc repo versions)
+  const ursaMetadata = await getUrsaMetadata(_source);
+
+  // Phase: Load cache
+  profiler.startPhase('Load cache');
+  progress.startTimer('Cache');
+  
+  // Load content hash cache from .ursa folder in source directory
+  let hashCache = new Map();
+  if (!_clean) {
+    hashCache = await loadHashCache(source);
+    progress.logTimed(`Loaded ${hashCache.size} cached hashes [${progress.stopTimer('Cache')}]`);
+  } else {
+    progress.logTimed(`Clean build: ignoring cached hashes`);
+    progress.stopTimer('Cache');
+  }
+  
+  // Load content timestamps from .ursa.json (survives --clean)
+  // These track when content actually changed, not filesystem mtime
+  const contentTimestamps = loadContentTimestamps(source);
+  const buildTimestamp = Date.now();
+  progress.logTimed(`Loaded ${contentTimestamps.size} content timestamps`);
+  profiler.endPhase('Load cache');
+
+  // Phase: Copy meta/public files
+  profiler.startPhase('Copy meta files');
+  progress.startTimer('Meta');
+  
+  // create public folder
+  const pub = join(output, "public");
+  await mkdir(pub, { recursive: true });
+  
+  // Copy meta assets with new template folder structure
+  const { copiedFiles, orphanedFiles } = await copyMetaAssets(meta, pub);
+  
+  // Warn about orphaned files in meta that aren't part of any template
+  if (orphanedFiles.length > 0) {
+    console.warn(`\n⚠️  Warning: Found ${orphanedFiles.length} orphaned file(s) in meta directory:`);
+    console.warn(`   These files are not in meta/templates/ or meta/shared/ and won't be included:`);
+    for (const file of orphanedFiles.slice(0, 10)) {
+      console.warn(`   - ${file}`);
+    }
+    if (orphanedFiles.length > 10) {
+      console.warn(`   ... and ${orphanedFiles.length - 10} more`);
+    }
+    console.warn(`   Move them to meta/templates/{templateName}/ or meta/shared/ to include them.\n`);
+  }
+
+  // Bundle meta template assets (CSS + JS) into single files per template
+  // This must happen after copying meta to public but before cache-busting
+  templates = await bundleMetaTemplateAssets(templates, meta, pub, { minify: true, sourcemap: false });
+  progress.logTimed(`Meta template assets bundled`);
+
+  // Build React runtime for MDX hydration (React 19 has no UMD, so we bundle locally)
+  await buildReactRuntime(pub);
+
+  // Process all CSS files in the entire output directory tree for cache-busting
+  const allOutputFiles = await recurse(output, [() => false]);
+  for (const cssFile of allOutputFiles.filter(f => f.endsWith('.css'))) {
+    const cssContent = await readFile(cssFile, 'utf8');
+    const processedCss = addTimestampToCssUrls(cssContent, cacheBustTimestamp);
+    await outputFile(cssFile, processedCss);
+  }
+
+  // Process JS files in output for cache-busting fetch URLs
+  for (const jsFile of allOutputFiles.filter(f => f.endsWith('.js'))) {
+    let jsContent = await readFile(jsFile, 'utf8');
+    jsContent = jsContent.replace(
+      /fetch\(['"]([^'"\)]+\.(json))['"](?!\s*\+)/g,
+      `fetch('$1?v=${cacheBustTimestamp}'`
+    );
+    await outputFile(jsFile, jsContent);
+  }
+  
+  progress.logTimed(`Meta files copied and processed [${progress.stopTimer('Meta')}]`);
+  profiler.endPhase('Copy meta files');
+
+  // Track errors for error report
+  const errors = [];
+
+  // Search index: built incrementally during article processing (lighter memory footprint)
+  const searchIndex = [];
+  // Full-text index: collect documents for word-to-document mapping
+  const fullTextDocs = [];
+  // Recent activity: collect {title, url, mtime} for all articles, keep top 10 by mtime
+  const recentActivity = [];
+  // Track paths of documents that were regenerated (for incremental index updates)
+  const changedPaths = new Set();
+  // Directory index cache: only stores minimal data needed for directory indices
+  // Uses WeakRef-style approach - store only what's needed, clear as we go
+  const dirIndexCache = new Map();
+  
+  // Track CSS files that have been copied to avoid duplicates
+  const copiedCssFiles = new Set();
+
+  // Identify all image files from the filtered source list
+  const imageExtensions = IMAGE_EXTENSIONS;
+  let allSourceFilenamesThatAreImages = allSourceFilenames.filter(
+    (filename) => filename.match(imageExtensions) && !isHiddenOrSystem(filename)
+  );
+  
+  // When using a whitelist, also include images referenced by whitelisted documents
+  // This ensures that images used in whitelisted articles are processed even if not explicitly whitelisted
+  if (_whitelist) {
+    progress.logTimed('Scanning whitelisted articles for image references...');
+    const referencedImages = new Set();
+    
+    for (const articlePath of allSourceFilenamesThatAreArticles) {
+      try {
+        const content = await readFile(articlePath, 'utf8');
+        const imageRefs = extractImageReferences(content, articlePath, source);
+        imageRefs.forEach(img => referencedImages.add(img));
+      } catch (e) {
+        // Ignore read errors - file might not exist or be unreadable
+      }
+    }
+    
+    // Get all images from the unfiltered source list that are referenced
+    const allImagesUnfiltered = allSourceFilenamesUnfiltered.filter(
+      (filename) => filename.match(imageExtensions) && !isHiddenOrSystem(filename)
+    );
+    
+    // Add referenced images that aren't already in the list
+    const additionalImages = allImagesUnfiltered.filter(
+      img => referencedImages.has(img) && !allSourceFilenamesThatAreImages.includes(img)
+    );
+    
+    if (additionalImages.length > 0) {
+      progress.logTimed(`Found ${additionalImages.length} additional images referenced by whitelisted documents`);
+      allSourceFilenamesThatAreImages = [...allSourceFilenamesThatAreImages, ...additionalImages];
+    }
+  }
+  
+  // Phase: Process images
+  profiler.startPhase('Process images');
+  progress.startTimer('Images');
+  
+  // Handle images based on deferred mode
+  let imageMap = new Map();
+  let deferredImageProcessingPromise = null;
+  
+  if (_deferImages) {
+    // Fast mode: just copy images without processing, defer preview generation
+    progress.logTimed(`Copying ${allSourceFilenamesThatAreImages.length} images (preview generation deferred)...`);
+    await copyAllImagesFast(
+      allSourceFilenamesThatAreImages,
+      source,
+      output,
+      (current, total, path) => {
+        progress.status('Images (copy)', `${current}/${total} ${path}`);
+      }
+    );
+    progress.done('Images (copy)', `${allSourceFilenamesThatAreImages.length} copied (previews deferred)`);
+    profiler.endPhase('Process images');
+    
+    // Create promise for background image processing (will be returned to caller)
+    deferredImageProcessingPromise = (async () => {
+      progress.logTimed(`\n🖼️  Starting deferred image preview generation...`);
+      const startTime = Date.now();
+      const processedImageMap = await processAllImages(
+        allSourceFilenamesThatAreImages,
+        source,
+        output,
+        (current, total, path) => {
+          progress.status('Images (previews)', `${current}/${total} ${path}`);
+        }
+      );
+      const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+      progress.done('Images (previews)', `${allSourceFilenamesThatAreImages.length} done (${processedImageMap.size} with previews) in ${elapsed}s`);
+      
+      // Update the watch cache with the processed image map
+      watchModeCache.imageMap = processedImageMap;
+      
+      return processedImageMap;
+    })();
+  } else {
+    // Normal mode: process all images FIRST to build the preview image map
+    // This is done before articles so we can transform img tags in the HTML
+    progress.logTimed(`Processing ${allSourceFilenamesThatAreImages.length} images for preview generation...`);
+    imageMap = await processAllImages(
+      allSourceFilenamesThatAreImages,
+      source,
+      output,
+      (current, total, path) => {
+        progress.status('Images', `${current}/${total} ${path}`);
+      }
+    );
+    progress.done('Images', `${allSourceFilenamesThatAreImages.length} done (${imageMap.size} with previews) [${progress.stopTimer('Images')}]`);
+    profiler.endPhase('Process images');
+  }
+
+  // Phase: Process articles
+  profiler.startPhase('Process articles');
+  progress.startTimer('Articles');
+  
+  // Track files that were regenerated (for incremental mode stats)
+  let regeneratedCount = 0;
+  let skippedCount = 0;
+  let processedCount = 0;
+  const totalArticles = allSourceFilenamesThatAreArticles.length;
+
+  progress.logTimed(`Processing ${totalArticles} articles in batches of ${BATCH_SIZE}...`);
+
+  // Single pass: process all articles with batched concurrency to limit memory usage
+  await processBatched(allSourceFilenamesThatAreArticles, async (file) => {
+    try {
+      processedCount++;
+      const shortFile = file.replace(source, '');
+      progress.status('Articles', `${processedCount}/${totalArticles} ${shortFile}`);
+      
+      // Use streaming for large files to reduce memory pressure
+      const { size: fileSize, useStreaming } = await checkFileSize(file);
+      let rawBody;
+      if (useStreaming) {
+        rawBody = await readFileStreaming(file, 'utf8');
+        progress.log(`📄 Streaming large file ${shortFile} (${formatFileSize(fileSize)})`);
+      } else {
+        rawBody = await readFile(file, "utf8");
+      }
+      
+      const type = parse(file).ext;
+      const ext = extname(file);
+      const base = basename(file, ext);
+      const dir = addTrailingSlash(dirname(file)).replace(source, "");
+      
+      // Calculate output paths for this file
+      const outputFilename = file
+        .replace(source, output)
+        .replace(parse(file).ext, ".html");
+      const url = '/' + outputFilename.replace(output, '');
+      
+      // Generate URL path relative to output (for search index)
+      const relativePath = file.replace(source, '').replace(/\.(md|mdx|txt|yml)$/, '.html');
+      const searchUrl = relativePath.startsWith('/') ? relativePath : '/' + relativePath;
+      
+      // Generate title from filename (in title case)
+      // For index/home files, use parent folder name instead
+      const titleBase = (base === 'index' || base === 'home') ? basename(dirname(file)) : base;
+      const title = toTitleCase(titleBase || base);
+      
+      // Always add to search index (lightweight: title + path only, content added lazily)
+      searchIndex.push({
+        title: title,
+        path: relativePath,
+        url: searchUrl,
+        content: '' // Content excerpts built lazily to save memory
+      });
+
+      // Add document to full-text index (uses raw markdown content)
+      fullTextDocs.push({
+        path: relativePath,
+        title: title,
+        content: rawBody
+      });
+
+      // Collect timestamp for recent activity tracking
+      // Use stored content timestamp if available, otherwise fall back to file mtime
+      // Content timestamps track when content actually changed, not filesystem mtime
+      const storedTimestamp = contentTimestamps.get(relativePath);
+      let activityTimestamp = storedTimestamp;
+      if (!activityTimestamp) {
+        // No stored timestamp - use file mtime as initial value
+        try {
+          const fileStat = await stat(file);
+          activityTimestamp = fileStat.mtimeMs;
+        } catch (e) {
+          activityTimestamp = 0;
+        }
+      }
+      recentActivity.push({
+        title: title,
+        url: searchUrl,
+        mtime: activityTimestamp
+      });
+      
+      // Check if a corresponding .html file already exists in source directory
+      const outputHtmlRelative = relativePath.startsWith('/') ? relativePath.slice(1) : relativePath;
+      if (existingHtmlFiles.has(outputHtmlRelative)) {
+        progress.log(`⚠️  Warning: Skipping ${shortFile} - would overwrite existing ${outputHtmlRelative} in source`);
+        skippedCount++;
+        return;
+      }
+
+      // Skip metadata-only index files - they exist only to provide folder metadata
+      // The auto-index system will generate the actual index.html for these folders
+      if (base === 'index' && (type === '.md' || type === '.mdx') && isMetadataOnly(rawBody)) {
+        progress.log(`ℹ️  Skipping metadata-only ${shortFile} - auto-index will generate listing`);
+        skippedCount++;
+        return;
+      }
+
+      // Metadata is needed both for the cache decision below and for rendering.
+      const fileMeta = extractMetadata(rawBody);
+
+      // A document with `generate-auto-index: true` renders a listing of the
+      // source tree, so its output depends on which folders and files exist —
+      // not just on its own text. A content hash cannot see that: adding,
+      // renaming, or deleting a folder anywhere in the tree leaves the hash
+      // untouched and the listing stale forever. Only a handful of documents
+      // opt in, so always rebuild them.
+      const hasAutoIndex = getAutoIndexConfig(fileMeta).enabled;
+
+      // Check if file needs regeneration.
+      // An unchanged hash is not enough: the hash cache lives in the source tree
+      // and is shared across output dirs, so also require that every output this
+      // document emits is actually present before skipping it.
+      const needsRegen =
+        _clean ||
+        hasAutoIndex ||
+        needsRegeneration(file, rawBody, hashCache) ||
+        !outputsExist([
+          outputFilename,
+          outputFilename.replace(".html", ".json"),
+          outputFilename.replace(".html", ".xml"),
+        ]);
+
+      if (!needsRegen) {
+        skippedCount++;
+        // For directory indices, store minimal data (not full bodyHtml)
+        // But include metadata for directory JSON files
+        dirIndexCache.set(file, {
+          name: base,
+          url,
+          metadata: fileMeta,
+        });
+        return; // Skip regenerating this file
+      }
+      
+      regeneratedCount++;
+      // Track this path for incremental search index updates
+      changedPaths.add(relativePath);
+
+      const rawMeta = extractRawMetadata(rawBody);
+      
+      // Lazy metadata transform - only compute if template actually uses it
+      // This defers the potentially expensive custom transform function load
+      let transformedMetadata = null;
+      const getTransformedMeta = async () => {
+        if (transformedMetadata === null) {
+          transformedMetadata = await getTransformedMetadata(dirname(file), fileMeta);
+        }
+        return transformedMetadata;
+      };
+      
+      // Calculate the document's URL path (e.g., "/character/index.html")
+      const docUrlPath = '/' + dir + base + '.html';
+
+      // Use async rendering with worker threads for parallel markdown parsing
+      // Wikitext (.txt) files will fall back to main thread
+      // For MDX files, enable hydration if frontmatter has `hydrate: true`
+      const shouldHydrate = type === '.mdx' && fileMeta?.hydrate === true;
+      
+      let renderResult = await renderFileAsync({
+        fileContents: rawBody,
+        type,
+        dirname: dir,
+        basename: base,
+        filePath: file,
+        sourceRoot: source,
+        useWorker: true,
+        hydrate: shouldHydrate,
+      });
+
+      // Handle the result - can be string or { html, hydrationScript }
+      let body;
+      let hydrationScript = '';
+      if (typeof renderResult === 'object' && renderResult.html) {
+        body = renderResult.html;
+        hydrationScript = renderResult.hydrationScript || '';
+      } else {
+        body = renderResult;
+      }
+
+      // Inject default H1 if body doesn't start with one
+      if (!body || !body.trimStart().startsWith('<h1')) {
+        const h1Title = fileMeta?.title || title;
+        body = `<h1>${h1Title}</h1>\n` + (body || '');
+      }
+
+      // Inject breadcrumbs before the H1
+      const breadcrumbs = generateBreadcrumbs(dir, base, fileMeta);
+      if (breadcrumbs) {
+        body = breadcrumbs + body;
+      }
+
+      // Inject frontmatter table after first H1 (for markdown files with metadata)
+      if ((type === '.md' || type === '.mdx') && fileMeta) {
+        body = injectFrontmatterTable(body, fileMeta);
+      }
+
+      // Handle auto-index generation for index files with generate-auto-index: true
+      if (base === 'index' && fileMeta) {
+        const autoIndexConfig = getAutoIndexConfig(fileMeta);
+        if (autoIndexConfig.enabled) {
+          // Generate auto-index HTML for this directory from source
+          // Using source avoids race conditions with concurrent file generation
+          const sourceDir = dirname(file);
+          const autoIndexHtml = await generateAutoIndexHtmlFromSource(sourceDir, autoIndexConfig.depth);
+          
+          if (autoIndexHtml) {
+            if (autoIndexConfig.position === 'bottom') {
+              body = body + '\n' + autoIndexHtml;
+            } else {
+              body = autoIndexHtml + '\n' + body;
+            }
+          }
+        }
+      }
+
+      // Find all style.css files up the tree and bundle them into a single CSS file per folder path
+      // (Generate mode: one CSS bundle per unique folder, minimizing requests per page load)
+      let styleLink = "";
+      try {
+        const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
+        const folderRelative = (dir === "/" || dir === "") ? "" : dir;
+
+        // Check bundle cache first (dirs with same CSS ancestry share the same bundle)
+        let cachedBundleUrl = docBundleCache.get(`css:${dirKey}`);
+        if (cachedBundleUrl !== undefined) {
+          if (cachedBundleUrl) {
+            styleLink = `<link rel="stylesheet" href="${cachedBundleUrl}" />`;
+          }
+        } else {
+          let cssPaths = cssPathCache.get(dirKey);
+          if (cssPaths === undefined) {
+            cssPaths = await findAllStyleCss(dirKey, _source);
+            cssPathCache.set(dirKey, cssPaths);
+          }
+          if (cssPaths.length > 0) {
+            // Copy all source CSS files to output (still needed for serve mode fallback)
+            for (const cssPath of cssPaths) {
+              if (!copiedCssFiles.has(cssPath)) {
+                const cssOutputPath = cssPath.replace(source, output);
+                const cssContent = await readFile(cssPath, 'utf8');
+                await outputFile(cssOutputPath, cssContent);
+                copiedCssFiles.add(cssPath);
+              }
+            }
+            // Bundle into a single file
+            const bundleUrl = await bundleDocumentCss(cssPaths, output, source, folderRelative, { minify: true });
+            docBundleCache.set(`css:${dirKey}`, bundleUrl);
+            styleLink = `<link rel="stylesheet" href="${bundleUrl}" />`;
+          } else {
+            docBundleCache.set(`css:${dirKey}`, null);
+          }
+        }
+      } catch (e) {
+        // ignore
+        console.error(e);
+      }
+
+      // Find all script.js files from docroot to current dir and bundle them
+      // (Generate mode: one JS bundle per unique folder, external not inlined)
+      let customScript = "";
+      try {
+        const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
+        const folderRelative = (dir === "/" || dir === "") ? "" : dir;
+
+        let cachedBundleUrl = docBundleCache.get(`js:${dirKey}`);
+        if (cachedBundleUrl !== undefined) {
+          if (cachedBundleUrl) {
+            customScript = `<script src="${cachedBundleUrl}"></script>`;
+          }
+        } else {
+          let scriptPaths = scriptPathCache.get(dirKey);
+          if (scriptPaths === undefined) {
+            scriptPaths = await findAllScriptJs(dirKey, _source);
+            scriptPathCache.set(dirKey, scriptPaths);
+          }
+          if (scriptPaths.length > 0) {
+            const bundleUrl = await bundleDocumentJs(scriptPaths, output, source, folderRelative, { minify: true });
+            docBundleCache.set(`js:${dirKey}`, bundleUrl);
+            customScript = `<script src="${bundleUrl}"></script>`;
+          } else {
+            docBundleCache.set(`js:${dirKey}`, null);
+          }
+        }
+      } catch (e) {
+        // ignore
+        console.error(e);
+      }
+
+      const requestedTemplateName = fileMeta && fileMeta.template;
+      const templateName = requestedTemplateName || DEFAULT_TEMPLATE_NAME;
+      const template = templates[templateName];
+
+      if (!template) {
+        throw new Error(`Template not found. Requested: "${templateName}". Available templates: ${Object.keys(templates).join(', ') || 'none'}`);
+      }
+
+      // Register this document's dependencies for invalidation tracking
+      {
+        const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
+        const cssDeps = cssPathCache.get(dirKey) || [];
+        const jsDeps = scriptPathCache.get(dirKey) || [];
+        dependencyTracker.registerDocument(file, {
+          templateName,
+          cssPaths: cssDeps,
+          scriptPaths: jsDeps,
+        });
+      }
+
+      // Check if this file has a custom menu
+      const customMenuInfo = getCustomMenuForFile(file, source, customMenus);
+
+      // Lazy evaluation of transformed metadata - only compute if template uses it
+      // This defers expensive custom transform function loading until actually needed
+      const templateUsesTransformedMeta = template.includes('${transformedMetadata}');
+      const lazyTransformedMeta = templateUsesTransformedMeta 
+        ? await getTransformedMeta() 
+        : '';
+
+      // Build final HTML with all replacements in a single regex pass
+      // This avoids creating 8 intermediate strings
+      // Append hydration script to customScript if present (for MDX with hydrate: true)
+      const finalCustomScript = hydrationScript 
+        ? customScript + '\n' + hydrationScript 
+        : customScript;
+      
+      const replacements = {
+        "${title}": fileMeta?.title || title,
+        "${menu}": menu,
+        "${meta}": JSON.stringify(fileMeta),
+        "${transformedMetadata}": lazyTransformedMeta,
+        "${body}": body,
+        "${styleLink}": styleLink,
+        "${customScript}": finalCustomScript,
+        "${searchIndex}": "[]", // Placeholder - search index written separately as JSON file
+        "${footer}": footer
+      };
+      // Single-pass replacement using regex alternation
+      const pattern = /\$\{(title|menu|meta|transformedMetadata|body|styleLink|customScript|searchIndex|footer)\}/g;
+      let finalHtml = template.replace(pattern, (match) => replacements[match] ?? match);
+
+      // Add menu data attributes to body
+      if (customMenuInfo) {
+        const menuPosition = customMenuInfo.menuPosition || 'top';
+        finalHtml = finalHtml.replace(
+          /<body([^>]*)>/,
+          `<body$1 data-custom-menu="${customMenuInfo.menuJsonPath}" data-menu-position="${menuPosition}">`
+        );
+      } else {
+        // No custom menu — default to top menu
+        finalHtml = finalHtml.replace(
+          /<body([^>]*)>/,
+          `<body$1 data-menu-position="top">`
+        );
+      }
+
+      // Resolve relative URLs in raw HTML elements (img src, etc.)
+      finalHtml = resolveRelativeUrls(finalHtml, docUrlPath);
+      
+      // Resolve links and mark broken internal links as inactive
+      finalHtml = markInactiveLinks(finalHtml, validPaths, docUrlPath, false);
+
+      // Transform image tags to use preview images with data-fullsrc for originals
+      // Skip in deferred mode - images will use original paths until preview generation completes
+      if (!_deferImages) {
+        finalHtml = transformImageTags(finalHtml, imageMap, docUrlPath);
+      }
+
+      // Add cache-busting timestamps to static file references
+      finalHtml = addTimestampToHtmlStaticRefs(finalHtml, cacheBustTimestamp);
+
+      await outputFile(outputFilename, finalHtml);
+      
+      // Clear finalHtml reference to allow GC
+      finalHtml = null;
+
+      // JSON output
+      const jsonOutputFilename = outputFilename.replace(".html", ".json");
+      
+      // Extract sections for markdown files
+      const sections = (type === '.md' || type === '.mdx') ? extractSections(rawBody) : [];
+      
+      // Use lazy metadata for JSON output - may have been computed above for HTML
+      const jsonTransformedMeta = await getTransformedMeta();
+      
+      const jsonObject = {
+        name: base,
+        url,
+        contents: rawBody,
+        bodyHtml: body,
+        metadata: fileMeta,
+        sections,
+        transformedMetadata: jsonTransformedMeta,
+        _ursa_metadata: ursaMetadata,
+      };
+      
+      // Store minimal data for directory indices, including metadata
+      dirIndexCache.set(file, {
+        name: base,
+        url,
+        metadata: fileMeta,
+      });
+      
+      const json = JSON.stringify(jsonObject);
+      await outputFile(jsonOutputFilename, json);
+
+      // XML output
+      const xmlOutputFilename = outputFilename.replace(".html", ".xml");
+      const xml = `<article>${o2x(jsonObject)}</article>`;
+      await outputFile(xmlOutputFilename, xml);
+      
+      // Update the content hash for this file
+      updateHash(file, rawBody, hashCache);
+      
+      // Update content timestamp since this file was regenerated (content changed)
+      contentTimestamps.set(relativePath, buildTimestamp);
+      // Also update the recentActivity entry we pushed earlier with the new timestamp
+      const activityEntry = recentActivity.find(e => e.url === searchUrl);
+      if (activityEntry) {
+        activityEntry.mtime = buildTimestamp;
+      }
+    } catch (e) {
+      progress.log(`Error processing ${file}: ${e.message}`);
+      errors.push({ file, phase: 'article-generation', error: e });
+    }
+  });
+
+  // Complete the articles status line
+  progress.done('Articles', `${totalArticles} done (${regeneratedCount} regenerated, ${skippedCount} unchanged) [${progress.stopTimer('Articles')}]`);
+  profiler.endPhase('Process articles');
+
+  // Phase: Write search index
+  // Can be deferred in serve mode for faster startup
+  let deferredSearchIndexPromise = null;
+  
+  const buildSearchIndex = async () => {
+    const indexStartTime = Date.now();
+    progress.logTimed('Building search index...');
+    
+    // Write search index as a separate JSON file (not embedded in each page)
+    const searchIndexPath = join(output, 'public', 'search-index.json');
+    progress.log(`Writing search index with ${searchIndex.length} entries`);
+    await outputFile(searchIndexPath, JSON.stringify(searchIndex));
+
+    // Build full-text index - use incremental mode when possible
+    let fullTextIndex;
+    if (!_clean && changedPaths.size > 0 && changedPaths.size < fullTextDocs.length) {
+      // Incremental update: only re-index changed documents
+      progress.log(`Incremental full-text index: ${changedPaths.size} changed, ${fullTextDocs.length - changedPaths.size} cached`);
+      fullTextIndex = buildIncrementalIndex(fullTextDocs, changedPaths, source);
+    } else {
+      // Full rebuild: clean build or all documents changed
+      progress.log(`Building full-text index from ${fullTextDocs.length} documents...`);
+      fullTextIndex = buildFullTextIndex(fullTextDocs);
+      // Save to cache for future incremental updates
+      saveIndexCache(source, fullTextIndex);
+    }
+    
+    const fullTextIndexPath = join(output, 'public', 'fulltext-index.json');
+    const fullTextIndexJson = JSON.stringify(fullTextIndex);
+    const wordCount = Object.keys(fullTextIndex).length;
+    progress.log(`Writing full-text index (${wordCount} unique words, ${(fullTextIndexJson.length / 1024).toFixed(1)} KB)`);
+    await outputFile(fullTextIndexPath, fullTextIndexJson);
+    
+    const elapsed = ((Date.now() - indexStartTime) / 1000).toFixed(1);
+    return { entries: searchIndex.length, words: wordCount, elapsed };
+  };
+  
+  if (_deferSearchIndex) {
+    // Deferred mode: start building in background, return promise
+    profiler.startPhase('Write search index (deferred)');
+    progress.startTimer('Search index');
+    progress.log('Search index building deferred for faster startup...');
+    deferredSearchIndexPromise = buildSearchIndex().then(result => {
+      progress.done('Search index (background)', `${result.entries} entries, ${result.words} words in ${result.elapsed}s`);
+      return result;
+    });
+    progress.done('Search index', 'deferred [0ms]');
+    profiler.endPhase('Write search index (deferred)');
+  } else {
+    // Normal mode: build search index now
+    profiler.startPhase('Write search index');
+    progress.startTimer('Search index');
+    const result = await buildSearchIndex();
+    progress.done('Search index', `${result.entries} entries, ${result.words} words [${progress.stopTimer('Search index')}]`);
+    profiler.endPhase('Write search index');
+  }
+
+  // Phase: Write recent activity data
+  profiler.startPhase('Write recent activity');
+  progress.startTimer('Recent activity');
+  // Sort by mtime descending, keep top 10
+  recentActivity.sort((a, b) => b.mtime - a.mtime);
+  const top10 = recentActivity.slice(0, 10);
+  const recentActivityPath = join(output, 'public', 'recent-activity.json');
+  await outputFile(recentActivityPath, JSON.stringify(top10));
+  progress.done('Recent activity', `${top10.length} entries [${progress.stopTimer('Recent activity')}]`);
+  profiler.endPhase('Write recent activity');
+
+  // Phase: Write menu data
+  profiler.startPhase('Write menu data');
+  progress.startTimer('Menu data');
+  // Write menu data as a separate JSON file (not embedded in each page)
+  // This dramatically reduces HTML file sizes for large sites
+  const menuDataPath = join(output, 'public', 'menu-data.json');
+  const menuDataJson = JSON.stringify(menuData);
+  progress.log(`Writing menu data (${(menuDataJson.length / 1024).toFixed(1)} KB)`);
+  await outputFile(menuDataPath, menuDataJson);
+
+  // Write custom menu JSON files
+  for (const [menuDir, menuInfo] of customMenus) {
+    const customMenuPath = join(output, menuInfo.menuJsonPath);
+    // Include menuPosition in the JSON so client knows how to render
+    const customMenuJson = JSON.stringify({
+      menuData: menuInfo.menuData,
+      menuPosition: menuInfo.menuPosition || 'top',
+    });
+    progress.log(`Writing custom menu: ${menuInfo.menuJsonPath}`);
+    await outputFile(customMenuPath, customMenuJson);
+  }
+  progress.done('Menu data', `${customMenus.size + 1} files [${progress.stopTimer('Menu data')}]`);
+  profiler.endPhase('Write menu data');
+
+  // Phase: Process directory indices
+  profiler.startPhase('Process directories');
+  progress.startTimer('Directories');
+
+  // Output paths that a source document already owns. The generated directory
+  // listing below writes to <dir>.html, which collides with two things: an
+  // article named after its own folder (settings/dying-light.md renders to
+  // settings/dying-light.html) and a hand-written .html copied from the source
+  // tree. Those documents win — the listing must never overwrite them.
+  const documentOwnedOutputs = new Set(
+    allSourceFilenamesThatAreArticles.map((filename) =>
+      filename.replace(source, output).replace(/\.(md|mdx|txt|yml)$/, ".html")
+    )
+  );
+  for (const relativeHtmlPath of existingHtmlFiles) {
+    documentOwnedOutputs.add(join(output, relativeHtmlPath));
+  }
+
+  // Process directory indices with batched concurrency
+  const totalDirs = allSourceFilenamesThatAreDirectories.length;
+  let processedDirs = 0;
+  progress.log(`Processing ${totalDirs} directories...`);
+  await processBatched(allSourceFilenamesThatAreDirectories, async (dirPath) => {
+    try {
+      processedDirs++;
+      const shortDir = dirPath.replace(source, '');
+      progress.status('Directories', `${processedDirs}/${totalDirs} ${shortDir}`);
+
+      const pathsInThisDirectory = allSourceFilenames.filter((filename) =>
+        filename.match(new RegExp(`${dirPath}.+`))
+      );
+
+      // Use minimal directory index cache instead of full jsonCache
+      const jsonObjects = pathsInThisDirectory
+        .map((path) => {
+          const object = dirIndexCache.get(path);
+          return typeof object === "object" ? object : null;
+        })
+        .filter((a) => a);
+
+      const json = JSON.stringify(jsonObjects);
+
+      const outputFilename = dirPath.replace(source, output) + ".json";
+      await outputFile(outputFilename, json);
+
+      // html
+      // Rewritten every build: the listing reflects the directory's contents,
+      // so skipping it whenever the file already exists (the old behaviour)
+      // froze it at whatever the tree looked like the first time it was
+      // written, and new or removed documents never showed up again.
+      const htmlOutputFilename = dirPath.replace(source, output) + ".html";
+      if (!documentOwnedOutputs.has(htmlOutputFilename)) {
+        const template = templates["default-template"];
+        const indexHtml = `<ul>${pathsInThisDirectory
+          .map((path) => {
+            const partialPath = path
+              .replace(source, "")
+              .replace(parse(path).ext, ".html");
+            const name = basename(path, parse(path).ext);
+            return `<li><a href="${partialPath}">${name}</a></li>`;
+          })
+          .join("")}</ul>`;
+        let finalHtml = template;
+        const replacements = {
+          "${menu}": menu,
+          "${body}": indexHtml,
+          "${searchIndex}": "[]", // Search index now in separate file
+          "${title}": "Index",
+          "${meta}": "{}",
+          "${transformedMetadata}": "",
+          "${styleLink}": "",
+          "${footer}": footer
+        };
+        for (const [key, value] of Object.entries(replacements)) {
+          finalHtml = finalHtml.replace(key, value);
+        }
+        // Add cache-busting timestamps to static file references
+        finalHtml = addTimestampToHtmlStaticRefs(finalHtml, cacheBustTimestamp);
+        await outputFile(htmlOutputFilename, finalHtml);
+      }
+    } catch (e) {
+      progress.log(`Error processing directory ${dirPath}: ${e.message}`);
+      errors.push({ file: dirPath, phase: 'directory-index', error: e });
+    }
+  });
+  
+  progress.done('Directories', `${totalDirs} done [${progress.stopTimer('Directories')}]`);
+  profiler.endPhase('Process directories');
+
+  // Clear directory index cache to free memory before processing static files
+  dirIndexCache.clear();
+
+  // Phase: Process static files
+  profiler.startPhase('Process static files');
+  progress.startTimer('Static files');
+  // Copy static HTML files (images were already processed above with preview generation)
+  // Note: Images are processed before articles to enable preview transformation in HTML
+  
+  // Also copy existing HTML files from source to output (they're treated as static)
+  const allSourceFilenamesThatAreHtml = allSourceFilenames.filter(
+    (filename) => filename.match(/\.html$/) && !isHiddenOrSystem(filename)
+  );
+  
+  // Fonts, audio, video, PDFs: copied through untouched. Images are handled
+  // separately above because they also get previews; everything else that is
+  // neither an article nor a stylesheet belongs here. Leaving this out is what
+  // let `ursa serve` and `ursa generate` disagree — serve reads these straight
+  // off disk, so the missing copy step only ever showed up in a built site.
+  const allSourceFilenamesThatAreMedia = allSourceFilenames.filter(
+    (filename) => isMedia(filename) && !isHiddenOrSystem(filename)
+  );
+
+  const allStaticFiles = [...allSourceFilenamesThatAreHtml, ...allSourceFilenamesThatAreMedia];
+  const totalStatic = allStaticFiles.length;
+  let processedStatic = 0;
+  let copiedStatic = 0;
+  progress.log(
+    `Processing ${totalStatic} static files ` +
+    `(${allSourceFilenamesThatAreHtml.length} HTML, ${allSourceFilenamesThatAreMedia.length} media)...`
+  );
+  await processBatched(allStaticFiles, async (file) => {
+    try {
+      processedStatic++;
+      const shortFile = file.replace(source, '');
+      progress.status('Static files', `${processedStatic}/${totalStatic} ${shortFile}`);
+
+      // Check if file has changed using file stat as a quick check
+      const fileStat = await stat(file);
+      const statKey = `${file}:stat`;
+      const newStatHash = `${fileStat.size}:${fileStat.mtimeMs}`;
+      const outputFilename = file.replace(source, output);
+      // As with articles: an unchanged stat only means the source is untouched,
+      // so the output must exist before this copy can be skipped.
+      if (hashCache.get(statKey) === newStatHash && outputsExist([outputFilename])) {
+        return; // Skip unchanged static file
+      }
+      hashCache.set(statKey, newStatHash);
+      copiedStatic++;
+
+      await mkdir(dirname(outputFilename), { recursive: true });
+
+      if (file.endsWith('.css')) {
+        // Process CSS for cache busting
+        const cssContent = await readFile(file, 'utf8');
+        const processedCss = addTimestampToCssUrls(cssContent, cacheBustTimestamp);
+        await outputFile(outputFilename, processedCss);
+      } else if (file.endsWith('.html')) {
+        // Process HTML files for link resolution
+        let htmlContent = await readFile(file, 'utf8');
+        // Calculate the document's URL path for relative link resolution
+        const docUrlPath = '/' + file.replace(source, '').replace(/^\//, '');
+        // Resolve relative URLs in raw HTML elements (img src, etc.)
+        htmlContent = resolveRelativeUrls(htmlContent, docUrlPath);
+        // Resolve internal links to have proper .html extensions
+        htmlContent = markInactiveLinks(htmlContent, validPaths, docUrlPath, false);
+        // Transform image tags to use preview images with data-fullsrc for originals
+        // Skip in deferred mode - images will use original paths until preview generation completes
+        if (!_deferImages) {
+          htmlContent = transformImageTags(htmlContent, imageMap, docUrlPath);
+        }
+        // Add cache-busting timestamps
+        htmlContent = addTimestampToHtmlStaticRefs(htmlContent, cacheBustTimestamp);
+        await outputFile(outputFilename, htmlContent);
+      } else {
+        await copyFile(file, outputFilename);
+      }
+    } catch (e) {
+      progress.log(`Error processing static file ${file}: ${e.message}`);
+      errors.push({ file, phase: 'static-file', error: e });
+    }
+  });
+  
+  progress.done('Static files', `${totalStatic} done (${copiedStatic} copied) [${progress.stopTimer('Static files')}]`);
+  profiler.endPhase('Process static files');
+
+  // Phase: Auto-index generation
+  profiler.startPhase('Auto-index generation');
+  progress.startTimer('Auto-index');
+  // Automatic index generation for folders without index.html
+  progress.log(`Checking for missing index files...`);
+  await generateAutoIndices(output, allSourceFilenamesThatAreDirectories, source, templates, menu, footer, allSourceFilenamesThatAreArticles, copiedCssFiles, existingHtmlFiles, cacheBustTimestamp, progress, customMenus);
+  progress.done('Auto-index', `checked ${allSourceFilenamesThatAreDirectories.length} directories [${progress.stopTimer('Auto-index')}]`);
+  profiler.endPhase('Auto-index generation');
+
+  // Phase: Finalization
+  profiler.startPhase('Finalization');
+  progress.startTimer('Finalization');
+  // Save the hash cache to .ursa folder in source directory
+  if (hashCache.size > 0) {
+    await saveHashCache(source, hashCache);
+  }
+  
+  // Save content timestamps to .ursa.json (tracks when content actually changed)
+  if (contentTimestamps.size > 0) {
+    saveContentTimestamps(source, contentTimestamps);
+    progress.log(`Saved ${contentTimestamps.size} content timestamps`);
+  }
+
+  // Persist the dependency tracker so hash-skipped documents keep their
+  // edges on the next warm start (invalidation plans stay accurate)
+  await saveDependencyTracker(source);
+
+  // Populate watch mode cache for fast single-file regeneration
+  watchModeCache.templates = templates;
+  watchModeCache.menu = menu;
+  watchModeCache.footer = footer;
+  watchModeCache.validPaths = validPaths;
+  watchModeCache.source = source;
+  watchModeCache.meta = meta;
+  watchModeCache.output = output;
+  watchModeCache.hashCache = hashCache;
+  watchModeCache.cacheBustTimestamp = cacheBustTimestamp;
+  watchModeCache.cacheBustHashes = cacheBustHashes;
+  watchModeCache.allArticlePaths = [...allSourceFilenamesThatAreArticles];
+  watchModeCache.imageMap = imageMap;
+  watchModeCache.customMenus = customMenus;
+  watchModeCache.ursaMetadata = ursaMetadata;
+  watchModeCache.lastFullBuild = Date.now();
+  watchModeCache.isInitialized = true;
+  const depStats = dependencyTracker.getStats();
+  progress.log(`Watch cache initialized (${depStats.totalDocuments} documents, ${depStats.uniqueFiles} dependencies tracked)`);
+
+  // Write error report if there were any errors
+  if (errors.length > 0) {
+    const errorReportPath = join(output, '_errors.log');
+    const failedFiles = errors.map(e => e.file);
+    
+    let report = `URSA GENERATION ERROR REPORT\n`;
+    report += `Generated: ${new Date().toISOString()}\n`;
+    report += `Total errors: ${errors.length}\n\n`;
+    report += `${'='.repeat(60)}\n`;
+    report += `FAILED FILES:\n`;
+    report += `${'='.repeat(60)}\n\n`;
+    failedFiles.forEach(f => {
+      report += `  - ${f}\n`;
+    });
+    report += `\n${'='.repeat(60)}\n`;
+    report += `ERROR DETAILS:\n`;
+    report += `${'='.repeat(60)}\n\n`;
+    
+    errors.forEach(({ file, phase, error }) => {
+      report += `${'─'.repeat(60)}\n`;
+      report += `File: ${file}\n`;
+      report += `Phase: ${phase}\n`;
+      report += `Error: ${error.message}\n`;
+      if (error.stack) {
+        report += `Stack:\n${error.stack}\n`;
+      }
+      report += `\n`;
+    });
+    
+    await outputFile(errorReportPath, report);
+    progress.log(`\n⚠️  ${errors.length} error(s) occurred during generation.`);
+    progress.log(`   Error report written to: ${errorReportPath}\n`);
+  } else {
+    progress.log(`\n✅ Generation complete with no errors.\n`);
+  }
+  
+  progress.done('Finalization', `complete [${progress.stopTimer('Finalization')}]`);
+  profiler.endPhase('Finalization');
+  
+  // Print profiler report
+  progress.log(profiler.report());
+  
+  // Terminate worker pool so threads don't keep the process alive
+  await terminateParserPool();
+
+  // Return deferred processing promises if in deferred mode
+  // Caller can await these to know when background processing is complete
+  return {
+    deferredImageProcessing: deferredImageProcessingPromise,
+    deferredSearchIndex: deferredSearchIndexPromise
+  };
+}
+
+/**
+ * Regenerate multiple documents affected by a dependency change (e.g., style.css, script.js, template).
+ * Uses the watchModeCache and dependency tracker to efficiently re-render affected documents
+ * with updated cache-bust timestamps.
+ *
+ * @param {string[]} documentPaths - Absolute paths to documents to regenerate
+ * @param {Object} options
+ * @param {string} options._source - Source directory
+ * @param {string} options._meta - Meta directory
+ * @param {string} options._output - Output directory
+ * @param {string} [options.reason] - Reason for regeneration (for logging)
+ * @param {string[]} [options.priorityPaths] - Document paths to regenerate first (e.g. client-viewed docs)
+ * @param {function} [options.onPriorityComplete] - Callback after priority paths are done (receives { regenerated, failed })
+ * @returns {Promise<{success: boolean, message: string, regenerated: number, failed: number}>}
+ */
+export async function regenerateAffectedDocuments(documentPaths, {
+  _source,
+  _meta,
+  _output,
+  reason = "dependency change",
+  priorityPaths = [],
+  onPriorityComplete = null,
+} = {}) {
+  const startTime = Date.now();
+
+  if (!watchModeCache.isInitialized) {
+    return { success: false, message: "Cache not initialized - need full build first", regenerated: 0, failed: 0 };
+  }
+
+  if (documentPaths.length === 0) {
+    return { success: true, message: "No documents to regenerate", regenerated: 0, failed: 0 };
+  }
+
+  // Generate a fresh cache-bust timestamp for this invalidation pass
+  const newTimestamp = generateCacheBustTimestamp();
+  watchModeCache.cacheBustTimestamp = newTimestamp;
+
+  let regenerated = 0;
+  let failed = 0;
+
+  // Separate priority paths from the rest
+  const prioritySet = new Set(priorityPaths.map(p => resolve(p)));
+  const priorityDocs = documentPaths.filter(p => prioritySet.has(resolve(p)));
+  const remainingDocs = documentPaths.filter(p => !prioritySet.has(resolve(p)));
+
+  if (priorityDocs.length > 0) {
+    console.log(`🔄 Regenerating ${priorityDocs.length} priority documents first, then ${remainingDocs.length} remaining (${reason})`);
+  } else {
+    console.log(`🔄 Regenerating ${documentPaths.length} documents (${reason})`);
+  }
+
+  // Process priority documents first
+  for (const docPath of priorityDocs) {
+    try {
+      const result = await regenerateSingleFile(docPath, { _source, _meta, _output });
+      if (result.success) {
+        regenerated++;
+      } else {
+        console.warn(`  ⚠️  ${docPath}: ${result.message}`);
+        failed++;
+      }
+    } catch (e) {
+      console.error(`  ❌ ${docPath}: ${e.message}`);
+      failed++;
+    }
+  }
+
+  // Notify caller that priority docs are done (so server can reload those clients immediately)
+  if (priorityDocs.length > 0 && onPriorityComplete) {
+    try {
+      onPriorityComplete({ regenerated, failed, priorityDocs });
+    } catch (e) {
+      console.error(`  ⚠️ onPriorityComplete callback error: ${e.message}`);
+    }
+  }
+
+  // Process remaining documents
+  for (const docPath of remainingDocs) {
+    try {
+      const result = await regenerateSingleFile(docPath, { _source, _meta, _output });
+      if (result.success) {
+        regenerated++;
+      } else {
+        console.warn(`  ⚠️  ${docPath}: ${result.message}`);
+        failed++;
+      }
+    } catch (e) {
+      console.error(`  ❌ ${docPath}: ${e.message}`);
+      failed++;
+    }
+  }
+
+  // Persist updated dependency registrations (e.g. a doc switched templates)
+  if (regenerated > 0) {
+    await saveDependencyTracker(resolve(_source) + "/");
+  }
+
+  const elapsed = Date.now() - startTime;
+  const msg = `Regenerated ${regenerated}/${documentPaths.length} documents in ${elapsed}ms (${reason})${failed > 0 ? `, ${failed} failed` : ""}`;
+  console.log(`✅ ${msg}`);
+  return { success: failed === 0, message: msg, regenerated, failed };
+}
+
+/**
+ * Regenerate a single file without scanning the entire source directory.
+ * This is much faster for watch mode - only regenerate what changed.
+ * 
+ * @param {string} changedFile - Absolute path to the file that changed
+ * @param {Object} options - Same options as generate()
+ * @returns {Promise<{success: boolean, message: string}>}
+ */
+export async function regenerateSingleFile(changedFile, {
+  _source,
+  _meta,
+  _output,
+} = {}) {
+  const startTime = Date.now();
+  const source = resolve(_source) + "/";
+  const meta = resolve(_meta);
+  const output = resolve(_output) + "/";
+  
+  // Check if this is an article file we can regenerate
+  const articleExtensions = /\.(md|mdx|txt|yml)$/;
+  if (!changedFile.match(articleExtensions)) {
+    return { success: false, message: `Not an article file: ${changedFile}` };
+  }
+  
+  // Check if cache is initialized
+  if (!watchModeCache.isInitialized) {
+    return { success: false, message: 'Cache not initialized - need full build first' };
+  }
+  
+  // Verify paths match cached paths
+  if (watchModeCache.source !== source || watchModeCache.output !== output) {
+    return { success: false, message: 'Paths changed - need full rebuild' };
+  }
+  
+  try {
+    const { templates, menu, footer, validPaths, hashCache, cacheBustTimestamp, imageMap, customMenus, ursaMetadata } = watchModeCache;
+
+    const rawBody = await readFile(changedFile, "utf8");
+    const type = parse(changedFile).ext;
+    const ext = extname(changedFile);
+    const base = basename(changedFile, ext);
+    const dir = addTrailingSlash(dirname(changedFile)).replace(source, "");
+    
+    // Calculate output paths
+    const outputFilename = changedFile
+      .replace(source, output)
+      .replace(parse(changedFile).ext, ".html");
+    const url = '/' + outputFilename.replace(output, '');
+    
+    // Title from filename (for index/home, use parent folder name)
+    const titleBase = (base === 'index' || base === 'home') ? basename(dirname(changedFile)) : base;
+    const title = toTitleCase(titleBase || base);
+    
+    // Extract metadata
+    const fileMeta = extractMetadata(rawBody);
+    const transformedMetadata = await getTransformedMetadata(
+      dirname(changedFile),
+      fileMeta
+    );
+    
+    // Calculate the document's URL path
+    const docUrlPath = '/' + dir + base + '.html';
+    
+    // Check if hydration should be enabled for MDX files
+    const shouldHydrate = type === '.mdx' && fileMeta?.hydrate === true;
+    
+    // Render body (use async for .mdx, sync for .md/.txt)
+    let body;
+    let hydrationScript = '';
+    if (type === '.mdx') {
+      const renderResult = await renderFileAsync({
+        fileContents: rawBody,
+        type,
+        dirname: dir,
+        basename: base,
+        filePath: changedFile,
+        sourceRoot: source,
+        useWorker: false,
+        hydrate: shouldHydrate,
+      });
+      
+      // Handle the result - can be string or { html, hydrationScript }
+      if (typeof renderResult === 'object' && renderResult.html) {
+        body = renderResult.html;
+        hydrationScript = renderResult.hydrationScript || '';
+      } else {
+        body = renderResult;
+      }
+    } else {
+      body = renderFile({
+        fileContents: rawBody,
+        type,
+        dirname: dir,
+        basename: base,
+      });
+    }
+
+    // Inject default H1 if body doesn't start with one
+    if (!body || !body.trimStart().startsWith('<h1')) {
+      const h1Title = fileMeta?.title || title;
+      body = `<h1>${h1Title}</h1>\n` + (body || '');
+    }
+
+    // Inject breadcrumbs before the H1
+    const breadcrumbs = generateBreadcrumbs(dir, base, fileMeta);
+    if (breadcrumbs) {
+      body = breadcrumbs + body;
+    }
+    
+    // Inject frontmatter table for markdown/mdx files
+    if ((type === '.md' || type === '.mdx') && fileMeta) {
+      body = injectFrontmatterTable(body, fileMeta);
+    }
+
+    // Handle auto-index generation for index files with generate-auto-index: true
+    if (base === 'index' && fileMeta) {
+      const autoIndexConfig = getAutoIndexConfig(fileMeta);
+      if (autoIndexConfig.enabled) {
+        // Generate auto-index HTML for this directory from source
+        const sourceDir = dirname(changedFile);
+        const autoIndexHtml = await generateAutoIndexHtmlFromSource(sourceDir, autoIndexConfig.depth);
+        
+        if (autoIndexHtml) {
+          if (autoIndexConfig.position === 'bottom') {
+            body = body + '\n' + autoIndexHtml;
+          } else {
+            body = autoIndexHtml + '\n' + body;
+          }
+        }
+      }
+    }
+
+    // Find all CSS files up the tree and create separate link tags
+    // (regenerateSingleFile is used in serve mode, so separate tags per level for invalidation)
+    let styleLink = "";
+    let cssPaths = [];
+    try {
+      const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
+      cssPaths = await findAllStyleCss(dirKey, _source);
+      if (cssPaths.length > 0) {
+        // Copy all CSS files to output (always copy in single-file mode to ensure up to date)
+        for (const cssPath of cssPaths) {
+          const cssOutputPath = cssPath.replace(source, output);
+          const cssContent = await readFile(cssPath, 'utf8');
+          await outputFile(cssOutputPath, cssContent);
+        }
+        styleLink = generateSeparateCssTags(cssPaths, source);
+      }
+    } catch (e) {
+      // ignore
+    }
+    
+    // Get template
+    const requestedTemplateName = fileMeta && fileMeta.template;
+    const template =
+      templates[requestedTemplateName] || templates[DEFAULT_TEMPLATE_NAME];
+    
+    if (!template) {
+      return { success: false, message: `Template not found: ${requestedTemplateName || DEFAULT_TEMPLATE_NAME}` };
+    }
+    
+    // Find all script.js files from docroot to current dir and serve as separate external tags
+    // (Serve mode: separate tags per level for individual invalidation)
+    let customScript = "";
+    let scriptPaths = [];
+    try {
+      const dirKey = (dir === "/" || dir === "") ? _source : resolve(_source, dir);
+      scriptPaths = await findAllScriptJs(dirKey, _source);
+      if (scriptPaths.length > 0) {
+        // Copy all script files to output so they can be served
+        for (const scriptPath of scriptPaths) {
+          const scriptOutputPath = scriptPath.replace(source, output);
+          const scriptContent = await readFile(scriptPath, 'utf8');
+          await outputFile(scriptOutputPath, scriptContent);
+        }
+        customScript = generateSeparateJsTags(scriptPaths, source);
+      }
+    } catch (e) {
+      // ignore
+    }
+
+    // Register this document's dependencies (template, inherited CSS/JS) so
+    // invalidation plans stay accurate after frontmatter/template changes.
+    // Persisted to .ursa/dependency-graph.json by regenerateAffectedDocuments.
+    const usedTemplateName = (requestedTemplateName && templates[requestedTemplateName])
+      ? requestedTemplateName
+      : DEFAULT_TEMPLATE_NAME;
+    dependencyTracker.registerDocument(changedFile, {
+      templateName: usedTemplateName,
+      cssPaths,
+      scriptPaths,
+    });
+
+    // Check if this file has a custom menu
+    const customMenuInfo = customMenus ? getCustomMenuForFile(changedFile, source, customMenus) : null;
+
+    // Append hydration script to customScript if present (for MDX with hydrate: true)
+    const finalCustomScript = hydrationScript 
+      ? customScript + '\n' + hydrationScript 
+      : customScript;
+
+    // Build final HTML
+    let finalHtml = template;
+    const replacements = {
+      "${title}": fileMeta?.title || title,
+      "${menu}": menu,
+      "${meta}": JSON.stringify(fileMeta),
+      "${transformedMetadata}": transformedMetadata,
+      "${body}": body,
+      "${styleLink}": styleLink,
+      "${customScript}": finalCustomScript,
+      "${searchIndex}": "[]",
+      "${footer}": footer
+    };
+    for (const [key, value] of Object.entries(replacements)) {
+      finalHtml = finalHtml.replace(key, value);
+    }
+
+    // If this page has a custom menu, add data attributes to body
+    if (customMenuInfo) {
+      const menuPosition = customMenuInfo.menuPosition || 'top';
+      finalHtml = finalHtml.replace(
+        /<body([^>]*)>/,
+        `<body$1 data-custom-menu="${customMenuInfo.menuJsonPath}" data-menu-position="${menuPosition}">`
+      );
+    } else {
+      // No custom menu — default to top menu
+      finalHtml = finalHtml.replace(
+        /<body([^>]*)>/,
+        `<body$1 data-menu-position="top">`
+      );
+    }
+    
+    // Resolve relative URLs in raw HTML elements (img src, etc.)
+    finalHtml = resolveRelativeUrls(finalHtml, docUrlPath);
+    
+    // Mark broken links
+    finalHtml = markInactiveLinks(finalHtml, validPaths, docUrlPath, false);
+    
+    // Transform image tags to use preview images with data-fullsrc for originals
+    if (imageMap) {
+      finalHtml = transformImageTags(finalHtml, imageMap, docUrlPath);
+    }
+    
+    // Add cache-busting timestamps to static file references
+    finalHtml = addTimestampToHtmlStaticRefs(finalHtml, cacheBustTimestamp);
+    
+    await outputFile(outputFilename, finalHtml);
+    
+    // JSON output
+    const jsonOutputFilename = outputFilename.replace(".html", ".json");
+    const sections = (type === '.md' || type === '.mdx') ? extractSections(rawBody) : [];
+    const jsonObject = {
+      name: base,
+      url,
+      contents: rawBody,
+      bodyHtml: body,
+      metadata: fileMeta,
+      sections,
+      transformedMetadata,
+      _ursa_metadata: ursaMetadata,
+    };
+    const json = JSON.stringify(jsonObject);
+    await outputFile(jsonOutputFilename, json);
+    
+    // XML output
+    const xmlOutputFilename = outputFilename.replace(".html", ".xml");
+    const xml = `<article>${o2x(jsonObject)}</article>`;
+    await outputFile(xmlOutputFilename, xml);
+
+    // Folder-named index promotion (mirrors autoIndex behavior on full builds):
+    // If the file is named like its parent folder (e.g. aletheia/aletheia.md) and
+    // no explicit index.{md,mdx,txt,yml,html} exists alongside it, also write the
+    // same outputs to <dir>/index.html|json|xml so the canonical URL stays fresh.
+    const sourceDirOfFile = dirname(changedFile);
+    const parentFolderName = basename(sourceDirOfFile);
+    if (base && parentFolderName && base === parentFolderName) {
+      const hasExplicitIndex = ['index.md', 'index.mdx', 'index.txt', 'index.yml', 'index.html']
+        .some(name => existsSync(join(sourceDirOfFile, name)));
+      if (!hasExplicitIndex) {
+        const outDirOfFile = dirname(outputFilename);
+        await outputFile(join(outDirOfFile, 'index.html'), finalHtml);
+        await outputFile(join(outDirOfFile, 'index.json'), json);
+        await outputFile(join(outDirOfFile, 'index.xml'), xml);
+      }
+    }
+
+    // Update hash cache
+    updateHash(changedFile, rawBody, hashCache);
+    
+    // Update recent-activity.json with this file's new content timestamp
+    try {
+      const now = Date.now();
+      const recentActivityPath = join(output, 'public', 'recent-activity.json');
+      let recentActivity = [];
+      try {
+        const existing = await readFile(recentActivityPath, 'utf8');
+        recentActivity = JSON.parse(existing);
+      } catch (e) { /* no existing file, start fresh */ }
+      // Remove old entry for this URL if present
+      recentActivity = recentActivity.filter(r => r.url !== url);
+      // Add updated entry with current timestamp (content changed now)
+      recentActivity.push({ title, url, mtime: now });
+      // Sort by mtime descending, keep top 10
+      recentActivity.sort((a, b) => b.mtime - a.mtime);
+      recentActivity = recentActivity.slice(0, 10);
+      await outputFile(recentActivityPath, JSON.stringify(recentActivity));
+      
+      // Also update content timestamp in .ursa.json for persistence
+      const relativePath = '/' + changedFile.replace(source, '').replace(/\.(md|mdx|txt|yml)$/, '.html');
+      updateContentTimestamp(source, relativePath, now);
+    } catch (e) {
+      // ignore recent activity update errors
+    }
+    
+    const elapsed = Date.now() - startTime;
+    const shortFile = changedFile.replace(source, '');
+    return { success: true, message: `Regenerated ${shortFile} in ${elapsed}ms` };
+  } catch (e) {
+    return { success: false, message: `Error: ${e.message}` };
+  }
 }