@kenjura/ursa 0.44.0 → 0.46.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.46.0
+2025-12-20
+
+- Normalized handling of trailing slashes in URLs
+
+# 0.45.0
+2025-12-20
+
+- Added --exclude flag to ignore specified files or directories during generation
+- Improved performance of the serve command with optimized file watching
+- Automatically generating index.html for directories without an index file
+
 # 0.44.0
 2025-12-16
 

package/README.md

@@ -94,6 +94,26 @@ important-document
 classes/wizard
 ```
 
+### Large Workloads
+
+For sites with many documents (hundreds or thousands), you may need to increase Node.js memory limits:
+
+```bash
+# Increase heap size to 8GB for large sites
+node --max-old-space-size=8192 $(which ursa) serve content
+
+# Or use the npm scripts
+npm run serve:large content
+npm run generate:large content
+
+# You can also set environment variables to tune batch processing
+URSA_BATCH_SIZE=25 ursa serve content  # Process fewer files at once (default: 50)
+```
+
+**Environment Variables for Performance Tuning:**
+- `URSA_BATCH_SIZE` - Number of files to process concurrently (default: 50). Lower values use less memory but are slower.
+- `NODE_OPTIONS="--max-old-space-size=8192"` - Increase Node.js heap size for very large sites.
+
 ## Library Usage
 
 ### ES Modules (recommended)

package/bin/ursa.js

@@ -38,6 +38,11 @@ yargs(hideBin(process.argv))
           describe: 'Path to whitelist file containing patterns for files to include',
           type: 'string'
         })
+        .option('exclude', {
+          alias: 'x',
+          describe: 'Folders to exclude: comma-separated paths relative to source, or path to file with one folder per line',
+          type: 'string'
+        })
         .option('clean', {
           alias: 'c',
           describe: 'Ignore cached hashes and regenerate all files',
@@ -50,12 +55,16 @@ yargs(hideBin(process.argv))
       const meta = argv.meta ? resolve(argv.meta) : PACKAGE_META;
       const output = resolve(argv.output);
       const whitelist = argv.whitelist ? resolve(argv.whitelist) : null;
+      const exclude = argv.exclude || null;
       const clean = argv.clean;
       
       console.log(`Generating site from ${source} to ${output} using meta from ${meta}`);
       if (whitelist) {
         console.log(`Using whitelist: ${whitelist}`);
       }
+      if (exclude) {
+        console.log(`Excluding: ${exclude}`);
+      }
       if (clean) {
         console.log(`Clean build: ignoring cached hashes`);
       }
@@ -66,6 +75,7 @@ yargs(hideBin(process.argv))
           _meta: meta,
           _output: output,
           _whitelist: whitelist,
+          _exclude: exclude,
           _clean: clean
         });
         console.log('Site generation completed successfully!');
@@ -107,6 +117,11 @@ yargs(hideBin(process.argv))
           describe: 'Path to whitelist file containing patterns for files to include',
           type: 'string'
         })
+        .option('exclude', {
+          alias: 'x',
+          describe: 'Folders to exclude: comma-separated paths relative to source, or path to file with one folder per line',
+          type: 'string'
+        })
         .option('clean', {
           alias: 'c',
           describe: 'Ignore cached hashes and regenerate all files',
@@ -120,6 +135,7 @@ yargs(hideBin(process.argv))
       const output = resolve(argv.output);
       const port = argv.port;
       const whitelist = argv.whitelist ? resolve(argv.whitelist) : null;
+      const exclude = argv.exclude || null;
       const clean = argv.clean;
       
       console.log(`Starting development server...`);
@@ -130,6 +146,9 @@ yargs(hideBin(process.argv))
       if (whitelist) {
         console.log(`Using whitelist: ${whitelist}`);
       }
+      if (exclude) {
+        console.log(`Excluding: ${exclude}`);
+      }
       
       try {
         const { serve } = await import('../src/serve.js');
@@ -139,6 +158,7 @@ yargs(hideBin(process.argv))
           _output: output,
           port: port,
           _whitelist: whitelist,
+          _exclude: exclude,
           _clean: clean
         });
       } catch (error) {

package/meta/default-template.html

@@ -9,8 +9,15 @@
         ${embeddedStyle}
      </style>
      <script>
-        // Embed search index data
+        // Search index loaded asynchronously from separate file to reduce page size
         window.SEARCH_INDEX = ${searchIndex};
+        // Lazy load full search index if placeholder is empty
+        if (!window.SEARCH_INDEX || window.SEARCH_INDEX.length === 0) {
+          fetch('/public/search-index.json')
+            .then(r => r.json())
+            .then(data => { window.SEARCH_INDEX = data; })
+            .catch(() => { window.SEARCH_INDEX = []; });
+        }
      </script>
      <script src="/public/search.js"></script>
      

package/package.json

@@ -2,7 +2,7 @@
   "name": "@kenjura/ursa",
   "author": "Andrew London <andrew@kenjura.com>",
   "type": "module",
-  "version": "0.44.0",
+  "version": "0.46.0",
   "description": "static site generator from MD/wikitext/YML",
   "main": "lib/index.js",
   "bin": {
@@ -11,6 +11,8 @@
   "scripts": {
     "serve": "nodemon --config nodemon.json  src/serve.js",
     "serve:debug": "nodemon --config nodemon.json  --inspect-brk src/serve.js",
+    "serve:large": "node --max-old-space-size=8192 bin/ursa.js serve",
+    "generate:large": "node --max-old-space-size=8192 bin/ursa.js generate",
     "cli:debug": "node --inspect bin/ursa.js",
     "cli:debug-brk": "node --inspect-brk bin/ursa.js",
     "start": "node src/index.js",

package/src/helper/recursive-readdir.js

@@ -1,13 +1,34 @@
 import { resolve } from "path";
 import { readdir } from "fs/promises";
 
+/**
+ * Recursively read directory contents.
+ * Optimized to be more memory-efficient by using iteration instead of deep recursion.
+ * @param {string} dir - Directory to read
+ * @returns {Promise<string[]>} Array of file paths
+ */
 export async function recurse(dir) {
-  const dirents = await readdir(dir, { withFileTypes: true });
-  const files = await Promise.all(
-    dirents.map(async (dirent) => {
-      const res = resolve(dir, dirent.name);
-      return dirent.isDirectory() ? [res, ...(await recurse(res))] : res;
-    })
-  );
-  return Array.prototype.concat(...files);
+  const results = [];
+  const stack = [dir];
+  
+  while (stack.length > 0) {
+    const currentDir = stack.pop();
+    try {
+      const dirents = await readdir(currentDir, { withFileTypes: true });
+      for (const dirent of dirents) {
+        const res = resolve(currentDir, dirent.name);
+        if (dirent.isDirectory()) {
+          results.push(res);
+          stack.push(res);
+        } else {
+          results.push(res);
+        }
+      }
+    } catch (e) {
+      // Skip directories we can't read (permission errors, etc.)
+      console.warn(`Warning: Could not read directory ${currentDir}: ${e.message}`);
+    }
+  }
+  
+  return results;
 }

package/src/jobs/generate.js

@@ -1,6 +1,77 @@
 import { recurse } from "../helper/recursive-readdir.js";
 
 import { copyFile, mkdir, readdir, readFile, stat } from "fs/promises";
+
+// Concurrency limiter for batch processing to avoid memory exhaustion
+const BATCH_SIZE = parseInt(process.env.URSA_BATCH_SIZE || '50', 10);
+
+/**
+ * Progress reporter that updates lines in place (like pnpm)
+ */
+class ProgressReporter {
+  constructor() {
+    this.lines = {};
+    this.isTTY = process.stdout.isTTY;
+  }
+  
+  // Update a named status line in place
+  status(name, message) {
+    if (this.isTTY) {
+      // Save cursor, move to line, clear it, write, restore cursor
+      const line = `${name}: ${message}`;
+      this.lines[name] = line;
+      // Clear line and write
+      process.stdout.write(`\r\x1b[K${line}`);
+    }
+  }
+  
+  // Complete a status line (print final state and newline)
+  done(name, message) {
+    if (this.isTTY) {
+      process.stdout.write(`\r\x1b[K${name}: ${message}\n`);
+    } else {
+      console.log(`${name}: ${message}`);
+    }
+    delete this.lines[name];
+  }
+  
+  // Regular log that doesn't get overwritten
+  log(message) {
+    if (this.isTTY) {
+      // Clear current line first, print message, then newline
+      process.stdout.write(`\r\x1b[K${message}\n`);
+    } else {
+      console.log(message);
+    }
+  }
+  
+  // Clear all status lines
+  clear() {
+    if (this.isTTY) {
+      process.stdout.write(`\r\x1b[K`);
+    }
+  }
+}
+
+const progress = new ProgressReporter();
+
+/**
+ * Process items in batches to limit memory usage
+ * @param {Array} items - Items to process
+ * @param {Function} processor - Async function to process each item
+ * @param {number} batchSize - Max concurrent operations
+ */
+async function processBatched(items, processor, batchSize = BATCH_SIZE) {
+  const results = [];
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+    const batchResults = await Promise.all(batch.map(processor));
+    results.push(...batchResults);
+    // Allow GC to run between batches
+    if (global.gc) global.gc();
+  }
+  return results;
+}
 import { getAutomenu } from "../helper/automenu.js";
 import { filterAsync } from "../helper/filterAsync.js";
 import { isDirectory } from "../helper/isDirectory.js";
@@ -68,15 +139,80 @@ import { createWhitelistFilter } from "../helper/whitelistFilter.js";
 const DEFAULT_TEMPLATE_NAME =
   process.env.DEFAULT_TEMPLATE_NAME ?? "default-template";
 
+/**
+ * Parse exclude option - can be comma-separated paths or a file path
+ * @param {string} excludeOption - The exclude option value
+ * @param {string} source - Source directory path
+ * @returns {Promise<Set<string>>} Set of excluded folder paths (normalized)
+ */
+async function parseExcludeOption(excludeOption, source) {
+  const excludedPaths = new Set();
+  
+  if (!excludeOption) return excludedPaths;
+  
+  // Check if it's a file path (exists as a file)
+  const isFile = existsSync(excludeOption) && (await stat(excludeOption)).isFile();
+  
+  let patterns;
+  if (isFile) {
+    // Read patterns from file (one per line)
+    const content = await readFile(excludeOption, 'utf8');
+    patterns = content.split('\n')
+      .map(line => line.trim())
+      .filter(line => line && !line.startsWith('#')); // Skip empty lines and comments
+  } else {
+    // Treat as comma-separated list
+    patterns = excludeOption.split(',').map(p => p.trim()).filter(Boolean);
+  }
+  
+  // Normalize patterns to absolute paths
+  for (const pattern of patterns) {
+    // Remove leading/trailing slashes and normalize
+    const normalized = pattern.replace(/^\/+|\/+$/g, '');
+    // Store as relative path for easier matching
+    excludedPaths.add(normalized);
+  }
+  
+  return excludedPaths;
+}
+
+/**
+ * Create a filter function that excludes files in specified folders
+ * @param {Set<string>} excludedPaths - Set of excluded folder paths
+ * @param {string} source - Source directory path
+ * @returns {Function} Filter function
+ */
+function createExcludeFilter(excludedPaths, source) {
+  if (excludedPaths.size === 0) {
+    return () => true; // No exclusions, allow all
+  }
+  
+  return (filePath) => {
+    // Get path relative to source
+    const relativePath = filePath.replace(source, '').replace(/^\/+/, '');
+    
+    // Check if file is in any excluded folder
+    for (const excluded of excludedPaths) {
+      if (relativePath === excluded || 
+          relativePath.startsWith(excluded + '/') ||
+          relativePath.startsWith(excluded + '\\')) {
+        return false; // Exclude this file
+      }
+    }
+    return true; // Include this file
+  };
+}
+
 export async function generate({
   _source = join(process.cwd(), "."),
   _meta = join(process.cwd(), "meta"),
   _output = join(process.cwd(), "build"),
   _whitelist = null,
+  _exclude = null,
   _incremental = false,  // Legacy flag, now ignored (always incremental)
   _clean = false,  // When true, ignore cache and regenerate all files
 } = {}) {
-  console.log({ _source, _meta, _output, _whitelist, _clean });
+  console.log({ _source, _meta, _output, _whitelist, _exclude, _clean });
   const source = resolve(_source) + "/";
   const meta = resolve(_meta);
   const output = resolve(_output) + "/";
@@ -90,6 +226,15 @@ export async function generate({
     : Boolean;
   let allSourceFilenames = allSourceFilenamesUnfiltered.filter(includeFilter);
   
+  // Apply exclude filter if specified
+  if (_exclude) {
+    const excludedPaths = await parseExcludeOption(_exclude, source);
+    const excludeFilter = createExcludeFilter(excludedPaths, source);
+    const beforeCount = allSourceFilenames.length;
+    allSourceFilenames = allSourceFilenames.filter(excludeFilter);
+    progress.log(`Exclude filter applied: ${beforeCount - allSourceFilenames.length} files excluded`);
+  }
+  
   // Apply whitelist filter if specified
   if (_whitelist) {
     const whitelistFilter = await createWhitelistFilter(_whitelist, source);
@@ -126,13 +271,13 @@ export async function generate({
 
   // Build set of valid internal paths for link validation (must be before menu)
   const validPaths = buildValidPaths(allSourceFilenamesThatAreArticles, source);
-  console.log(`Built ${validPaths.size} valid paths for link validation`);
+  progress.log(`Built ${validPaths.size} valid paths for link validation`);
 
   const menu = await getMenu(allSourceFilenames, source, validPaths);
 
   // Get and increment build ID from .ursa.json
   const buildId = getAndIncrementBuildId(resolve(_source));
-  console.log(`Build #${buildId}`);
+  progress.log(`Build #${buildId}`);
 
   // Generate footer content
   const footer = await getFooter(source, _source, buildId);
@@ -141,9 +286,9 @@ export async function generate({
   let hashCache = new Map();
   if (!_clean) {
     hashCache = await loadHashCache(source);
-    console.log(`Loaded ${hashCache.size} cached content hashes from .ursa folder`);
+    progress.log(`Loaded ${hashCache.size} cached content hashes from .ursa folder`);
   } else {
-    console.log(`Clean build: ignoring cached hashes`);
+    progress.log(`Clean build: ignoring cached hashes`);
   }
 
   // create public folder
@@ -154,294 +299,286 @@ export async function generate({
   // Track errors for error report
   const errors = [];
 
-  // First pass: collect search index data
+  // Search index: built incrementally during article processing (lighter memory footprint)
   const searchIndex = [];
-  const jsonCache = new Map();
-  
-  // Collect basic data for search index
-  for (const file of allSourceFilenamesThatAreArticles) {
+  // Directory index cache: only stores minimal data needed for directory indices
+  // Uses WeakRef-style approach - store only what's needed, clear as we go
+  const dirIndexCache = new Map();
+
+  // Track files that were regenerated (for incremental mode stats)
+  let regeneratedCount = 0;
+  let skippedCount = 0;
+  let processedCount = 0;
+  const totalArticles = allSourceFilenamesThatAreArticles.length;
+
+  progress.log(`Processing ${totalArticles} articles in batches of ${BATCH_SIZE}...`);
+
+  // Single pass: process all articles with batched concurrency to limit memory usage
+  await processBatched(allSourceFilenamesThatAreArticles, async (file) => {
     try {
+      processedCount++;
+      const shortFile = file.replace(source, '');
+      progress.status('Articles', `${processedCount}/${totalArticles} ${shortFile}`);
+      
       const rawBody = await readFile(file, "utf8");
       const type = parse(file).ext;
       const ext = extname(file);
       const base = basename(file, ext);
       const dir = addTrailingSlash(dirname(file)).replace(source, "");
       
+      // Calculate output paths for this file
+      const outputFilename = file
+        .replace(source, output)
+        .replace(parse(file).ext, ".html");
+      const url = '/' + outputFilename.replace(output, '');
+      
+      // Generate URL path relative to output (for search index)
+      const relativePath = file.replace(source, '').replace(/\.(md|txt|yml)$/, '.html');
+      const searchUrl = relativePath.startsWith('/') ? relativePath : '/' + relativePath;
+      
       // Generate title from filename (in title case)
       const title = toTitleCase(base);
       
-      // Generate URL path relative to output
-      const relativePath = file.replace(source, '').replace(/\.(md|txt|yml)$/, '.html');
-      const url = relativePath.startsWith('/') ? relativePath : '/' + relativePath;
+      // Always add to search index (lightweight: title + path only, content added lazily)
+      searchIndex.push({
+        title: title,
+        path: relativePath,
+        url: searchUrl,
+        content: '' // Content excerpts built lazily to save memory
+      });
+      
+      // Check if file needs regeneration
+      const needsRegen = _clean || needsRegeneration(file, rawBody, hashCache);
+      
+      if (!needsRegen) {
+        skippedCount++;
+        // For directory indices, store minimal data (not full bodyHtml)
+        dirIndexCache.set(file, {
+          name: base,
+          url,
+          // Don't store contents or bodyHtml - saves significant memory
+        });
+        return; // Skip regenerating this file
+      }
+      
+      regeneratedCount++;
+
+      const fileMeta = extractMetadata(rawBody);
+      const rawMeta = extractRawMetadata(rawBody);
+      const transformedMetadata = await getTransformedMetadata(
+        dirname(file),
+        fileMeta
+      );
       
-      // Basic content processing for search (without full rendering)
+      // Calculate the document's URL path (e.g., "/character/index.html")
+      const docUrlPath = '/' + dir + base + '.html';
+
       const body = renderFile({
         fileContents: rawBody,
         type,
         dirname: dir,
         basename: base,
       });
-      
-      // Extract text content from body (strip HTML tags for search)
-      const textContent = body && body.replace && body.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim() || 'body is undefined for some reason'
-      const excerpt = textContent.substring(0, 200); // First 200 chars for preview
-      
-      searchIndex.push({
-        title: title,
-        path: relativePath,
-        url: url,
-        content: excerpt
-      });
-    } catch (e) {
-      console.error(`Error processing ${file} (first pass): ${e.message}`);
-      errors.push({ file, phase: 'search-index', error: e });
-    }
-  }
-  
-  console.log(`Built search index with ${searchIndex.length} entries`);
 
-  // Track files that were regenerated (for incremental mode stats)
-  let regeneratedCount = 0;
-  let skippedCount = 0;
-
-  // Second pass: process individual articles with search data available
-  await Promise.all(
-    allSourceFilenamesThatAreArticles.map(async (file) => {
+      // Find nearest style.css or _style.css up the tree
+      let embeddedStyle = "";
       try {
-        const rawBody = await readFile(file, "utf8");
-        const type = parse(file).ext;
-        const ext = extname(file);
-        const base = basename(file, ext);
-        const dir = addTrailingSlash(dirname(file)).replace(source, "");
-        
-        // Calculate output paths for this file
-        const outputFilename = file
-          .replace(source, output)
-          .replace(parse(file).ext, ".html");
-        const url = '/' + outputFilename.replace(output, '');
-        
-        // Skip files that haven't changed (unless --clean flag is set)
-        if (!_clean && !needsRegeneration(file, rawBody, hashCache)) {
-          skippedCount++;
-          // Still need to populate jsonCache for directory indices
-          const meta = extractMetadata(rawBody);
-          const body = renderFile({
-            fileContents: rawBody,
-            type,
-            dirname: dir,
-            basename: base,
-          });
-          // Extract sections for markdown files
-          const sections = type === '.md' ? extractSections(rawBody) : [];
-          
-          jsonCache.set(file, {
-            name: base,
-            url,
-            contents: rawBody,
-            bodyHtml: body,
-            metadata: meta,
-            sections,
-            transformedMetadata: '',
-          });
-          return; // Skip regenerating this file
-        }
-        
-        console.log(`processing article ${file}`);
-        regeneratedCount++;
-
-        const meta = extractMetadata(rawBody);
-        const rawMeta = extractRawMetadata(rawBody);
-        const bodyLessMeta = rawMeta ? rawBody.replace(rawMeta, "") : rawBody;
-        const transformedMetadata = await getTransformedMetadata(
-          dirname(file),
-          meta
-        );
-        
-        // Calculate the document's URL path (e.g., "/character/index.html")
-        const docUrlPath = '/' + dir + base + '.html';
-        
-        // Generate title from filename (in title case)
-        const title = toTitleCase(base);
-
-        const body = renderFile({
-          fileContents: rawBody,
-          type,
-          dirname: dir,
-          basename: base,
-        });
-
-        // Find nearest style.css or _style.css up the tree
-        let embeddedStyle = "";
-        try {
-          const css = await findStyleCss(resolve(_source, dir));
-          if (css) {
-            embeddedStyle = css;
-          }
-        } catch (e) {
-          // ignore
-          console.error(e);
-        }
-
-        const requestedTemplateName = meta && meta.template;
-        const template =
-          templates[requestedTemplateName] || templates[DEFAULT_TEMPLATE_NAME];
-
-        if (!template) {
-          throw new Error(`Template not found. Requested: "${requestedTemplateName || DEFAULT_TEMPLATE_NAME}". Available templates: ${Object.keys(templates).join(', ') || 'none'}`);
+        const css = await findStyleCss(resolve(_source, dir));
+        if (css) {
+          embeddedStyle = css;
         }
+      } catch (e) {
+        // ignore
+        console.error(e);
+      }
 
-        // Insert embeddedStyle just before </head> if present, else at top
-        let finalHtml = template
-          .replace("${title}", title)
-          .replace("${menu}", menu)
-          .replace("${meta}", JSON.stringify(meta))
-          .replace("${transformedMetadata}", transformedMetadata)
-          .replace("${body}", body)
-          .replace("${embeddedStyle}", embeddedStyle)
-          .replace("${searchIndex}", JSON.stringify(searchIndex))
-          .replace("${footer}", footer);
-
-        // Resolve links and mark broken internal links as inactive (debug mode on)
-        // Pass docUrlPath so relative links can be resolved correctly
-        finalHtml = markInactiveLinks(finalHtml, validPaths, docUrlPath, false);
-
-        console.log(`writing article to ${outputFilename}`);
+      const requestedTemplateName = fileMeta && fileMeta.template;
+      const template =
+        templates[requestedTemplateName] || templates[DEFAULT_TEMPLATE_NAME];
 
-        await outputFile(outputFilename, finalHtml);
+      if (!template) {
+        throw new Error(`Template not found. Requested: "${requestedTemplateName || DEFAULT_TEMPLATE_NAME}". Available templates: ${Object.keys(templates).join(', ') || 'none'}`);
+      }
 
-        // json
+      // Build final HTML with all replacements in a single chain to reduce intermediate strings
+      let finalHtml = template;
+      // Use a map of replacements to minimize string allocations
+      const replacements = {
+        "${title}": title,
+        "${menu}": menu,
+        "${meta}": JSON.stringify(fileMeta),
+        "${transformedMetadata}": transformedMetadata,
+        "${body}": body,
+        "${embeddedStyle}": embeddedStyle,
+        "${searchIndex}": "[]", // Placeholder - search index written separately as JSON file
+        "${footer}": footer
+      };
+      for (const [key, value] of Object.entries(replacements)) {
+        finalHtml = finalHtml.replace(key, value);
+      }
 
-        const jsonOutputFilename = outputFilename.replace(".html", ".json");
-        
-        // Extract sections for markdown files
-        const sections = type === '.md' ? extractSections(rawBody) : [];
-        
-        const jsonObject = {
-          name: base,
-          url,
-          contents: rawBody,
-          // bodyLessMeta: bodyLessMeta,
-          bodyHtml: body,
-          metadata: meta,
-          sections,
-          transformedMetadata,
-          // html: finalHtml,
-        };
-        jsonCache.set(file, jsonObject);
-        const json = JSON.stringify(jsonObject);
-        console.log(`writing article to ${jsonOutputFilename}`);
-        await outputFile(jsonOutputFilename, json);
+      // Resolve links and mark broken internal links as inactive
+      finalHtml = markInactiveLinks(finalHtml, validPaths, docUrlPath, false);
 
-        // xml
+      await outputFile(outputFilename, finalHtml);
+      
+      // Clear finalHtml reference to allow GC
+      finalHtml = null;
 
-        const xmlOutputFilename = outputFilename.replace(".html", ".xml");
-        const xml = `<article>${o2x(jsonObject)}</article>`;
-        await outputFile(xmlOutputFilename, xml);
-        
-        // Update the content hash for this file
-        updateHash(file, rawBody, hashCache);
-      } catch (e) {
-        console.error(`Error processing ${file} (second pass): ${e.message}`);
-        errors.push({ file, phase: 'article-generation', error: e });
-      }
-    })
-  );
+      // JSON output
+      const jsonOutputFilename = outputFilename.replace(".html", ".json");
+      
+      // Extract sections for markdown files
+      const sections = type === '.md' ? extractSections(rawBody) : [];
+      
+      const jsonObject = {
+        name: base,
+        url,
+        contents: rawBody,
+        bodyHtml: body,
+        metadata: fileMeta,
+        sections,
+        transformedMetadata,
+      };
+      
+      // Store minimal data for directory indices
+      dirIndexCache.set(file, {
+        name: base,
+        url,
+      });
+      
+      const json = JSON.stringify(jsonObject);
+      await outputFile(jsonOutputFilename, json);
 
-  // Log build stats
-  console.log(`Build: ${regeneratedCount} regenerated, ${skippedCount} unchanged`);
+      // XML output
+      const xmlOutputFilename = outputFilename.replace(".html", ".xml");
+      const xml = `<article>${o2x(jsonObject)}</article>`;
+      await outputFile(xmlOutputFilename, xml);
+      
+      // Update the content hash for this file
+      updateHash(file, rawBody, hashCache);
+    } catch (e) {
+      progress.log(`Error processing ${file}: ${e.message}`);
+      errors.push({ file, phase: 'article-generation', error: e });
+    }
+  });
 
-  console.log(jsonCache.keys());
-  
-  // process directory indices
-  await Promise.all(
-    allSourceFilenamesThatAreDirectories.map(async (dir) => {
-      try {
-        console.log(`processing directory ${dir}`);
+  // Complete the articles status line
+  progress.done('Articles', `${totalArticles} done (${regeneratedCount} regenerated, ${skippedCount} unchanged)`);
 
-        const pathsInThisDirectory = allSourceFilenames.filter((filename) =>
-          filename.match(new RegExp(`${dir}.+`))
-        );
+  // Write search index as a separate JSON file (not embedded in each page)
+  const searchIndexPath = join(output, 'public', 'search-index.json');
+  progress.log(`Writing search index with ${searchIndex.length} entries`);
+  await outputFile(searchIndexPath, JSON.stringify(searchIndex));
 
-        const jsonObjects = pathsInThisDirectory
+  // Process directory indices with batched concurrency
+  const totalDirs = allSourceFilenamesThatAreDirectories.length;
+  let processedDirs = 0;
+  progress.log(`Processing ${totalDirs} directories...`);
+  await processBatched(allSourceFilenamesThatAreDirectories, async (dirPath) => {
+    try {
+      processedDirs++;
+      const shortDir = dirPath.replace(source, '');
+      progress.status('Directories', `${processedDirs}/${totalDirs} ${shortDir}`);
+
+      const pathsInThisDirectory = allSourceFilenames.filter((filename) =>
+        filename.match(new RegExp(`${dirPath}.+`))
+      );
+
+      // Use minimal directory index cache instead of full jsonCache
+      const jsonObjects = pathsInThisDirectory
+        .map((path) => {
+          const object = dirIndexCache.get(path);
+          return typeof object === "object" ? object : null;
+        })
+        .filter((a) => a);
+
+      const json = JSON.stringify(jsonObjects);
+
+      const outputFilename = dirPath.replace(source, output) + ".json";
+      await outputFile(outputFilename, json);
+
+      // html
+      const htmlOutputFilename = dirPath.replace(source, output) + ".html";
+      const indexAlreadyExists = fileExists(htmlOutputFilename);
+      if (!indexAlreadyExists) {
+        const template = templates["default-template"];
+        const indexHtml = `<ul>${pathsInThisDirectory
           .map((path) => {
-            const object = jsonCache.get(path);
-            return typeof object === "object" ? object : null;
+            const partialPath = path
+              .replace(source, "")
+              .replace(parse(path).ext, ".html");
+            const name = basename(path, parse(path).ext);
+            return `<li><a href="${partialPath}">${name}</a></li>`;
           })
-          .filter((a) => a);
-
-        const json = JSON.stringify(jsonObjects);
-
-        const outputFilename = dir.replace(source, output) + ".json";
-
-        console.log(`writing directory index to ${outputFilename}`);
-        await outputFile(outputFilename, json);
-
-        // html
-        const htmlOutputFilename = dir.replace(source, output) + ".html";
-        const indexAlreadyExists = fileExists(htmlOutputFilename);
-        if (!indexAlreadyExists) {
-          const template = templates["default-template"]; // TODO: figure out a way to specify template for a directory index
-          const indexHtml = `<ul>${pathsInThisDirectory
-            .map((path) => {
-              const partialPath = path
-                .replace(source, "")
-                .replace(parse(path).ext, ".html");
-              const name = basename(path, parse(path).ext);
-              return `<li><a href="${partialPath}">${name}</a></li>`;
-            })
-            .join("")}</ul>`;
-          const finalHtml = template
-            .replace("${menu}", menu)
-            .replace("${body}", indexHtml)
-            .replace("${searchIndex}", JSON.stringify(searchIndex))
-            .replace("${title}", "Index")
-            .replace("${meta}", "{}")
-            .replace("${transformedMetadata}", "")
-            .replace("${embeddedStyle}", "")
-            .replace("${footer}", footer);
-          console.log(`writing directory index to ${htmlOutputFilename}`);
-          await outputFile(htmlOutputFilename, finalHtml);
+          .join("")}</ul>`;
+        let finalHtml = template;
+        const replacements = {
+          "${menu}": menu,
+          "${body}": indexHtml,
+          "${searchIndex}": "[]", // Search index now in separate file
+          "${title}": "Index",
+          "${meta}": "{}",
+          "${transformedMetadata}": "",
+          "${embeddedStyle}": "",
+          "${footer}": footer
+        };
+        for (const [key, value] of Object.entries(replacements)) {
+          finalHtml = finalHtml.replace(key, value);
         }
-      } catch (e) {
-        console.error(`Error processing directory ${dir}: ${e.message}`);
-        errors.push({ file: dir, phase: 'directory-index', error: e });
+        await outputFile(htmlOutputFilename, finalHtml);
       }
-    })
-  );
+    } catch (e) {
+      progress.log(`Error processing directory ${dirPath}: ${e.message}`);
+      errors.push({ file: dirPath, phase: 'directory-index', error: e });
+    }
+  });
+  
+  progress.done('Directories', `${totalDirs} done`);
+
+  // Clear directory index cache to free memory before processing static files
+  dirIndexCache.clear();
 
-  // copy all static files (i.e. images)
+  // copy all static files (i.e. images) with batched concurrency
   const imageExtensions = /\.(jpg|jpeg|png|gif|webp|svg|ico)/; // static asset extensions
   const allSourceFilenamesThatAreImages = allSourceFilenames.filter(
     (filename) => filename.match(imageExtensions)
   );
-  await Promise.all(
-    allSourceFilenamesThatAreImages.map(async (file) => {
-      try {
-        // For incremental mode, check if file has changed using file stat as a quick check
-        if (_incremental) {
-          const fileStat = await stat(file);
-          const statKey = `${file}:stat`;
-          const newStatHash = `${fileStat.size}:${fileStat.mtimeMs}`;
-          if (hashCache.get(statKey) === newStatHash) {
-            return; // Skip unchanged static file
-          }
-          hashCache.set(statKey, newStatHash);
-        }
-        
-        console.log(`processing static file ${file}`);
+  const totalStatic = allSourceFilenamesThatAreImages.length;
+  let processedStatic = 0;
+  let copiedStatic = 0;
+  progress.log(`Processing ${totalStatic} static files...`);
+  await processBatched(allSourceFilenamesThatAreImages, async (file) => {
+    try {
+      processedStatic++;
+      const shortFile = file.replace(source, '');
+      progress.status('Static files', `${processedStatic}/${totalStatic} ${shortFile}`);
+      
+      // Check if file has changed using file stat as a quick check
+      const fileStat = await stat(file);
+      const statKey = `${file}:stat`;
+      const newStatHash = `${fileStat.size}:${fileStat.mtimeMs}`;
+      if (hashCache.get(statKey) === newStatHash) {
+        return; // Skip unchanged static file
+      }
+      hashCache.set(statKey, newStatHash);
+      copiedStatic++;
 
-        const outputFilename = file.replace(source, output);
+      const outputFilename = file.replace(source, output);
 
-        console.log(`writing static file to ${outputFilename}`);
+      await mkdir(dirname(outputFilename), { recursive: true });
+      return await copyFile(file, outputFilename);
+    } catch (e) {
+      progress.log(`Error processing static file ${file}: ${e.message}`);
+      errors.push({ file, phase: 'static-file', error: e });
+    }
+  });
+  
+  progress.done('Static files', `${totalStatic} done (${copiedStatic} copied)`);
 
-        await mkdir(dirname(outputFilename), { recursive: true });
-        return await copyFile(file, outputFilename);
-      } catch (e) {
-        console.error(`Error processing static file ${file}: ${e.message}`);
-        errors.push({ file, phase: 'static-file', error: e });
-      }
-    })
-  );
+  // Automatic index generation for folders without index.html
+  progress.log(`Checking for missing index files...`);
+  await generateAutoIndices(output, allSourceFilenamesThatAreDirectories, source, templates, menu, footer);
 
   // Save the hash cache to .ursa folder in source directory
   if (hashCache.size > 0) {
@@ -478,10 +615,138 @@ export async function generate({
     });
     
     await outputFile(errorReportPath, report);
-    console.log(`\n⚠️  ${errors.length} error(s) occurred during generation.`);
-    console.log(`   Error report written to: ${errorReportPath}\n`);
+    progress.log(`\n⚠️  ${errors.length} error(s) occurred during generation.`);
+    progress.log(`   Error report written to: ${errorReportPath}\n`);
+  } else {
+    progress.log(`\n✅ Generation complete with no errors.\n`);
+  }
+}
+
+/**
+ * Generate automatic index.html files for folders that don't have one
+ * @param {string} output - Output directory path
+ * @param {string[]} directories - List of source directories
+ * @param {string} source - Source directory path
+ * @param {object} templates - Template map
+ * @param {string} menu - Rendered menu HTML
+ * @param {string} footer - Footer HTML
+ */
+async function generateAutoIndices(output, directories, source, templates, menu, footer) {
+  // Alternate index file names to look for (in priority order)
+  const INDEX_ALTERNATES = ['_index.html', 'home.html', '_home.html'];
+  
+  // Normalize paths (remove trailing slashes for consistent replacement)
+  const sourceNorm = source.replace(/\/+$/, '');
+  const outputNorm = output.replace(/\/+$/, '');
+  
+  // Get all output directories (including root)
+  const outputDirs = new Set([outputNorm]);
+  for (const dir of directories) {
+    // Handle both with and without trailing slash in source
+    const outputDir = dir.replace(sourceNorm, outputNorm);
+    outputDirs.add(outputDir);
+  }
+  
+  let generatedCount = 0;
+  let renamedCount = 0;
+  
+  for (const dir of outputDirs) {
+    const indexPath = join(dir, 'index.html');
+    
+    // Skip if index.html already exists
+    if (existsSync(indexPath)) {
+      continue;
+    }
+    
+    // Get folder name for (foldername).html check
+    const folderName = basename(dir);
+    const folderNameAlternate = `${folderName}.html`;
+    
+    // Check for alternate index files
+    let foundAlternate = null;
+    for (const alt of [...INDEX_ALTERNATES, folderNameAlternate]) {
+      const altPath = join(dir, alt);
+      if (existsSync(altPath)) {
+        foundAlternate = altPath;
+        break;
+      }
+    }
+    
+    if (foundAlternate) {
+      // Rename/copy alternate to index.html
+      try {
+        const content = await readFile(foundAlternate, 'utf8');
+        await outputFile(indexPath, content);
+        renamedCount++;
+        progress.status('Auto-index', `Promoted ${basename(foundAlternate)} → index.html in ${dir.replace(outputNorm, '') || '/'}`);
+      } catch (e) {
+        progress.log(`Error promoting ${foundAlternate} to index.html: ${e.message}`);
+      }
+    } else {
+      // Generate a simple index listing direct children
+      try {
+        const children = await readdir(dir, { withFileTypes: true });
+        
+        // Filter to only include relevant files and folders
+        const items = children
+          .filter(child => {
+            // Skip hidden files and index alternates we just checked
+            if (child.name.startsWith('.')) return false;
+            if (child.name === 'index.html') return false;
+            // Include directories and html files
+            return child.isDirectory() || child.name.endsWith('.html');
+          })
+          .map(child => {
+            const isDir = child.isDirectory();
+            const name = isDir ? child.name : child.name.replace('.html', '');
+            const href = isDir ? `${child.name}/` : child.name;
+            const displayName = toTitleCase(name);
+            const icon = isDir ? '📁' : '📄';
+            return `<li>${icon} <a href="${href}">${displayName}</a></li>`;
+          });
+        
+        if (items.length === 0) {
+          // Empty folder, skip generating index
+          continue;
+        }
+        
+        const folderDisplayName = dir === outputNorm ? 'Home' : toTitleCase(folderName);
+        const indexHtml = `<h1>${folderDisplayName}</h1>\n<ul class="auto-index">\n${items.join('\n')}\n</ul>`;
+        
+        const template = templates["default-template"];
+        if (!template) {
+          progress.log(`Warning: No default template for auto-index in ${dir}`);
+          continue;
+        }
+        
+        let finalHtml = template;
+        const replacements = {
+          "${menu}": menu,
+          "${body}": indexHtml,
+          "${searchIndex}": "[]",
+          "${title}": folderDisplayName,
+          "${meta}": "{}",
+          "${transformedMetadata}": "",
+          "${embeddedStyle}": "",
+          "${footer}": footer
+        };
+        for (const [key, value] of Object.entries(replacements)) {
+          finalHtml = finalHtml.replace(key, value);
+        }
+        
+        await outputFile(indexPath, finalHtml);
+        generatedCount++;
+        progress.status('Auto-index', `Generated index.html for ${dir.replace(outputNorm, '') || '/'}`);
+      } catch (e) {
+        progress.log(`Error generating auto-index for ${dir}: ${e.message}`);
+      }
+    }
+  }
+  
+  if (generatedCount > 0 || renamedCount > 0) {
+    progress.done('Auto-index', `${generatedCount} generated, ${renamedCount} promoted`);
   } else {
-    console.log(`\n✅ Generation complete with no errors.\n`);
+    progress.log(`Auto-index: All folders already have index.html`);
   }
 }
 

package/src/serve.js

@@ -4,7 +4,7 @@ import { generate } from "./jobs/generate.js";
 import { join, resolve } from "path";
 import fs from "fs";
 import { promises } from "fs";
-const { readdir } = promises;
+const { readdir, mkdir } = promises;
 
 /**
  * Configurable serve function for CLI and library use
@@ -15,31 +15,39 @@ export async function serve({
   _output,
   port = 8080,
   _whitelist = null,
-  _clean = false
+  _clean = false,
+  _exclude = null
 } = {}) {
   const sourceDir = resolve(_source);
   const metaDir = resolve(_meta);
   const outputDir = resolve(_output);
 
-  console.log({ source: sourceDir, meta: metaDir, output: outputDir, port, whitelist: _whitelist, clean: _clean });
+  console.log({ source: sourceDir, meta: metaDir, output: outputDir, port, whitelist: _whitelist, exclude: _exclude, clean: _clean });
 
-  // Initial generation (use _clean flag only for initial generation)
-  console.log("Generating initial site...");
-  await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _clean });
-  console.log("Initial generation complete. Starting server...");
-
-  // Start file server
+  // Ensure output directory exists and start server immediately
+  await mkdir(outputDir, { recursive: true });
   serveFiles(outputDir, port);
+  console.log(`🚀 Development server running at http://localhost:${port}`);
+  console.log("📁 Serving files from:", outputDir);
+  console.log("⏳ Generating site in background...\n");
+
+  // Initial generation (use _clean flag only for initial generation)
+  generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _exclude, _clean })
+    .then(() => console.log("\n✅ Initial generation complete.\n"))
+    .catch((error) => console.error("Error during initial generation:", error.message));
 
   // Watch for changes
-  console.log("Watching for file changes...");
+  console.log("👀 Watching for changes in:");
+  console.log("   Source:", sourceDir, "(incremental)");
+  console.log("   Meta:", metaDir, "(full rebuild)");
+  console.log("\nPress Ctrl+C to stop the server\n");
   
   // Meta changes trigger full rebuild (templates, CSS, etc. affect all pages)
   watch(metaDir, { recursive: true, filter: /\.(js|json|css|html|md|txt|yml|yaml)$/ }, async (evt, name) => {
     console.log(`Meta files changed! Event: ${evt}, File: ${name}`);
     console.log("Full rebuild required (meta files affect all pages)...");
     try {
-      await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _clean: true });
+      await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _exclude, _clean: true });
       console.log("Regeneration complete.");
     } catch (error) {
       console.error("Error during regeneration:", error.message);
@@ -64,7 +72,7 @@ export async function serve({
     if (isCssChange) {
       console.log("CSS change detected - full rebuild required...");
       try {
-        await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _clean: true });
+        await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _exclude, _clean: true });
         console.log("Regeneration complete.");
       } catch (error) {
         console.error("Error during regeneration:", error.message);
@@ -72,20 +80,13 @@ export async function serve({
     } else {
       console.log("Incremental rebuild...");
       try {
-        await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist });
+        await generate({ _source: sourceDir, _meta: metaDir, _output: outputDir, _whitelist, _exclude });
         console.log("Regeneration complete.");
       } catch (error) {
         console.error("Error during regeneration:", error.message);
       }
     }
   });
-
-  console.log(`🚀 Development server running at http://localhost:${port}`);
-  console.log("📁 Serving files from:", outputDir);
-  console.log("👀 Watching for changes in:");
-  console.log("   Source:", sourceDir, "(incremental)");
-  console.log("   Meta:", metaDir, "(full rebuild)");
-  console.log("\nPress Ctrl+C to stop the server");
 }
 
 /**