@kenjura/ursa 0.10.0 → 0.33.0

package/src/helper/whitelistFilter.js

@@ -0,0 +1,66 @@
+import { readFile } from 'fs/promises';
+import { resolve, relative } from 'path';
+import { existsSync } from 'fs';
+
+/**
+ * Creates a filter function based on a whitelist file
+ * @param {string} whitelistPath - Path to the whitelist file
+ * @param {string} sourceRoot - Root source directory for relative path matching
+ * @returns {Function} Filter function that returns true if file should be included
+ */
+export async function createWhitelistFilter(whitelistPath, sourceRoot) {
+  if (!whitelistPath || !existsSync(whitelistPath)) {
+    return () => true; // No whitelist = include all files
+  }
+
+  try {
+    const whitelistContent = await readFile(whitelistPath, 'utf8');
+    const patterns = whitelistContent
+      .split('\n')
+      .map(line => line.trim())
+      .filter(line => line && !line.startsWith('#')); // Remove empty lines and comments
+
+    if (patterns.length === 0) {
+      return () => true; // Empty whitelist = include all files
+    }
+
+    return (filePath) => {
+      const absolutePath = resolve(filePath);
+      const relativePath = relative(sourceRoot, absolutePath);
+      
+      return patterns.some(pattern => {
+        // Full absolute path match
+        if (pattern.startsWith('/') && absolutePath === pattern) {
+          return true;
+        }
+        
+        // Relative path match (from source root)
+        if (relativePath === pattern || relativePath.includes(pattern)) {
+          return true;
+        }
+        
+        // Directory match (pattern ends with /)
+        if (pattern.endsWith('/')) {
+          const dirPattern = pattern.slice(0, -1);
+          return relativePath.startsWith(dirPattern + '/') || relativePath === dirPattern;
+        }
+        
+        // Filename match
+        const fileName = absolutePath.split('/').pop();
+        if (fileName === pattern) {
+          return true;
+        }
+        
+        // Partial path match (anywhere in the path)
+        if (absolutePath.includes(pattern) || relativePath.includes(pattern)) {
+          return true;
+        }
+        
+        return false;
+      });
+    };
+  } catch (error) {
+    console.warn(`Warning: Could not read whitelist file ${whitelistPath}:`, error.message);
+    return () => true; // Fallback to include all files
+  }
+}

package/src/helper/wikitextHelper.js

@@ -1,11 +1,14 @@
+import { getImageTag } from './WikiImage.js';
+
 let instance = {};
 
 export function wikiToHtml({ wikitext, articleName, args } = {}) {
   if (!args) args = { db: "noDB", noSection: true, noTOC: true };
   if (!wikitext) return "nothing to render";
 
-  const linkbase = ("/" + args.db + "/").replace(/\/\//g, "/");
-  const imageroot = ("/" + args.db + "/img/").replace(/\/\//g, "/");
+  const db = args.db || "noDB";
+  const linkbase = ("/" + db + "/").replace(/\/\//g, "/");
+  const imageroot = ("/" + db + "/img/").replace(/\/\//g, "/");
 
   const allArticles = args.allArticles || [];
 
@@ -330,7 +333,7 @@ export function wikiToHtml({ wikitext, articleName, args } = {}) {
       case "IFRAME":
         return '<iframe src="' + articleName + '"' + getArg(0) + "></iframe>";
       case "IMAGE":
-        return WikiImage.getImageTag({
+        return getImageTag({
           name: articleName,
           args: args,
           imgUrl: imageroot + articleName,

package/src/jobs/generate.js

@@ -8,7 +8,51 @@ import {
   extractMetadata,
   extractRawMetadata,
 } from "../helper/metadataExtractor.js";
+import {
+  hashContent,
+  loadHashCache,
+  saveHashCache,
+  needsRegeneration,
+  updateHash,
+} from "../helper/contentHash.js";
+import {
+  buildValidPaths,
+  markInactiveLinks,
+} from "../helper/linkValidator.js";
+
+// Helper function to build search index from processed files
+function buildSearchIndex(jsonCache, source, output) {
+  const searchIndex = [];
+  
+  for (const [filePath, jsonObject] of jsonCache.entries()) {
+    // Generate URL path relative to output
+    const relativePath = filePath.replace(source, '').replace(/\.(md|txt|yml)$/, '.html');
+    const url = relativePath.startsWith('/') ? relativePath : '/' + relativePath;
+    
+    // Extract text content from body (strip HTML tags for search)
+    const textContent = jsonObject.bodyHtml.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
+    const excerpt = textContent.substring(0, 200); // First 200 chars for preview
+    
+    searchIndex.push({
+      title: toTitleCase(jsonObject.name),
+      path: relativePath,
+      url: url,
+      content: excerpt
+    });
+  }
+  
+  return searchIndex;
+}
+
+// Helper function to convert filename to title case
+function toTitleCase(filename) {
+  return filename
+    .split(/[-_\s]+/) // Split on hyphens, underscores, and spaces
+    .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
+    .join(' ');
+}
 import { renderFile } from "../helper/fileRenderer.js";
+import { findStyleCss } from "../helper/findStyleCss.js";
 import { copy as copyDir, emptyDir, outputFile } from "fs-extra";
 import { basename, dirname, extname, join, parse, resolve } from "path";
 import { URL } from "url";
@@ -16,6 +60,8 @@ import o2x from "object-to-xml";
 import { existsSync } from "fs";
 import { fileExists } from "../helper/fileExists.js";
 
+import { createWhitelistFilter } from "../helper/whitelistFilter.js";
+
 const DEFAULT_TEMPLATE_NAME =
   process.env.DEFAULT_TEMPLATE_NAME ?? "default-template";
 
@@ -23,18 +69,30 @@ export async function generate({
   _source = join(process.cwd(), "."),
   _meta = join(process.cwd(), "meta"),
   _output = join(process.cwd(), "build"),
+  _whitelist = null,
+  _incremental = false,  // Legacy flag, now ignored (always incremental)
+  _clean = false,  // When true, ignore cache and regenerate all files
 } = {}) {
-  console.log({ _source, _meta, _output });
+  console.log({ _source, _meta, _output, _whitelist, _clean });
   const source = resolve(_source) + "/";
   const meta = resolve(_meta);
   const output = resolve(_output) + "/";
   console.log({ source, meta, output });
 
   const allSourceFilenamesUnfiltered = await recurse(source, [() => false]);
+  
+  // Apply include filter (existing functionality)
   const includeFilter = process.env.INCLUDE_FILTER
     ? (fileName) => fileName.match(process.env.INCLUDE_FILTER)
     : Boolean;
-  const allSourceFilenames = allSourceFilenamesUnfiltered.filter(includeFilter);
+  let allSourceFilenames = allSourceFilenamesUnfiltered.filter(includeFilter);
+  
+  // Apply whitelist filter if specified
+  if (_whitelist) {
+    const whitelistFilter = await createWhitelistFilter(_whitelist, source);
+    allSourceFilenames = allSourceFilenames.filter(whitelistFilter);
+    console.log(`Whitelist applied: ${allSourceFilenames.length} files after filtering`);
+  }
   // console.log(allSourceFilenames);
 
   // if (source.substr(-1) !== "/") source += "/"; // warning: might not work in windows
@@ -43,138 +101,263 @@ export async function generate({
   const templates = await getTemplates(meta); // todo: error if no default template
   // console.log({ templates });
 
-  const menu = await getMenu(allSourceFilenames, source);
-
-  // clean build directory
-  await emptyDir(output);
-
-  // create public folder
-  const pub = join(output, "public");
-  await mkdir(pub);
-  await copyDir(meta, pub);
-
   // read all articles, process them, copy them to build
   const articleExtensions = /\.(md|txt|yml)/;
+  const hiddenOrSystemDirs = /[\/\\]\.(?!\.)|[\/\\]node_modules[\/\\]/;  // Matches hidden folders (starting with .) or node_modules
   const allSourceFilenamesThatAreArticles = allSourceFilenames.filter(
-    (filename) => filename.match(articleExtensions)
+    (filename) => filename.match(articleExtensions) && !filename.match(hiddenOrSystemDirs)
   );
-  const allSourceFilenamesThatAreDirectories = await filterAsync(
+  const allSourceFilenamesThatAreDirectories = (await filterAsync(
     allSourceFilenames,
     (filename) => isDirectory(filename)
-  );
+  )).filter((filename) => !filename.match(hiddenOrSystemDirs));
 
-  // process individual articles
-  const jsonCache = new Map();
-  await Promise.all(
-    allSourceFilenamesThatAreArticles.map(async (file) => {
-      console.log(`processing article ${file}`);
+  // Build set of valid internal paths for link validation (must be before menu)
+  const validPaths = buildValidPaths(allSourceFilenamesThatAreArticles, source);
+  console.log(`Built ${validPaths.size} valid paths for link validation`);
+
+  const menu = await getMenu(allSourceFilenames, source, validPaths);
+
+  // Load content hash cache from .ursa folder in source directory
+  let hashCache = new Map();
+  if (!_clean) {
+    hashCache = await loadHashCache(source);
+    console.log(`Loaded ${hashCache.size} cached content hashes from .ursa folder`);
+  } else {
+    console.log(`Clean build: ignoring cached hashes`);
+  }
+
+  // create public folder
+  const pub = join(output, "public");
+  await mkdir(pub, { recursive: true });
+  await copyDir(meta, pub);
 
+  // Track errors for error report
+  const errors = [];
+
+  // First pass: collect search index data
+  const searchIndex = [];
+  const jsonCache = new Map();
+  
+  // Collect basic data for search index
+  for (const file of allSourceFilenamesThatAreArticles) {
+    try {
       const rawBody = await readFile(file, "utf8");
       const type = parse(file).ext;
-      const meta = extractMetadata(rawBody);
-      const rawMeta = extractRawMetadata(rawBody);
-      const bodyLessMeta = rawBody.replace(rawMeta, "");
-      const transformedMetadata = await getTransformedMetadata(
-        dirname(file),
-        meta
-      );
       const ext = extname(file);
       const base = basename(file, ext);
       const dir = addTrailingSlash(dirname(file)).replace(source, "");
+      
+      // Generate title from filename (in title case)
+      const title = toTitleCase(base);
+      
+      // Generate URL path relative to output
+      const relativePath = file.replace(source, '').replace(/\.(md|txt|yml)$/, '.html');
+      const url = relativePath.startsWith('/') ? relativePath : '/' + relativePath;
+      
+      // Basic content processing for search (without full rendering)
       const body = renderFile({
         fileContents: rawBody,
         type,
         dirname: dir,
         basename: base,
       });
+      
+      // Extract text content from body (strip HTML tags for search)
+      const textContent = body && body.replace && body.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim() || 'body is undefined for some reason'
+      const excerpt = textContent.substring(0, 200); // First 200 chars for preview
+      
+      searchIndex.push({
+        title: title,
+        path: relativePath,
+        url: url,
+        content: excerpt
+      });
+    } catch (e) {
+      console.error(`Error processing ${file} (first pass): ${e.message}`);
+      errors.push({ file, phase: 'search-index', error: e });
+    }
+  }
+  
+  console.log(`Built search index with ${searchIndex.length} entries`);
+
+  // Track files that were regenerated (for incremental mode stats)
+  let regeneratedCount = 0;
+  let skippedCount = 0;
 
-      const requestedTemplateName = meta && meta.template;
-      const template =
-        templates[requestedTemplateName] || templates[DEFAULT_TEMPLATE_NAME];
-      // console.log({ requestedTemplateName, templates: templates.keys });
-
-      const finalHtml = template
-        .replace("${menu}", menu)
-        .replace("${meta}", JSON.stringify(meta))
-        .replace("${transformedMetadata}", transformedMetadata)
-        .replace("${body}", body);
-
-      const outputFilename = file
-        .replace(source, output)
-        .replace(parse(file).ext, ".html");
-
-      console.log(`writing article to ${outputFilename}`);
-
-      await outputFile(outputFilename, finalHtml);
-
-      // json
-
-      const jsonOutputFilename = outputFilename.replace(".html", ".json");
-      const jsonObject = {
-        name: base,
-        contents: rawBody,
-        // bodyLessMeta: bodyLessMeta,
-        bodyHtml: body,
-        metadata: meta,
-        transformedMetadata,
-        // html: finalHtml,
-      };
-      jsonCache.set(file, jsonObject);
-      const json = JSON.stringify(jsonObject);
-      console.log(`writing article to ${jsonOutputFilename}`);
-      await outputFile(jsonOutputFilename, json);
-
-      // xml
-
-      const xmlOutputFilename = outputFilename.replace(".html", ".xml");
-      const xml = `<article>${o2x(jsonObject)}</article>`;
-      await outputFile(xmlOutputFilename, xml);
+  // Second pass: process individual articles with search data available
+  await Promise.all(
+    allSourceFilenamesThatAreArticles.map(async (file) => {
+      try {
+        const rawBody = await readFile(file, "utf8");
+        
+        // Skip files that haven't changed (unless --clean flag is set)
+        if (!_clean && !needsRegeneration(file, rawBody, hashCache)) {
+          skippedCount++;
+          return; // Skip this file
+        }
+        
+        console.log(`processing article ${file}`);
+        regeneratedCount++;
+
+        const type = parse(file).ext;
+        const meta = extractMetadata(rawBody);
+        const rawMeta = extractRawMetadata(rawBody);
+        const bodyLessMeta = rawMeta ? rawBody.replace(rawMeta, "") : rawBody;
+        const transformedMetadata = await getTransformedMetadata(
+          dirname(file),
+          meta
+        );
+        const ext = extname(file);
+        const base = basename(file, ext);
+        const dir = addTrailingSlash(dirname(file)).replace(source, "");
+        
+        // Calculate the document's URL path (e.g., "/character/index.html")
+        const docUrlPath = '/' + dir + base + '.html';
+        
+        // Generate title from filename (in title case)
+        const title = toTitleCase(base);
+
+        const body = renderFile({
+          fileContents: rawBody,
+          type,
+          dirname: dir,
+          basename: base,
+        });
+
+        // Find nearest style.css or _style.css up the tree
+        let embeddedStyle = "";
+        try {
+          const css = await findStyleCss(resolve(_source, dir));
+          if (css) {
+            embeddedStyle = css;
+          }
+        } catch (e) {
+          // ignore
+          console.error(e);
+        }
+
+        const requestedTemplateName = meta && meta.template;
+        const template =
+          templates[requestedTemplateName] || templates[DEFAULT_TEMPLATE_NAME];
+
+        if (!template) {
+          throw new Error(`Template not found. Requested: "${requestedTemplateName || DEFAULT_TEMPLATE_NAME}". Available templates: ${Object.keys(templates).join(', ') || 'none'}`);
+        }
+
+        // Insert embeddedStyle just before </head> if present, else at top
+        let finalHtml = template
+          .replace("${title}", title)
+          .replace("${menu}", menu)
+          .replace("${meta}", JSON.stringify(meta))
+          .replace("${transformedMetadata}", transformedMetadata)
+          .replace("${body}", body)
+          .replace("${embeddedStyle}", embeddedStyle)
+          .replace("${searchIndex}", JSON.stringify(searchIndex));
+
+        // Resolve links and mark broken internal links as inactive (debug mode on)
+        // Pass docUrlPath so relative links can be resolved correctly
+        finalHtml = markInactiveLinks(finalHtml, validPaths, docUrlPath, false);
+
+        const outputFilename = file
+          .replace(source, output)
+          .replace(parse(file).ext, ".html");
+
+        console.log(`writing article to ${outputFilename}`);
+
+        await outputFile(outputFilename, finalHtml);
+
+        // json
+
+        const jsonOutputFilename = outputFilename.replace(".html", ".json");
+        const url = '/' + outputFilename.replace(output, '');
+        const jsonObject = {
+          name: base,
+          url,
+          contents: rawBody,
+          // bodyLessMeta: bodyLessMeta,
+          bodyHtml: body,
+          metadata: meta,
+          transformedMetadata,
+          // html: finalHtml,
+        };
+        jsonCache.set(file, jsonObject);
+        const json = JSON.stringify(jsonObject);
+        console.log(`writing article to ${jsonOutputFilename}`);
+        await outputFile(jsonOutputFilename, json);
+
+        // xml
+
+        const xmlOutputFilename = outputFilename.replace(".html", ".xml");
+        const xml = `<article>${o2x(jsonObject)}</article>`;
+        await outputFile(xmlOutputFilename, xml);
+        
+        // Update the content hash for this file
+        updateHash(file, rawBody, hashCache);
+      } catch (e) {
+        console.error(`Error processing ${file} (second pass): ${e.message}`);
+        errors.push({ file, phase: 'article-generation', error: e });
+      }
     })
   );
 
+  // Log build stats
+  console.log(`Build: ${regeneratedCount} regenerated, ${skippedCount} unchanged`);
+
   console.log(jsonCache.keys());
+  
   // process directory indices
   await Promise.all(
     allSourceFilenamesThatAreDirectories.map(async (dir) => {
-      console.log(`processing directory ${dir}`);
-
-      const pathsInThisDirectory = allSourceFilenames.filter((filename) =>
-        filename.match(new RegExp(`${dir}.+`))
-      );
-
-      const jsonObjects = pathsInThisDirectory
-        .map((path) => {
-          const object = jsonCache.get(path);
-          return typeof object === "object" ? object : null;
-        })
-        .filter((a) => a);
+      try {
+        console.log(`processing directory ${dir}`);
 
-      const json = JSON.stringify(jsonObjects);
+        const pathsInThisDirectory = allSourceFilenames.filter((filename) =>
+          filename.match(new RegExp(`${dir}.+`))
+        );
 
-      const outputFilename = dir.replace(source, output) + ".json";
-
-      console.log(`writing directory index to ${outputFilename}`);
-      await outputFile(outputFilename, json);
-
-      // html
-      const htmlOutputFilename = dir.replace(source, output) + ".html";
-      const indexAlreadyExists = fileExists(htmlOutputFilename);
-      if (!indexAlreadyExists) {
-        const template = templates["default-template"]; // TODO: figure out a way to specify template for a directory index
-        const indexHtml = `<ul>${pathsInThisDirectory
+        const jsonObjects = pathsInThisDirectory
           .map((path) => {
-            const partialPath = path
-              .replace(source, "")
-              .replace(parse(path).ext, ".html");
-            const name = basename(path, parse(path).ext);
-            return `<li><a href="${partialPath}">${name}</a></li>`;
+            const object = jsonCache.get(path);
+            return typeof object === "object" ? object : null;
           })
-          .join("")}</ul>`;
-        const finalHtml = template
-          .replace("${menu}", menu)
-          .replace("${body}", indexHtml);
-        console.log(`writing directory index to ${htmlOutputFilename}`);
-        await outputFile(htmlOutputFilename, finalHtml);
+          .filter((a) => a);
+
+        const json = JSON.stringify(jsonObjects);
+
+        const outputFilename = dir.replace(source, output) + ".json";
+
+        console.log(`writing directory index to ${outputFilename}`);
+        await outputFile(outputFilename, json);
+
+        // html
+        const htmlOutputFilename = dir.replace(source, output) + ".html";
+        const indexAlreadyExists = fileExists(htmlOutputFilename);
+        if (!indexAlreadyExists) {
+          const template = templates["default-template"]; // TODO: figure out a way to specify template for a directory index
+          const indexHtml = `<ul>${pathsInThisDirectory
+            .map((path) => {
+              const partialPath = path
+                .replace(source, "")
+                .replace(parse(path).ext, ".html");
+              const name = basename(path, parse(path).ext);
+              return `<li><a href="${partialPath}">${name}</a></li>`;
+            })
+            .join("")}</ul>`;
+          const finalHtml = template
+            .replace("${menu}", menu)
+            .replace("${body}", indexHtml)
+            .replace("${searchIndex}", JSON.stringify(searchIndex))
+            .replace("${title}", "Index")
+            .replace("${meta}", "{}")
+            .replace("${transformedMetadata}", "")
+            .replace("${embeddedStyle}", "");
+          console.log(`writing directory index to ${htmlOutputFilename}`);
+          await outputFile(htmlOutputFilename, finalHtml);
+        }
+      } catch (e) {
+        console.error(`Error processing directory ${dir}: ${e.message}`);
+        errors.push({ file: dir, phase: 'directory-index', error: e });
       }
     })
   );
@@ -186,15 +369,73 @@ export async function generate({
   );
   await Promise.all(
     allSourceFilenamesThatAreImages.map(async (file) => {
-      console.log(`processing static file ${file}`);
-
-      const outputFilename = file.replace(source, output);
-
-      console.log(`writing static file to ${outputFilename}`);
-
-      return await copyFile(file, outputFilename);
+      try {
+        // For incremental mode, check if file has changed using file stat as a quick check
+        if (_incremental) {
+          const fileStat = await stat(file);
+          const statKey = `${file}:stat`;
+          const newStatHash = `${fileStat.size}:${fileStat.mtimeMs}`;
+          if (hashCache.get(statKey) === newStatHash) {
+            return; // Skip unchanged static file
+          }
+          hashCache.set(statKey, newStatHash);
+        }
+        
+        console.log(`processing static file ${file}`);
+
+        const outputFilename = file.replace(source, output);
+
+        console.log(`writing static file to ${outputFilename}`);
+
+        await mkdir(dirname(outputFilename), { recursive: true });
+        return await copyFile(file, outputFilename);
+      } catch (e) {
+        console.error(`Error processing static file ${file}: ${e.message}`);
+        errors.push({ file, phase: 'static-file', error: e });
+      }
     })
   );
+
+  // Save the hash cache to .ursa folder in source directory
+  if (hashCache.size > 0) {
+    await saveHashCache(source, hashCache);
+  }
+
+  // Write error report if there were any errors
+  if (errors.length > 0) {
+    const errorReportPath = join(output, '_errors.log');
+    const failedFiles = errors.map(e => e.file);
+    
+    let report = `URSA GENERATION ERROR REPORT\n`;
+    report += `Generated: ${new Date().toISOString()}\n`;
+    report += `Total errors: ${errors.length}\n\n`;
+    report += `${'='.repeat(60)}\n`;
+    report += `FAILED FILES:\n`;
+    report += `${'='.repeat(60)}\n\n`;
+    failedFiles.forEach(f => {
+      report += `  - ${f}\n`;
+    });
+    report += `\n${'='.repeat(60)}\n`;
+    report += `ERROR DETAILS:\n`;
+    report += `${'='.repeat(60)}\n\n`;
+    
+    errors.forEach(({ file, phase, error }) => {
+      report += `${'─'.repeat(60)}\n`;
+      report += `File: ${file}\n`;
+      report += `Phase: ${phase}\n`;
+      report += `Error: ${error.message}\n`;
+      if (error.stack) {
+        report += `Stack:\n${error.stack}\n`;
+      }
+      report += `\n`;
+    });
+    
+    await outputFile(errorReportPath, report);
+    console.log(`\n⚠️  ${errors.length} error(s) occurred during generation.`);
+    console.log(`   Error report written to: ${errorReportPath}\n`);
+  } else {
+    console.log(`\n✅ Generation complete with no errors.\n`);
+  }
 }
 
 /**
@@ -222,10 +463,10 @@ async function getTemplates(meta) {
   return templates;
 }
 
-async function getMenu(allSourceFilenames, source) {
+async function getMenu(allSourceFilenames, source, validPaths) {
   // todo: handle various incarnations of menu filename
 
-  const rawMenu = await getAutomenu(source);
+  const rawMenu = await getAutomenu(source, validPaths);
   const menuBody = renderFile({ fileContents: rawMenu, type: ".md" });
   return menuBody;