boltdocs 1.10.2 → 2.0.0

package/src/node/routes/index.ts

@@ -1,179 +1,272 @@
-import fastGlob from "fast-glob";
-import { BoltdocsConfig } from "../config";
-import { capitalize } from "../utils";
+import fastGlob from 'fast-glob'
+import type { BoltdocsConfig } from '../config'
+import { capitalize } from '../utils'
 
-import { RouteMeta, ParsedDocFile } from "./types";
-import { docCache, invalidateRouteCache, invalidateFile } from "./cache";
-import { parseDocFile } from "./parser";
-import { sortRoutes } from "./sorter";
+import type { RouteMeta, ParsedDocFile } from './types'
+import { docCache, invalidateRouteCache, invalidateFile } from './cache'
+import { parseDocFile } from './parser'
+import { sortRoutes } from './sorter'
 
 // Re-export public API
-export type { RouteMeta };
-export { invalidateRouteCache, invalidateFile };
+export type { RouteMeta }
+export { invalidateRouteCache, invalidateFile }
+
+// Cache for localized path computations
+const localizedPathCache = new Map<string, string>()
 
 /**
  * Generates the entire route map for the documentation site.
- * This reads all `.md` and `.mdx` files in the `docsDir`, parses them (using cache),
- * infers group hierarchies based on directory structure and `index.md` files,
- * and returns a sorted array of RouteMeta objects intended for the client.
+ * OPTIMIZED: Uses Map-based i18n lookups, chunked processing, and path caching.
+ *
+ * Automatically handles versioning and i18n routing, including fallback
+ * generation for missing translations.
  *
- * @param docsDir - The root directory containing markdown files
- * @param config - Optional configuration for i18n and versioning
- * @param basePath - The base URL path to prefix to generated routes (e.g., '/docs')
- * @returns A promise that resolves to the final list of RouteMeta objects
+ * @param docsDir - The root documentation directory
+ * @param config - The Boltdocs configuration
+ * @param basePath - The base URL path for the routes (default: '/docs')
+ * @returns A promise resolving to an array of RouteMeta objects
  */
 export async function generateRoutes(
   docsDir: string,
   config?: BoltdocsConfig,
-  basePath: string = "/docs",
+  basePath: string = '/docs',
 ): Promise<RouteMeta[]> {
-  // Load persistent cache on first call
-  docCache.load();
-  docCache.invalidateAll(); // FORCE RE-PARSE to pick up new _content field
+  const start = performance.now()
+
+  // Load persistent cache
+  docCache.load()
+
+  // Clear path computation cache between generations
+  localizedPathCache.clear()
+
+  // Force re-parse if specifically requested (e.g. for content/config changes)
+  if (process.env.BOLTDOCS_FORCE_REPARSE === 'true' || config?.i18n) {
+    docCache.invalidateAll()
+  }
 
-  const files = await fastGlob(["**/*.md", "**/*.mdx"], {
+  // 1. FAST SCAN
+  const files = await fastGlob(['**/*.md', '**/*.mdx'], {
     cwd: docsDir,
     absolute: true,
-  });
+    suppressErrors: true,
+    followSymbolicLinks: false,
+  })
 
   // Prune cache entries for deleted files
-  docCache.pruneStale(new Set(files));
+  docCache.pruneStale(new Set(files))
 
-  // Invalidate all caches if config changes drastically (e.g. i18n enabled)
-  if (config?.i18n) {
-    docCache.invalidateAll();
-  }
+  // 2. CHUNKED PROCESSING (prevents blocking event loop)
+  const CHUNK_SIZE = 50
+  const parsed: ParsedDocFile[] = []
+  let cacheHits = 0
 
-  // Parse files in parallel using Promise.all for increased efficiency
-  let cacheHits = 0;
-  const parsed: ParsedDocFile[] = await Promise.all(
-    files.map(async (file) => {
-      const cached = docCache.get(file);
-      if (cached) {
-        cacheHits++;
-        return cached;
-      }
+  for (let i = 0; i < files.length; i += CHUNK_SIZE) {
+    const chunk = files.slice(i, i + CHUNK_SIZE)
+
+    const chunkResults = await Promise.all(
+      chunk.map(async (file) => {
+        const cached = docCache.get(file)
+        if (cached) {
+          cacheHits++
+          return cached
+        }
+
+        const result = parseDocFile(file, docsDir, basePath, config)
+        docCache.set(file, result)
+        return result
+      }),
+    )
 
-      const result = parseDocFile(file, docsDir, basePath, config);
-      docCache.set(file, result);
-      return result;
-    }),
-  );
+    parsed.push(...chunkResults)
 
-  if (files.length > 0) {
-    console.log(
-      `[boltdocs] Routes generated: ${files.length} files (${cacheHits} from cache, ${files.length - cacheHits} parsed)`,
-    );
+    // Yield to event loop between chunks if there's more to process
+    if (i + CHUNK_SIZE < files.length) {
+      await new Promise((resolve) => setImmediate(resolve))
+    }
   }
 
-  // Save cache after batch processing
-  docCache.save();
+  // Save cache after processing
+  docCache.save()
 
-  // Collect group metadata from directory names and index files
+  // 3. OPTIMIZED METADATA COLLECTION
   const groupMeta = new Map<
     string,
     { title: string; position?: number; icon?: string }
-  >();
+  >()
+  const groupIndexFiles: ParsedDocFile[] = []
+
   for (const p of parsed) {
+    if (p.isGroupIndex && p.relativeDir) {
+      groupIndexFiles.push(p)
+    }
+
     if (p.relativeDir) {
-      if (!groupMeta.has(p.relativeDir)) {
-        groupMeta.set(p.relativeDir, {
+      let entry = groupMeta.get(p.relativeDir)
+      if (!entry) {
+        entry = {
           title: capitalize(p.relativeDir),
           position: p.inferredGroupPosition,
           icon: p.route.icon,
-        });
+        }
+        groupMeta.set(p.relativeDir, entry)
       } else {
-        const entry = groupMeta.get(p.relativeDir)!;
         if (
           entry.position === undefined &&
           p.inferredGroupPosition !== undefined
         ) {
-          entry.position = p.inferredGroupPosition;
+          entry.position = p.inferredGroupPosition
         }
         if (!entry.icon && p.route.icon) {
-          entry.icon = p.route.icon;
+          entry.icon = p.route.icon
         }
       }
     }
+  }
 
-    if (p.isGroupIndex && p.relativeDir && p.groupMeta) {
-      const entry = groupMeta.get(p.relativeDir)!;
-      entry.title = p.groupMeta.title;
-      if (p.groupMeta.position !== undefined) {
-        entry.position = p.groupMeta.position;
-      }
-      if (p.groupMeta.icon) {
-        entry.icon = p.groupMeta.icon;
-      }
+  // Override with explicit group index metadata
+  for (const p of groupIndexFiles) {
+    const entry = groupMeta.get(p.relativeDir!)!
+    if (p.groupMeta) {
+      entry.title = p.groupMeta.title
+      if (p.groupMeta.position !== undefined)
+        entry.position = p.groupMeta.position
+      if (p.groupMeta.icon) entry.icon = p.groupMeta.icon
     }
   }
 
-  // Build final routes with group info
-  const routes: RouteMeta[] = parsed.map((p) => {
-    const dir = p.relativeDir;
-    const meta = dir ? groupMeta.get(dir) : undefined;
+  // 4. BUILD BASE ROUTES
+  const routes: RouteMeta[] = new Array(parsed.length)
+  for (let i = 0; i < parsed.length; i++) {
+    const p = parsed[i]
+    const dir = p.relativeDir
+    const meta = dir ? groupMeta.get(dir) : undefined
 
-    return {
+    routes[i] = {
       ...p.route,
       group: dir,
       groupTitle: meta?.title || (dir ? capitalize(dir) : undefined),
       groupPosition: meta?.position,
       groupIcon: meta?.icon,
-    };
-  });
+    }
+  }
 
-  // Add fallbacks if i18n is enabled
+  // 5. OPTIMIZED I18N FALLBACKS
+  let finalRoutes = routes
   if (config?.i18n) {
-    const defaultLocale = config.i18n.defaultLocale;
-    const allLocales = Object.keys(config.i18n.locales);
+    const fallbacks = generateI18nFallbacks(routes, config, basePath)
+    finalRoutes = [...routes, ...fallbacks]
+  }
+
+  const sorted = sortRoutes(finalRoutes)
 
-    const fallbackRoutes: RouteMeta[] = [];
-    const defaultRoutes = routes.filter(
-      (r) => (r.locale || defaultLocale) === defaultLocale,
-    );
+  const duration = performance.now() - start
+  console.log(
+    `[boltdocs] Route generation: ${duration.toFixed(2)}ms (${files.length} files, ${cacheHits} cache hits)`,
+  )
 
-    for (const locale of allLocales) {
-      if (locale === defaultLocale) continue;
+  return sorted
+}
 
-      const localeRoutePaths = new Set(
-        routes.filter((r) => r.locale === locale).map((r) => r.path),
-      );
+/**
+ * Generates fallback routes for missing translations.
+ * Optimization: Uses Map for O(1) existence checks instead of nested filters.
+ */
+function generateI18nFallbacks(
+  routes: RouteMeta[],
+  config: BoltdocsConfig,
+  basePath: string,
+): RouteMeta[] {
+  const defaultLocale = config.i18n!.defaultLocale
+  const allLocales = Object.keys(config.i18n!.locales)
+  const fallbackRoutes: RouteMeta[] = []
 
-      for (const defRoute of defaultRoutes) {
-        let prefix = basePath;
-        if (defRoute.version) {
-          prefix += "/" + defRoute.version;
-        }
+  // Index existing routes by locale for O(1) lookup
+  const routesByLocale = new Map<string, Set<string>>()
+  const defaultRoutes: RouteMeta[] = []
 
-        let pathAfterVersion = defRoute.path.substring(prefix.length);
+  for (const r of routes) {
+    const locale = r.locale || defaultLocale
+    if (!routesByLocale.has(locale)) {
+      routesByLocale.set(locale, new Set())
+    }
+    routesByLocale.get(locale)!.add(r.path)
 
-        if (pathAfterVersion.startsWith("/" + defaultLocale + "/")) {
-          pathAfterVersion = pathAfterVersion.substring(
-            defaultLocale.length + 1,
-          );
-        } else if (pathAfterVersion === "/" + defaultLocale) {
-          pathAfterVersion = "/";
-        }
-        const targetPath =
-          prefix +
-          "/" +
-          locale +
-          (pathAfterVersion === "/" || pathAfterVersion === ""
-            ? ""
-            : pathAfterVersion);
-
-        if (!localeRoutePaths.has(targetPath)) {
-          fallbackRoutes.push({
-            ...defRoute,
-            path: targetPath,
-            locale: locale,
-          });
-        }
+    if (locale === defaultLocale) {
+      defaultRoutes.push(r)
+    }
+  }
+
+  for (const locale of allLocales) {
+    if (locale === defaultLocale) continue
+
+    const localePaths = routesByLocale.get(locale) || new Set<string>()
+
+    for (const defRoute of defaultRoutes) {
+      const targetPath = computeLocalizedPath(
+        defRoute.path,
+        defaultLocale,
+        locale,
+        basePath,
+      )
+
+      if (!localePaths.has(targetPath)) {
+        fallbackRoutes.push({
+          ...defRoute,
+          path: targetPath,
+          locale,
+        })
       }
     }
+  }
+
+  return fallbackRoutes
+}
+
+/**
+ * Computes a localized path based on the default locale and target locale.
+ * Uses a cache to avoid redundant string manipulation.
+ */
+function computeLocalizedPath(
+  path: string,
+  defaultLocale: string,
+  targetLocale: string,
+  basePath: string,
+): string {
+  const cacheKey = `${path}:${targetLocale}`
+  const cached = localizedPathCache.get(cacheKey)
+  if (cached) return cached
+
+  let prefix = basePath
+  const versionMatch = path.match(new RegExp(`^${basePath}/(v[0-9]+)`))
+  if (versionMatch) {
+    prefix += '/' + versionMatch[1]
+  }
 
-    return sortRoutes([...routes, ...fallbackRoutes]);
+  let pathAfterVersion = path.substring(prefix.length)
+
+  // Handle case where path already has default locale
+  const defaultLocaleSegment = `/${defaultLocale}`
+  if (pathAfterVersion.startsWith(defaultLocaleSegment + '/')) {
+    pathAfterVersion =
+      '/' +
+      targetLocale +
+      '/' +
+      pathAfterVersion.substring(defaultLocaleSegment.length + 1)
+  } else if (pathAfterVersion === defaultLocaleSegment) {
+    pathAfterVersion = '/' + targetLocale
+  } else if (pathAfterVersion === '/' || pathAfterVersion === '') {
+    pathAfterVersion = '/' + targetLocale
+  } else {
+    // Ensure pathAfterVersion starts with a slash if not already
+    const pathPrefix = pathAfterVersion.startsWith('/') ? '' : '/'
+    pathAfterVersion = '/' + targetLocale + pathPrefix + pathAfterVersion
   }
 
-  return sortRoutes(routes);
+  const result = prefix + pathAfterVersion
+
+  // Simple cache eviction to prevent memory leaks in extreme cases
+  if (localizedPathCache.size > 2000) localizedPathCache.clear()
+  localizedPathCache.set(cacheKey, result)
+
+  return result
 }

package/src/node/routes/parser.ts

@@ -1,7 +1,7 @@
-import path from "path";
-import GithubSlugger from "github-slugger";
-import { BoltdocsConfig } from "../config";
-import { ParsedDocFile } from "./types";
+import path from 'path'
+import GithubSlugger from 'github-slugger'
+import type { BoltdocsConfig } from '../config'
+import type { ParsedDocFile } from './types'
 import {
   normalizePath,
   parseFrontmatter,
@@ -9,16 +9,21 @@ import {
   capitalize,
   stripNumberPrefix,
   extractNumberPrefix,
-  escapeHtml,
-} from "../utils";
+  sanitizeHtml,
+  stripHtmlTags,
+} from '../utils'
 
 /**
  * Parses a single Markdown/MDX file and extracts its metadata for routing.
  * Checks frontmatter for explicit titles, descriptions, and sidebar positions.
  *
+ * Also performs security validation to prevent path traversal and basic
+ * XSS sanitization for metadata and headings.
+ *
  * @param file - The absolute path to the file
  * @param docsDir - The root documentation directory (e.g., 'docs')
  * @param basePath - The base URL path for the routes (default: '/docs')
+ * @param config - The Boltdocs configuration for versions and i18n
  * @returns A parsed structure ready for route assembly and caching
  */
 export function parseDocFile(
@@ -28,138 +33,138 @@ export function parseDocFile(
   config?: BoltdocsConfig,
 ): ParsedDocFile {
   // Security: Prevent path traversal
-  const decodedFile = decodeURIComponent(file);
-  const absoluteFile = path.resolve(decodedFile);
-  const absoluteDocsDir = path.resolve(docsDir);
+  const decodedFile = decodeURIComponent(file)
+  const absoluteFile = path.resolve(decodedFile)
+  const absoluteDocsDir = path.resolve(docsDir)
   const relativePath = normalizePath(
     path.relative(absoluteDocsDir, absoluteFile),
-  );
+  )
 
   if (
-    relativePath.startsWith("../") ||
-    relativePath === ".." ||
-    absoluteFile.includes("\0")
+    relativePath.startsWith('../') ||
+    relativePath === '..' ||
+    absoluteFile.includes('\0')
   ) {
     throw new Error(
       `Security breach: File is outside of docs directory or contains null bytes: ${file}`,
-    );
+    )
   }
 
-  const { data, content } = parseFrontmatter(file);
-  let parts = relativePath.split("/");
+  const { data, content } = parseFrontmatter(file)
+  let parts = relativePath.split('/')
 
-  let locale: string | undefined;
-  let version: string | undefined;
+  let locale: string | undefined
+  let version: string | undefined
 
   // Level 1: Check for version
   if (config?.versions && parts.length > 0) {
-    const potentialVersion = parts[0];
+    const potentialVersion = parts[0]
     if (config.versions.versions[potentialVersion]) {
-      version = potentialVersion;
-      parts = parts.slice(1);
+      version = potentialVersion
+      parts = parts.slice(1)
     }
   }
 
   // Level 2: Check for locale
   if (config?.i18n && parts.length > 0) {
-    const potentialLocale = parts[0];
+    const potentialLocale = parts[0]
     if (config.i18n.locales[potentialLocale]) {
-      locale = potentialLocale;
-      parts = parts.slice(1);
+      locale = potentialLocale
+      parts = parts.slice(1)
     }
   }
 
   // Level 3: Check for Tab hierarchy (name)
-  let inferredTab: string | undefined;
+  let inferredTab: string | undefined
   if (parts.length > 0) {
-    const tabMatch = parts[0].match(/^\((.+)\)$/);
+    const tabMatch = parts[0].match(/^\((.+)\)$/)
     if (tabMatch) {
-      inferredTab = tabMatch[1].toLowerCase();
-      parts = parts.slice(1);
+      inferredTab = tabMatch[1].toLowerCase()
+      parts = parts.slice(1)
     }
   }
 
-  const cleanRelativePath = parts.join("/");
+  const cleanRelativePath = parts.join('/')
 
-  let cleanRoutePath: string;
+  let cleanRoutePath: string
   if (data.permalink) {
     // If a permalink is specified, ensure it starts with a slash
-    cleanRoutePath = data.permalink.startsWith("/")
+    cleanRoutePath = data.permalink.startsWith('/')
       ? data.permalink
-      : `/${data.permalink}`;
+      : `/${data.permalink}`
   } else {
-    cleanRoutePath = fileToRoutePath(cleanRelativePath || "index.md");
+    cleanRoutePath = fileToRoutePath(cleanRelativePath || 'index.md')
   }
 
-  let finalPath = basePath;
+  let finalPath = basePath
   if (version) {
-    finalPath += "/" + version;
+    finalPath += '/' + version
   }
   if (locale) {
-    finalPath += "/" + locale;
+    finalPath += '/' + locale
   }
-  finalPath += cleanRoutePath === "/" ? "" : cleanRoutePath;
+  finalPath += cleanRoutePath === '/' ? '' : cleanRoutePath
 
-  if (!finalPath || finalPath === "") finalPath = "/";
+  if (!finalPath || finalPath === '') finalPath = '/'
 
-  const rawFileName = parts[parts.length - 1];
-  const cleanFileName = stripNumberPrefix(rawFileName);
+  const rawFileName = parts[parts.length - 1]
+  const cleanFileName = stripNumberPrefix(rawFileName)
   const inferredTitle = stripNumberPrefix(
     path.basename(file, path.extname(file)),
-  );
+  )
   const sidebarPosition =
-    data.sidebarPosition ?? extractNumberPrefix(rawFileName);
+    data.sidebarPosition ?? extractNumberPrefix(rawFileName)
+
+  const rawDirName = parts.length >= 2 ? parts[0] : undefined
+  const cleanDirName = rawDirName ? stripNumberPrefix(rawDirName) : undefined
 
-  const rawDirName = parts.length >= 2 ? parts[0] : undefined;
-  const cleanDirName = rawDirName ? stripNumberPrefix(rawDirName) : undefined;
+  const isGroupIndex = parts.length >= 2 && /^index\.mdx?$/.test(cleanFileName)
 
-  const isGroupIndex = parts.length >= 2 && /^index\.mdx?$/.test(cleanFileName);
+  const slugger = new GithubSlugger()
+  const headings: { level: number; text: string; id: string }[] = []
+  const headingsRegex = /^(#{2,4})\s+(.+)$/gm
 
-  const headings: { level: number; text: string; id: string }[] = [];
-  const slugger = new GithubSlugger();
-  const headingsRegex = /^(#{2,4})\s+(.+)$/gm;
-  let match;
+  let match
   while ((match = headingsRegex.exec(content)) !== null) {
-    const level = match[1].length;
-    // Strip simple markdown formatting specifically for the plain-text search index
-    const text = match[2]
-      .replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1")
-      .replace(/[_*`]/g, "")
-      .trim();
-    const id = slugger.slug(text);
-    // Security: Sanitize heading text for XSS
-    headings.push({ level, text, id });
+    const level = match[1].length
+    const rawText = match[2]
+      .replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1') // Strip markdown links
+      .replace(/[_*`]/g, '') // Strip markdown formatting
+      .trim()
+
+    const sanitizedText = sanitizeHtml(rawText).trim()
+    const id = slugger.slug(sanitizedText)
+
+    headings.push({ level, text: sanitizedText, id })
   }
 
-  const sanitizedTitle = data.title ? data.title : inferredTitle;
+  const sanitizedTitle = data.title
+    ? sanitizeHtml(String(data.title))
+    : inferredTitle
   let sanitizedDescription = data.description
-    ? data.description
-    : "";
+    ? sanitizeHtml(String(data.description))
+    : ''
 
   // If no description is provided, extract a summary from the content
   if (!sanitizedDescription && content) {
-    const summary = content
-      .replace(/^#+.*$/gm, "") // Remove headers
-      .replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1") // Simplify links
-      .replace(/[_*`]/g, "") // Remove formatting
-      .replace(/\n+/g, " ") // Normalize whitespace
+    const plainExcerpt = stripHtmlTags(
+      content
+        .replace(/^#+.*$/gm, '') // Remove headers
+        .replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1') // Simplify links
+        .replace(/[_*`]/g, '') // Remove formatting
+        .replace(/\s+/g, ' '), // Normalize whitespace
+    )
       .trim()
-      .slice(0, 160);
-    sanitizedDescription = summary;
+      .slice(0, 160)
+
+    sanitizedDescription = plainExcerpt
   }
 
-  const sanitizedBadge = data.badge ? data.badge : undefined;
-  const icon = data.icon ? String(data.icon) : undefined;
+  const sanitizedBadge = data.badge ? sanitizeHtml(String(data.badge)) : undefined
+  const icon = data.icon ? String(data.icon) : undefined
 
   // Extract full content as plain text for search indexing
-  const plainText = content
-    .replace(/^#+.*$/gm, "") // Remove headers
-    .replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1") // Simplify links
-    .replace(/<[^>]+>/g, "") // Remove HTML/JSX tags
-    .replace(/\{[^\}]+\}/g, "") // Remove JS expressions/curly braces
-    .replace(/[_*`]/g, "") // Remove formatting
-    .replace(/\n+/g, " ") // Normalize whitespace
-    .trim();
+  const plainText = parseContentToPlainText(content)
 
   return {
     route: {
@@ -183,10 +188,10 @@ export function parseDocFile(
     inferredTab,
     groupMeta: isGroupIndex
       ? {
-          title: 
+          title:
             data.groupTitle ||
-              data.title ||
-              (cleanDirName ? capitalize(cleanDirName) : ""),
+            data.title ||
+            (cleanDirName ? capitalize(cleanDirName) : ''),
           position:
             data.groupPosition ??
             data.sidebarPosition ??
@@ -197,5 +202,20 @@ export function parseDocFile(
     inferredGroupPosition: rawDirName
       ? extractNumberPrefix(rawDirName)
       : undefined,
-  };
+  }
+}
+
+/**
+ * Converts markdown content to plain text for search indexing.
+ * Strips headers, links, tags, and formatting.
+ */
+function parseContentToPlainText(content: string): string {
+  const plainText = content
+    .replace(/^#+.*$/gm, '') // Remove headers
+    .replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1') // Simplify links
+    .replace(/\{[^\}]+\}/g, '') // Remove JS expressions/curly braces
+    .replace(/[_*`]/g, '') // Remove formatting
+    .replace(/\s+/g, ' ') // Normalize whitespace
+
+  return stripHtmlTags(plainText).trim()
 }