boltdocs 1.0.4 → 1.3.0

package/src/node/cache.ts

@@ -1,43 +1,167 @@
+import fs from "fs";
+import path from "path";
+import crypto from "crypto";
+import zlib from "zlib";
+import { promisify } from "util";
 import { getFileMtime } from "./utils";
 
+const writeFile = promisify(fs.writeFile);
+const readFile = promisify(fs.readFile);
+const mkdir = promisify(fs.mkdir);
+const rename = promisify(fs.rename);
+const unlink = promisify(fs.unlink);
+
+/**
+ * Configuration constants for the caching system.
+ */
+const CACHE_DIR = process.env.BOLTDOCS_CACHE_DIR || ".boltdocs";
+const ASSETS_DIR = "assets";
+const SHARDS_DIR = "shards";
+
+/**
+ * Default limits for the caching system.
+ */
+const DEFAULT_LRU_LIMIT = parseInt(
+  process.env.BOLTDOCS_CACHE_LRU_LIMIT || "2000",
+  10,
+);
+const DEFAULT_COMPRESS = process.env.BOLTDOCS_CACHE_COMPRESS !== "0";
+
+/**
+ * Simple LRU cache implementation to prevent memory leaks.
+ */
+class LRUCache<K, V> {
+  private cache = new Map<K, V>();
+  constructor(private limit: number) {}
+
+  get(key: K): V | undefined {
+    const val = this.cache.get(key);
+    if (val !== undefined) {
+      this.cache.delete(key);
+      this.cache.set(key, val);
+    }
+    return val;
+  }
+
+  set(key: K, value: V): void {
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    } else if (this.cache.size >= this.limit) {
+      const firstKey = this.cache.keys().next().value;
+      if (firstKey !== undefined) {
+        this.cache.delete(firstKey);
+      }
+    }
+    this.cache.set(key, value);
+  }
+
+  get size() {
+    return this.cache.size;
+  }
+  clear() {
+    this.cache.clear();
+  }
+}
+
 /**
- * Per-file cache entry. Stores parsed data + the mtime at parse time.
+ * Simple background task queue to prevent blocking the main thread during IO.
  */
-interface FileCacheEntry<T> {
-  data: T;
-  mtime: number;
+class BackgroundQueue {
+  private queue: Promise<any> = Promise.resolve();
+  private pendingCount = 0;
+
+  add(task: () => Promise<any>) {
+    this.pendingCount++;
+    this.queue = this.queue.then(task).finally(() => {
+      this.pendingCount--;
+    });
+  }
+
+  async flush() {
+    await this.queue;
+  }
+
+  get pending() {
+    return this.pendingCount;
+  }
 }
 
+const backgroundQueue = new BackgroundQueue();
+
 /**
- * Generic file-based cache with per-file granularity.
- * Only re-parses files whose mtime has changed.
+ * Generic file-based cache with per-file granularity and asynchronous persistence.
  */
 export class FileCache<T> {
-  private entries = new Map<string, FileCacheEntry<T>>();
+  private entries = new Map<string, { data: T; mtime: number }>();
+  private readonly cachePath: string | null = null;
+  private readonly compress: boolean;
+
+  constructor(
+    options: { name?: string; root?: string; compress?: boolean } = {},
+  ) {
+    this.compress =
+      options.compress !== undefined ? options.compress : DEFAULT_COMPRESS;
+    if (options.name) {
+      const root = options.root || process.cwd();
+      const ext = this.compress ? "json.gz" : "json";
+      this.cachePath = path.resolve(root, CACHE_DIR, `${options.name}.${ext}`);
+    }
+  }
 
   /**
-   * Retrieves parsed data for a file from the cache.
-   * Compares the current filesystem mtime with the cached mtime.
-   *
-   * @param filePath - The absolute path of the file
-   * @returns The cached data if valid, or `null` if the file has changed or doesn't exist
+   * Loads the cache. Synchronous for startup simplicity but uses fast I/O.
    */
+  load(): void {
+    if (process.env.BOLTDOCS_NO_CACHE === "1") return;
+    if (!this.cachePath || !fs.existsSync(this.cachePath)) return;
+
+    try {
+      let raw = fs.readFileSync(this.cachePath);
+      if (this.cachePath.endsWith(".gz")) {
+        raw = zlib.gunzipSync(raw);
+      }
+      const data = JSON.parse(raw.toString("utf-8"));
+      this.entries = new Map(Object.entries(data));
+    } catch (e) {
+      // Fallback: ignore cache errors
+    }
+  }
+
+  /**
+   * Saves the cache in the background.
+   */
+  save(): void {
+    if (process.env.BOLTDOCS_NO_CACHE === "1") return;
+    if (!this.cachePath) return;
+
+    const data = Object.fromEntries(this.entries);
+    const content = JSON.stringify(data);
+    const target = this.cachePath;
+    const useCompress = this.compress;
+
+    backgroundQueue.add(async () => {
+      try {
+        await mkdir(path.dirname(target), { recursive: true });
+        let buffer = Buffer.from(content);
+        if (useCompress) {
+          buffer = zlib.gzipSync(buffer);
+        }
+        const tempPath = `${target}.${crypto.randomBytes(4).toString("hex")}.tmp`;
+        await writeFile(tempPath, buffer);
+        await rename(tempPath, target);
+      } catch (e) {
+        // Fallback: critical error logging skipped for performance
+      }
+    });
+  }
+
   get(filePath: string): T | null {
     const entry = this.entries.get(filePath);
     if (!entry) return null;
-
-    const currentMtime = getFileMtime(filePath);
-    if (currentMtime !== entry.mtime) return null;
-
+    if (getFileMtime(filePath) !== entry.mtime) return null;
     return entry.data;
   }
 
-  /**
-   * Stores parsed data for a file in the cache, recording its current mtime.
-   *
-   * @param filePath - The absolute path to the file
-   * @param data - The parsed data to store
-   */
   set(filePath: string, data: T): void {
     this.entries.set(filePath, {
       data,
@@ -45,40 +169,20 @@ export class FileCache<T> {
     });
   }
 
-  /**
-   * Checks if a specific file's cache is still valid (based on its mtime).
-   *
-   * @param filePath - The absolute path to the file
-   * @returns `true` if the cache is valid, `false` otherwise
-   */
   isValid(filePath: string): boolean {
-    return this.get(filePath) !== null;
+    const entry = this.entries.get(filePath);
+    if (!entry) return false;
+    return getFileMtime(filePath) === entry.mtime;
   }
 
-  /**
-   * Manually removes a specific file from the cache.
-   * Useful when forcefully invalidating a single updated file.
-   *
-   * @param filePath - The absolute path to the file
-   */
   invalidate(filePath: string): void {
     this.entries.delete(filePath);
   }
 
-  /**
-   * Clears the entire cache, forcing all files to be re-parsed on the next request.
-   * Useful when global dependencies (like config) change.
-   */
   invalidateAll(): void {
     this.entries.clear();
   }
 
-  /**
-   * Removes cached entries for files that no longer exist on the filesystem.
-   * Prevents memory leaks from deleted files.
-   *
-   * @param currentFiles - A Set of absolute file paths currently discovered on the disk
-   */
   pruneStale(currentFiles: Set<string>): void {
     for (const key of this.entries.keys()) {
       if (!currentFiles.has(key)) {
@@ -87,8 +191,218 @@ export class FileCache<T> {
     }
   }
 
-  /** Number of cached entries */
   get size(): number {
     return this.entries.size;
   }
+
+  async flush() {
+    await backgroundQueue.flush();
+  }
+}
+
+/**
+ * Sharded Cache: Optimized for large-scale data (like MDX transformations).
+ * Uses a memory index and individual files for each entry to avoid massive JSON parsing.
+ */
+export class TransformCache {
+  private index = new Map<string, string>(); // key -> hash
+  private memoryCache = new LRUCache<string, string>(DEFAULT_LRU_LIMIT);
+  private readonly baseDir: string;
+  private readonly shardsDir: string;
+  private readonly indexPath: string;
+
+  constructor(name: string, root: string = process.cwd()) {
+    this.baseDir = path.resolve(root, CACHE_DIR, `transform-${name}`);
+    this.shardsDir = path.resolve(this.baseDir, SHARDS_DIR);
+    this.indexPath = path.resolve(this.baseDir, "index.json");
+  }
+
+  /**
+   * Loads the index into memory.
+   */
+  load(): void {
+    if (process.env.BOLTDOCS_NO_CACHE === "1") return;
+    if (!fs.existsSync(this.indexPath)) return;
+
+    try {
+      const data = fs.readFileSync(this.indexPath, "utf-8");
+      this.index = new Map(Object.entries(JSON.parse(data)));
+    } catch (e) {
+      // Index might be corrupt, ignore
+    }
+  }
+
+  /**
+   * Persists the index in background.
+   */
+  save(): void {
+    if (process.env.BOLTDOCS_NO_CACHE === "1") return;
+    const data = JSON.stringify(Object.fromEntries(this.index));
+    const target = this.indexPath;
+
+    backgroundQueue.add(async () => {
+      await mkdir(path.dirname(target), { recursive: true });
+      await writeFile(target, data);
+    });
+  }
+
+  /**
+   * Batch Read: Retrieves multiple transformation results concurrently.
+   */
+  async getMany(keys: string[]): Promise<Map<string, string>> {
+    const results = new Map<string, string>();
+    const toLoad: string[] = [];
+
+    for (const key of keys) {
+      const mem = this.memoryCache.get(key);
+      if (mem) results.set(key, mem);
+      else if (this.index.has(key)) toLoad.push(key);
+    }
+
+    if (toLoad.length > 0) {
+      const shards = await Promise.all(
+        toLoad.map(async (key) => {
+          const hash = this.index.get(key)!;
+          const shardPath = path.resolve(this.shardsDir, `${hash}.gz`);
+          try {
+            const compressed = await readFile(shardPath);
+            const decompressed = zlib.gunzipSync(compressed).toString("utf-8");
+            this.memoryCache.set(key, decompressed);
+            return { key, val: decompressed };
+          } catch (e) {
+            return null;
+          }
+        }),
+      );
+
+      for (const s of shards) {
+        if (s) results.set(s.key, s.val);
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Retrieves a cached transformation. Fast lookup via index, lazy loading from disk.
+   */
+  get(key: string): string | null {
+    // 1. Check memory first (LRU)
+    const mem = this.memoryCache.get(key);
+    if (mem) return mem;
+
+    // 2. Check index
+    const hash = this.index.get(key);
+    if (!hash) return null;
+
+    // 3. Load from shard (synchronous read for Vite's transform hook compatibility)
+    const shardPath = path.resolve(this.shardsDir, `${hash}.gz`);
+    if (!fs.existsSync(shardPath)) return null;
+
+    try {
+      const compressed = fs.readFileSync(shardPath);
+      const decompressed = zlib.gunzipSync(compressed).toString("utf-8");
+      this.memoryCache.set(key, decompressed);
+      return decompressed;
+    } catch (e) {
+      return null;
+    }
+  }
+
+  /**
+   * Stores a transformation result.
+   */
+  set(key: string, result: string): void {
+    const hash = crypto.createHash("md5").update(result).digest("hex");
+    this.index.set(key, hash);
+    this.memoryCache.set(key, result);
+
+    const shardPath = path.resolve(this.shardsDir, `${hash}.gz`);
+
+    // Background write shard
+    backgroundQueue.add(async () => {
+      if (fs.existsSync(shardPath)) return; // Already exists
+      await mkdir(this.shardsDir, { recursive: true });
+
+      const compressed = zlib.gzipSync(Buffer.from(result));
+      const tempPath = `${shardPath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
+      await writeFile(tempPath, compressed);
+      await rename(tempPath, shardPath);
+    });
+  }
+
+  get size() {
+    return this.index.size;
+  }
+
+  async flush() {
+    await backgroundQueue.flush();
+  }
+}
+
+/**
+ * Specialized cache for processed assets (e.g., optimized images).
+ */
+export class AssetCache {
+  private readonly assetsDir: string;
+
+  constructor(root: string = process.cwd()) {
+    this.assetsDir = path.resolve(root, CACHE_DIR, ASSETS_DIR);
+  }
+
+  private getFileHash(filePath: string): string {
+    return crypto
+      .createHash("md5")
+      .update(fs.readFileSync(filePath))
+      .digest("hex");
+  }
+
+  get(sourcePath: string, cacheKey: string): string | null {
+    if (!fs.existsSync(sourcePath)) return null;
+    const sourceHash = this.getFileHash(sourcePath);
+    const cachedPath = this.getCachedPath(
+      sourcePath,
+      `${cacheKey}-${sourceHash}`,
+    );
+    return fs.existsSync(cachedPath) ? cachedPath : null;
+  }
+
+  set(sourcePath: string, cacheKey: string, content: Buffer | string): void {
+    const sourceHash = this.getFileHash(sourcePath);
+    const cachedPath = this.getCachedPath(
+      sourcePath,
+      `${cacheKey}-${sourceHash}`,
+    );
+
+    backgroundQueue.add(async () => {
+      await mkdir(this.assetsDir, { recursive: true });
+      const tempPath = `${cachedPath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
+      await writeFile(tempPath, content);
+      await rename(tempPath, cachedPath);
+    });
+  }
+
+  private getCachedPath(sourcePath: string, cacheKey: string): string {
+    const ext = path.extname(sourcePath);
+    const name = path.basename(sourcePath, ext);
+    const safeKey = cacheKey.replace(/[^a-z0-9]/gi, "-").toLowerCase();
+    return path.join(this.assetsDir, `${name}.${safeKey}${ext}`);
+  }
+
+  clear(): void {
+    if (fs.existsSync(this.assetsDir)) {
+      fs.rmSync(this.assetsDir, { recursive: true, force: true });
+    }
+  }
+
+  async flush() {
+    await backgroundQueue.flush();
+  }
+}
+
+/**
+ * Flushes all pending background cache operations.
+ */
+export async function flushCache() {
+  await backgroundQueue.flush();
 }

package/src/node/config.ts

@@ -64,6 +64,13 @@ export interface BoltdocsThemeConfig {
     className?: string;
     style?: any;
   };
+  /**
+   * The syntax highlighting theme for code blocks.
+   * Supports any Shiki theme name (e.g., 'github-dark', 'one-dark-pro', 'aurora-x').
+   * Can also be an object for multiple themes (e.g., { light: 'github-light', dark: 'github-dark' }).
+   * Default: 'one-dark-pro'
+   */
+  codeTheme?: string | Record<string, string>;
 }
 
 /**

package/src/node/mdx.ts

@@ -4,16 +4,27 @@ import remarkFrontmatter from "remark-frontmatter";
 import rehypeSlug from "rehype-slug";
 import rehypePrettyCode from "rehype-pretty-code";
 import type { Plugin } from "vite";
+import crypto from "crypto";
 
 import type { BoltdocsConfig } from "./config";
+import { TransformCache } from "./cache";
+
+/**
+ * Persistent cache for MDX transformations.
+ * Saves results to `.boltdocs/transform-mdx.json.gz`.
+ */
+const mdxCache = new TransformCache("mdx");
+let mdxCacheLoaded = false;
 
 /**
  * Configures the MDX compiler for Vite using `@mdx-js/rollup`.
  * Includes standard remark and rehype plugins for GitHub Flavored Markdown (GFM),
  * frontmatter extraction, auto-linking headers, and syntax highlighting via `rehype-pretty-code`.
  *
+ * Also wraps the plugin with a persistent cache to avoid re-compiling unchanged MDX files.
+ *
  * @param config - The Boltdocs configuration containing custom plugins
- * @returns A Vite plugin configured for MDX parsing
+ * @returns A Vite plugin configured for MDX parsing with caching
  */
 export function boltdocsMdxPlugin(config?: BoltdocsConfig): Plugin {
   const extraRemarkPlugins =
@@ -21,7 +32,7 @@ export function boltdocsMdxPlugin(config?: BoltdocsConfig): Plugin {
   const extraRehypePlugins =
     config?.plugins?.flatMap((p) => p.rehypePlugins || []) || [];
 
-  return mdxPlugin({
+  const baseMdxPlugin = mdxPlugin({
     remarkPlugins: [remarkGfm, remarkFrontmatter, ...extraRemarkPlugins],
     rehypePlugins: [
       rehypeSlug,
@@ -29,13 +40,81 @@ export function boltdocsMdxPlugin(config?: BoltdocsConfig): Plugin {
       [
         rehypePrettyCode,
         {
-          theme: "one-dark-pro",
+          theme: config?.themeConfig?.codeTheme || "one-dark-pro",
           keepBackground: false,
         },
       ],
     ],
-    // Provide React as default for JSX
     jsxRuntime: "automatic",
     providerImportSource: "@mdx-js/react",
   }) as Plugin;
+
+  return {
+    ...baseMdxPlugin,
+    name: "vite-plugin-boltdocs-mdx",
+
+    async buildStart() {
+      hits = 0;
+      total = 0;
+      if (!mdxCacheLoaded) {
+        mdxCache.load();
+        mdxCacheLoaded = true;
+      }
+      if (baseMdxPlugin.buildStart) {
+        await (baseMdxPlugin.buildStart as any).call(this);
+      }
+    },
+
+    async transform(code, id, options) {
+      if (!id.endsWith(".md") && !id.endsWith(".mdx")) {
+        return (baseMdxPlugin.transform as any)?.call(this, code, id, options);
+      }
+
+      total++;
+      // Create a cache key based on path, content, and config (simplified)
+      const contentHash = crypto.createHash("md5").update(code).digest("hex");
+      const cacheKey = `${id}:${contentHash}`;
+
+      const cached = mdxCache.get(cacheKey);
+      if (cached) {
+        hits++;
+        return { code: cached, map: null };
+      }
+
+      const result = await (baseMdxPlugin.transform as any).call(
+        this,
+        code,
+        id,
+        options,
+      );
+
+      if (result && typeof result === "object" && result.code) {
+        mdxCache.set(cacheKey, result.code);
+      } else if (typeof result === "string") {
+        mdxCache.set(cacheKey, result);
+      }
+
+      return result;
+    },
+
+    async buildEnd() {
+      mdxCache.save();
+      await mdxCache.flush(); // Use instance flush or global flushCache
+      if (baseMdxPlugin.buildEnd) {
+        await (baseMdxPlugin.buildEnd as any).call(this);
+      }
+    },
+  };
 }
+
+/**
+ * Returns the current MDX cache statistics.
+ * @returns An object with total and hit counts
+ * @deprecated Removed for performance
+ */
+export function getMdxCacheStats() {
+  return { hits: 0, total: 0 };
+}
+
+let hits = 0;
+let total = 0;

package/src/node/plugin/index.ts

@@ -166,6 +166,9 @@ export function boltdocsPlugin(
           : path.resolve(process.cwd(), "dist");
 
         await generateStaticPages({ docsDir, outDir, config });
+
+        const { flushCache } = await import("../cache");
+        await flushCache();
       },
     },
     ViteImageOptimizer({

package/src/node/routes/cache.ts

@@ -1,7 +1,11 @@
 import { FileCache } from "../cache";
 import { ParsedDocFile } from "./types";
 
-const docCache = new FileCache<ParsedDocFile>();
+/**
+ * Persistent cache for parsed documentation files.
+ * Saves data to `.boltdocs/routes.json`.
+ */
+const docCache = new FileCache<ParsedDocFile>({ name: "routes" });
 
 /**
  * Invalidate all cached routes.

package/src/node/routes/index.ts

@@ -27,6 +27,9 @@ export async function generateRoutes(
   config?: BoltdocsConfig,
   basePath: string = "/docs",
 ): Promise<RouteMeta[]> {
+  // Load persistent cache on first call
+  docCache.load();
+
   const files = await fastGlob(["**/*.md", "**/*.mdx"], {
     cwd: docsDir,
     absolute: true,
@@ -41,10 +44,14 @@ export async function generateRoutes(
   }
 
   // Parse files in parallel using Promise.all for increased efficiency
+  let cacheHits = 0;
   const parsed: ParsedDocFile[] = await Promise.all(
     files.map(async (file) => {
       const cached = docCache.get(file);
-      if (cached) return cached;
+      if (cached) {
+        cacheHits++;
+        return cached;
+      }
 
       const result = parseDocFile(file, docsDir, basePath, config);
       docCache.set(file, result);
@@ -52,6 +59,15 @@ export async function generateRoutes(
     }),
   );
 
+  if (files.length > 0) {
+    console.log(
+      `[boltdocs] Routes generated: ${files.length} files (${cacheHits} from cache, ${files.length - cacheHits} parsed)`,
+    );
+  }
+
+  // Save cache after batch processing
+  docCache.save();
+
   // Collect group metadata from directory names and index files
   const groupMeta = new Map<string, { title: string; position?: number }>();
   for (const p of parsed) {
@@ -126,7 +142,6 @@ export async function generateRoutes(
         } else if (pathAfterVersion === "/" + defaultLocale) {
           pathAfterVersion = "/";
         }
-
         const targetPath =
           prefix +
           "/" +

package/src/node/ssg/index.ts

@@ -107,4 +107,8 @@ export async function generateStaticPages(options: SSGOptions): Promise<void> {
   console.log(
     `[boltdocs] Generated ${routes.length} static pages + sitemap.xml`,
   );
+
+  // Ensure all cache operations (like index persistence) are finished
+  const { flushCache } = await import("../cache");
+  await flushCache();
 }

package/dist/Playground-B2FA34BC.mjs

@@ -1,6 +0,0 @@
-import {
-  Playground
-} from "./chunk-WPT4MWTQ.mjs";
-export {
-  Playground
-};