botdocs 0.3.0 → 0.5.0

package/README.md

@@ -1,5 +1,7 @@
 # Botdocs
 
+[![Read about the commits](https://img.shields.io/badge/commits-code%20blog-1a1a1a?style=flat-square)](https://wwel.sh/digest.html?repo=botdocs)
+
 [![npm version](https://img.shields.io/npm/v/botdocs.svg)](https://www.npmjs.com/package/botdocs)
 
 Convert markdown documentation into beautiful static sites with AI-powered semantic search — no backend required.
@@ -12,6 +14,8 @@ Convert markdown documentation into beautiful static sites with AI-powered seman
 - **Deep Links** - Search results link directly to sections
 - **No Backend** - Everything runs in the browser
 - **Fast** - Syntax highlighting with Shiki
+- **Live Preview** - `--watch` rebuilds on save and serves the site locally
+- **SEO** - Optional Open Graph/Twitter tags and `sitemap.xml` via `baseUrl`
 
 ## Installation
 
@@ -44,6 +48,9 @@ botdocs ./docs -c ./my-config.json
 
 # Combine multiple options
 botdocs ./docs -o ./public -t slate -v
+
+# Live preview: rebuild on save + local server
+botdocs ./docs --watch
 ```
 
 ### CLI Options
@@ -55,6 +62,8 @@ botdocs ./docs -o ./public -t slate -v
 | `--config <file>` | `-c` | Path to config file | `botdocs.config.json` |
 | `--theme <theme>` | `-t` | Theme to use | `classic` |
 | `--verbose` | `-v` | Enable verbose logging | `false` |
+| `--watch` | `-w` | Rebuild on changes and serve the site for live preview | `false` |
+| `--port <number>` | `-p` | Port for the preview server (with `--watch`) | `3000` |
 
 ### Available Themes
 
@@ -73,12 +82,16 @@ Create `botdocs.config.json` in your docs directory:
   "title": "My Documentation",
   "description": "Project docs",
   "theme": "classic",
+  "customCss": "custom.css",
   "attribution": true,
+  "baseUrl": "https://example.com/docs/",
   "chat": { "enabled": true },
   "build": {
     "chunkSize": 500,
     "chunkOverlap": 50,
-    "topK": 3
+    "minChunkSize": 15,
+    "topK": 3,
+    "minScore": 0.75
   }
 }
 ```
@@ -90,12 +103,16 @@ Create `botdocs.config.json` in your docs directory:
 | `title` | string | `"Documentation"` | Site title |
 | `description` | string | `"Project documentation"` | Site description |
 | `theme` | string | `"classic"` | Theme to use (classic, material, minimal, slate, modern) |
+| `customCss` | string | none | Path to a CSS file, resolved relative to the config file's directory. Appended after theme CSS in `bundle.css`, so same-specificity selectors override the theme without `!important` |
 | `attribution` | boolean | `true` | Show "Built with Botdocs" footer link |
+| `baseUrl` | string | none | Canonical URL where the site is hosted. When set, pages get `rel=canonical` and Open Graph/Twitter card tags, and a `sitemap.xml` is generated |
 | `chat.enabled` | boolean | `true` | Enable AI chatbot |
 | `chat.welcomeMessage` | string | `"Ask me anything about the docs!"` | Chatbot welcome message |
 | `build.chunkSize` | number | `500` | Text chunk size for embeddings |
 | `build.chunkOverlap` | number | `50` | Overlap between chunks |
+| `build.minChunkSize` | number | `15` | Chunks smaller than this (estimated tokens) get folded into a neighboring chunk instead of becoming a standalone, low-signal search result |
 | `build.topK` | number | `3` | Number of results to return |
+| `build.minScore` | number | `0.75` | Minimum vector similarity (0-1) a result must reach to be returned at all, regardless of `topK` — filters out weak/off-topic matches instead of always padding results. The e5 embedding model has a fairly high similarity floor even for unrelated text, so this needs to sit well above 0.5 to actually gate anything |
 
 ## Front Matter
 
@@ -113,11 +130,12 @@ description: Quick start guide
 1. **Build**: Parses markdown → generates embeddings → creates `vector-db.json`
 2. **Runtime**: User query → embed → search vector DB → return relevant chunks
 3. **No LLM**: Pure semantic search, not AI text generation
+4. **Consent**: On first use, visitors are asked before the embedding model downloads to their browser, with a disclosure of what runs locally
 
 ## Architecture
 
 - **Embedding Model**: `e5-small-v2` (384-dim vectors, 2.2x faster than all-MiniLM-L6-v2)
-- **Search**: Cosine similarity, client-side only
+- **Search**: Hybrid — vector cosine similarity fused with BM25 keyword scoring (Reciprocal Rank Fusion), gated by a minimum similarity threshold, client-side only
 - **Browser Bundle**: ~825KB (includes Transformers.js)
 - **Deployment**: Fully static, works on any host
 

package/dist/src/builder/chunker.d.ts

@@ -7,6 +7,13 @@ export interface TextChunk {
 export interface ChunkerOptions {
     maxChunkSize: number;
     chunkOverlap: number;
+    /**
+     * Chunks smaller than this (in estimated tokens) are folded into a
+     * neighboring chunk instead of being kept as standalone, low-information
+     * entries in the vector DB (e.g. a heading followed by a single short
+     * line). Set to 0 to disable merging.
+     */
+    minChunkSize: number;
 }
 /**
  * Text chunker that splits documents by headings while respecting token limits
@@ -18,6 +25,17 @@ export declare class Chunker {
      * Chunk a document into semantically meaningful pieces
      */
     chunkDocument(doc: ProcessedDocument, fileHash?: string): TextChunk[];
+    /**
+     * Fold chunks smaller than minChunkSize into a neighbor so a heading with
+     * little or no body content doesn't become its own low-signal retrieval
+     * candidate.
+     */
+    private mergeSmallChunks;
+    /**
+     * True for a line that is entirely markdown badges/images/links with no
+     * other prose (a bare link line, a shields.io badge, or a chain of both).
+     */
+    private isBoilerplateLine;
     /**
      * Create a chunk with metadata
      */

package/dist/src/builder/chunker.js

@@ -1,3 +1,6 @@
+// Matches a markdown image, a badge (image wrapped in a link), or a plain
+// link, e.g. `![alt](url)`, `[![alt](url)](url)`, `[text](url)`.
+const LINK_OR_IMAGE = /\[!\[[^\]]*\]\([^)]*\)\]\([^)]*\)|!?\[[^\]]*\]\([^)]*\)/g;
 /**
  * Text chunker that splits documents by headings while respecting token limits
  */
@@ -7,6 +10,7 @@ export class Chunker {
         this.options = {
             maxChunkSize: options.maxChunkSize || 500,
             chunkOverlap: options.chunkOverlap || 50,
+            minChunkSize: options.minChunkSize ?? 15,
         };
     }
     /**
@@ -54,6 +58,13 @@ export class Chunker {
                 currentChunk = [line];
             }
             else {
+                // Badges and bare reference links carry no retrievable prose, but
+                // their markup is character-heavy enough to dodge minChunkSize and
+                // their repeated project-name alt-text then dominates BM25 matches
+                // on any query mentioning that name. Drop them before they're ever
+                // embedded or indexed rather than filtering by size after the fact.
+                if (this.isBoilerplateLine(line))
+                    continue;
                 currentChunk.push(line);
                 // Check if chunk is getting too large
                 const tokenCount = this.estimateTokens(currentChunk.join('\n'));
@@ -71,7 +82,57 @@ export class Chunker {
         if (currentChunk.length > 0) {
             chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading, fileHash));
         }
-        return chunks.filter((chunk) => chunk.text.trim().length > 0);
+        const nonEmptyChunks = chunks.filter((chunk) => chunk.text.trim().length > 0);
+        return this.mergeSmallChunks(nonEmptyChunks);
+    }
+    /**
+     * Fold chunks smaller than minChunkSize into a neighbor so a heading with
+     * little or no body content doesn't become its own low-signal retrieval
+     * candidate.
+     */
+    mergeSmallChunks(chunks) {
+        const minTokens = this.options.minChunkSize;
+        if (minTokens <= 0 || chunks.length <= 1)
+            return chunks;
+        const merged = [];
+        for (const chunk of chunks) {
+            const prev = merged[merged.length - 1];
+            if (prev && this.estimateTokens(prev.text) < minTokens) {
+                // Absorb forward: the small chunk becomes the lead-in for the next
+                // section, which takes over as the chunk's heading/metadata.
+                merged[merged.length - 1] = {
+                    text: `${prev.text}\n\n${chunk.text}`,
+                    metadata: chunk.metadata,
+                };
+            }
+            else {
+                merged.push(chunk);
+            }
+        }
+        // A trailing chunk with nothing after it to absorb into gets folded
+        // backward instead, keeping the earlier (more substantial) heading.
+        if (merged.length > 1) {
+            const last = merged[merged.length - 1];
+            if (this.estimateTokens(last.text) < minTokens) {
+                const prev = merged[merged.length - 2];
+                merged[merged.length - 2] = {
+                    text: `${prev.text}\n\n${last.text}`,
+                    metadata: prev.metadata,
+                };
+                merged.pop();
+            }
+        }
+        return merged;
+    }
+    /**
+     * True for a line that is entirely markdown badges/images/links with no
+     * other prose (a bare link line, a shields.io badge, or a chain of both).
+     */
+    isBoilerplateLine(line) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            return false;
+        return trimmed.replace(LINK_OR_IMAGE, '').trim().length === 0;
     }
     /**
      * Create a chunk with metadata

package/dist/src/builder/embedder.js

@@ -1,8 +1,8 @@
-import { pipeline, env } from '@xenova/transformers';
+import { pipeline, env } from '@huggingface/transformers';
 // Disable local model loading - use Hugging Face
 env.allowLocalModels = false;
 export class Embedder {
-    model;
+    model = null;
     modelName;
     dimension;
     constructor(modelName = 'Xenova/e5-small-v2') {
@@ -24,6 +24,9 @@ export class Embedder {
         if (!this.model) {
             await this.initialize();
         }
+        if (!this.model) {
+            throw new Error('Failed to initialize embedding model');
+        }
         // Prepend "passage: " prefix for e5 models
         const prefixedText = `passage: ${text}`;
         // Generate embedding

package/dist/src/builder/index-size.d.ts

@@ -0,0 +1,3 @@
+export declare const INDEX_SIZE_WARN_BYTES: number;
+export declare function formatBytes(bytes: number): string;
+export declare function isIndexSizeWarning(bytes: number): boolean;

package/dist/src/builder/index-size.js

@@ -0,0 +1,16 @@
+export const INDEX_SIZE_WARN_BYTES = 2 * 1024 * 1024;
+export function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    const units = ['KB', 'MB', 'GB'];
+    let value = bytes;
+    let unit = -1;
+    do {
+        value /= 1024;
+        unit++;
+    } while (value >= 1024 && unit < units.length - 1);
+    return `${value.toFixed(1)} ${units[unit]}`;
+}
+export function isIndexSizeWarning(bytes) {
+    return bytes >= INDEX_SIZE_WARN_BYTES;
+}

package/dist/src/builder/index.js

@@ -1,11 +1,14 @@
 import { defaultConfig } from '../types/config.js';
 import { SiteGenerator } from './site-generator.js';
 import { VectorDBBuilder } from './vector-db-builder.js';
-import { existsSync, readFileSync, writeFileSync, copyFileSync, mkdirSync, cpSync } from 'fs';
+import { formatBytes, isIndexSizeWarning } from './index-size.js';
+import { existsSync, readFileSync, writeFileSync, copyFileSync, mkdirSync, cpSync, statSync } from 'fs';
 import { join, dirname, resolve } from 'path';
 import { fileURLToPath } from 'url';
 import { exec } from 'child_process';
 import { promisify } from 'util';
+import { INDEX_SIZE_WARN_BYTES } from './index-size.js';
+import { underSrc } from './paths.js';
 const execAsync = promisify(exec);
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -20,6 +23,11 @@ export async function build(options) {
     }
     // Load config
     const config = loadConfig(configPath, inputDir);
+    // customCss resolves relative to wherever the config file actually lives,
+    // not inputDir — inputDir is often a regenerated staging directory.
+    const configDir = configPath && existsSync(configPath)
+        ? dirname(resolve(configPath))
+        : inputDir;
     // Override chat enabled setting if specified in CLI
     if (chatEnabled !== undefined) {
         config.chat = config.chat || {};
@@ -53,8 +61,19 @@ export async function build(options) {
         const vectorDBBuilder = new VectorDBBuilder({
             chunkSize: config.build?.chunkSize,
             chunkOverlap: config.build?.chunkOverlap,
+            minChunkSize: config.build?.minChunkSize,
         });
         await vectorDBBuilder.build(documents, outputDir, verbose);
+        const dbPath = join(outputDir, 'vector-db.json');
+        if (existsSync(dbPath)) {
+            const { size } = statSync(dbPath);
+            console.log(`Search index: ${formatBytes(size)}`);
+            if (isIndexSizeWarning(size)) {
+                console.warn(`Warning: search index exceeds ${formatBytes(INDEX_SIZE_WARN_BYTES)} — ` +
+                    'it loads fully in the browser, so large indexes slow first paint. ' +
+                    'Consider fewer/smaller docs or raising build.minChunkSize.');
+            }
+        }
     }
     // Phase 3: Build client-side code with Vite
     console.log('Building client-side code...');
@@ -109,8 +128,7 @@ export async function build(options) {
     }
     // Phase 4: Copy styles
     console.log('Copying styles...');
-    // From dist/src/builder, go to project root, then to src/styles
-    const stylesDir = resolve(__dirname, '../../../src/styles');
+    const stylesDir = underSrc(__dirname, 'styles');
     const themesDir = join(stylesDir, 'themes');
     const outputCssDir = join(assetsDir, 'css');
     // Determine which theme to use
@@ -139,6 +157,19 @@ export async function build(options) {
             bundledCss += readFileSync(fallbackPath, 'utf-8') + '\n\n';
         }
     }
+    // Append user-supplied custom CSS last so it overrides theme rules of equal specificity
+    if (config.customCss) {
+        const customCssPath = resolve(configDir, config.customCss);
+        if (existsSync(customCssPath)) {
+            bundledCss += readFileSync(customCssPath, 'utf-8') + '\n';
+            if (verbose) {
+                console.log(`Appended custom CSS: ${customCssPath}`);
+            }
+        }
+        else {
+            console.warn(`customCss file not found: ${customCssPath}`);
+        }
+    }
     writeFileSync(join(outputCssDir, 'bundle.css'), bundledCss, 'utf-8');
     if (verbose) {
         console.log('Styles copied');

package/dist/src/builder/markdown-processor.d.ts

@@ -4,13 +4,20 @@ export declare class MarkdownProcessor {
     private shikiInitialized;
     constructor();
     private escapeHtml;
+    private decodeHtmlEntities;
     private setupShiki;
     /**
      * Process a markdown file and extract front matter
      */
     processFile(filePath: string, inputDir: string, content: string): Promise<ProcessedDocument>;
     /**
-     * Extract title from markdown content (first h1) or filename
+     * Extract title from markdown content (first h1) or filename.
+     *
+     * Checks, in order: a markdown `# ` heading, an HTML `<h1>` tag (common
+     * when the heading wraps a logo image), and a lone banner image's alt
+     * text (READMEs that open with `![Project Name](banner.svg)` instead of
+     * a text heading). All matching skips fenced code blocks so shell
+     * comments like `# Start the server:` aren't mistaken for headings.
      */
     private extractTitle;
     /**

package/dist/src/builder/markdown-processor.js

@@ -9,7 +9,7 @@ import sub from 'markdown-it-sub';
 import sup from 'markdown-it-sup';
 import { bundledLanguages, getHighlighter } from 'shiki';
 import matter from 'gray-matter';
-import { relative, basename } from 'path';
+import { relative, basename, dirname } from 'path';
 export class MarkdownProcessor {
     md;
     shikiInitialized = false;
@@ -65,6 +65,15 @@ export class MarkdownProcessor {
             .replace(/"/g, '"')
             .replace(/'/g, ''');
     }
+    decodeHtmlEntities(text) {
+        return text
+            .replace(/ /g, ' ')
+            .replace(/&/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&quot;/g, '"')
+            .replace(/&#0?39;/g, "'");
+    }
     async setupShiki() {
         if (this.shikiInitialized)
             return;
@@ -129,16 +138,40 @@ export class MarkdownProcessor {
         };
     }
     /**
-     * Extract title from markdown content (first h1) or filename
+     * Extract title from markdown content (first h1) or filename.
+     *
+     * Checks, in order: a markdown `# ` heading, an HTML `<h1>` tag (common
+     * when the heading wraps a logo image), and a lone banner image's alt
+     * text (READMEs that open with `![Project Name](banner.svg)` instead of
+     * a text heading). All matching skips fenced code blocks so shell
+     * comments like `# Start the server:` aren't mistaken for headings.
      */
     extractTitle(content, relativePath) {
-        const h1Match = content.match(/^#\s+(.+)$/m);
+        const withoutCodeFences = content.replace(/^```[\s\S]*?^```/gm, '');
+        const h1Match = withoutCodeFences.match(/^#\s+(.+)$/m);
         if (h1Match) {
-            return h1Match[1];
+            return h1Match[1].trim();
+        }
+        const htmlH1Match = withoutCodeFences.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
+        if (htmlH1Match) {
+            const text = this.decodeHtmlEntities(htmlH1Match[1].replace(/<[^>]+>/g, ''))
+                .replace(/\s+/g, ' ')
+                .trim();
+            if (text) {
+                return text;
+            }
+        }
+        const bannerImageMatch = withoutCodeFences.match(/^!\[([^\]]+)\]\([^)]*\)\s*$/m);
+        if (bannerImageMatch) {
+            return bannerImageMatch[1].trim();
         }
-        // Fallback to filename
-        return basename(relativePath, '.md')
-            .replace(/-/g, ' ')
+        // Fallback to filename, or the parent directory name for README/index
+        // files where the filename itself carries no useful title.
+        const base = basename(relativePath, '.md');
+        const parentDir = basename(dirname(relativePath));
+        const name = /^(readme|index)$/i.test(base) && parentDir !== '.' ? parentDir : base;
+        return name
+            .replace(/[-_]/g, ' ')
             .replace(/\b\w/g, (char) => char.toUpperCase());
     }
     /**

package/dist/src/builder/paths.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Resolve a directory under the project's src/ tree regardless of whether
+ * this module is executing compiled (dist/src/...) or from source (src/...)
+ * via tsx.
+ */
+export declare function underSrc(moduleDir: string, name: string): string;

package/dist/src/builder/paths.js

@@ -0,0 +1,13 @@
+import { existsSync } from 'fs';
+import { resolve } from 'path';
+/**
+ * Resolve a directory under the project's src/ tree regardless of whether
+ * this module is executing compiled (dist/src/...) or from source (src/...)
+ * via tsx.
+ */
+export function underSrc(moduleDir, name) {
+    const inSourceTree = resolve(moduleDir, '..', name);
+    if (existsSync(inSourceTree))
+        return inSourceTree;
+    return resolve(moduleDir, '..', '..', '..', 'src', name);
+}

package/dist/src/builder/site-generator.d.ts

@@ -1,5 +1,6 @@
 import { ProcessedDocument } from '../types/document.js';
 import { BotdocsConfig } from '../types/config.js';
+export declare function absoluteUrl(baseUrl: string, urlPath: string): string;
 export declare class SiteGenerator {
     private processor;
     private templateEngine;
@@ -9,14 +10,30 @@ export declare class SiteGenerator {
      * Generate the complete site
      */
     generate(inputDir: string, outputDir: string, config: BotdocsConfig): Promise<ProcessedDocument[]>;
+    private writeSitemap;
     /**
-     * Build navigation structure from documents
+     * Build navigation structure from documents, grouping by top-level
+     * folder so e.g. every doc under `path-of-python/` (its README plus
+     * anything in `path-of-python/docs/`) nests under one "Path of Python"
+     * entry instead of interleaving flat with every other folder's pages.
+     * Files at the root (no folder) stay flat, top-level entries.
      */
     private buildNavigation;
     /**
-     * Render navigation HTML
+     * Render navigation HTML. Folder groups only expand their children when
+     * the current page is the group's root or one of its children — every
+     * other page sees the group collapsed to a single link that leads to
+     * its root README.
      */
     private renderNavigation;
+    /**
+     * Compute prev/next page links that walk the top-level navigation
+     * (root README to root README) rather than the flat, alphabetical
+     * document list — so paging from a folder's root never dips into that
+     * folder's children. Paging from within a group's children walks those
+     * siblings first, then rolls into the next top-level entry.
+     */
+    private buildPageSequence;
     /**
      * Get all processed documents
      */