botdocs 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 usr-wwelsh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,93 @@
+# Botdocs
+
+Convert markdown documentation into beautiful static sites with AI-powered semantic search — no backend required.
+
+## Features
+
+- **Markdown to HTML** - Converts `.md` files into polished static sites
+- **Semantic Search** - Client-side vector search using Transformers.js
+- **Dark Mode** - Built-in theme switching
+- **Deep Links** - Search results link directly to sections
+- **No Backend** - Everything runs in the browser
+- **Fast** - Syntax highlighting with Shiki
+
+## Installation
+
+Install globally via npm:
+
+```bash
+npm install -g botdocs
+```
+
+## Usage
+
+```bash
+# Generate site from markdown
+botdocs ./docs
+
+# Disable chatbot
+botdocs ./docs --no-chat
+
+# Custom output directory
+botdocs ./docs -o ./public
+
+# Verbose logging
+botdocs ./docs -v
+```
+
+## Configuration
+
+Create `botdocs.config.json` in your docs directory:
+
+```json
+{
+  "title": "My Documentation",
+  "description": "Project docs",
+  "chat": { "enabled": true },
+  "build": {
+    "chunkSize": 500,
+    "chunkOverlap": 50,
+    "topK": 5
+  }
+}
+```
+
+## Front Matter
+
+```markdown
+---
+title: Getting Started
+description: Quick start guide
+---
+
+# Your content here
+```
+
+## How It Works
+
+1. **Build**: Parses markdown → generates embeddings → creates `vector-db.json`
+2. **Runtime**: User query → embed → search vector DB → return relevant chunks
+3. **No LLM**: Pure semantic search, not AI text generation
+
+## Architecture
+
+- **Embedding Model**: `e5-small-v2` (384-dim vectors, 2.2x faster than all-MiniLM-L6-v2)
+- **Search**: Cosine similarity, client-side only
+- **Browser Bundle**: ~825KB (includes Transformers.js)
+- **Deployment**: Fully static, works on any host
+
+## Development
+
+Building from source:
+
+```bash
+git clone https://github.com/usr-wwelsh/botdocs.git
+cd botdocs
+npm install
+npm run build && npm run build:client
+botdocs ./test-docs
+```
+
+## License
+
+MIT © [usr-wwelsh](https://github.com/usr-wwelsh)

package/bin/botdocs.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+import('../dist/src/cli/index.js').catch((err) => {
+  console.error('Failed to load botdocs CLI:', err);
+  process.exit(1);
+});

package/dist/src/builder/chunker.d.ts

@@ -0,0 +1,41 @@
+import { ProcessedDocument } from '../types/document.js';
+import { ChunkMetadata } from '../types/vector-db.js';
+export interface TextChunk {
+    text: string;
+    metadata: ChunkMetadata;
+}
+export interface ChunkerOptions {
+    maxChunkSize: number;
+    chunkOverlap: number;
+}
+/**
+ * Text chunker that splits documents by headings while respecting token limits
+ */
+export declare class Chunker {
+    private options;
+    constructor(options?: Partial<ChunkerOptions>);
+    /**
+     * Chunk a document into semantically meaningful pieces
+     */
+    chunkDocument(doc: ProcessedDocument): TextChunk[];
+    /**
+     * Create a chunk with metadata
+     */
+    private createChunk;
+    /**
+     * Convert heading text to anchor ID (matches markdown-it-anchor behavior)
+     */
+    private slugify;
+    /**
+     * Estimate token count (rough approximation: ~4 chars = 1 token)
+     */
+    private estimateTokens;
+    /**
+     * Get last N tokens worth of lines for overlap
+     */
+    private getOverlapLines;
+    /**
+     * Chunk multiple documents
+     */
+    chunkDocuments(documents: ProcessedDocument[]): TextChunk[];
+}

package/dist/src/builder/chunker.js

@@ -0,0 +1,139 @@
+/**
+ * Text chunker that splits documents by headings while respecting token limits
+ */
+export class Chunker {
+    options;
+    constructor(options = {}) {
+        this.options = {
+            maxChunkSize: options.maxChunkSize || 500,
+            chunkOverlap: options.chunkOverlap || 50,
+        };
+    }
+    /**
+     * Chunk a document into semantically meaningful pieces
+     */
+    chunkDocument(doc) {
+        const chunks = [];
+        const lines = doc.content.split('\n');
+        let currentChunk = [];
+        let currentHeading;
+        let inCodeBlock = false;
+        let codeBlockLines = [];
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            // Handle code blocks
+            if (line.trim().startsWith('```')) {
+                if (!inCodeBlock) {
+                    // Start of code block
+                    inCodeBlock = true;
+                    codeBlockLines = [line];
+                }
+                else {
+                    // End of code block
+                    inCodeBlock = false;
+                    codeBlockLines.push(line);
+                    // Add complete code block to current chunk
+                    currentChunk.push(...codeBlockLines);
+                    codeBlockLines = [];
+                }
+                continue;
+            }
+            if (inCodeBlock) {
+                codeBlockLines.push(line);
+                continue;
+            }
+            // Detect headings
+            const headingMatch = line.match(/^(#{1,3})\s+(.+)$/);
+            if (headingMatch) {
+                // Save current chunk if it exists
+                if (currentChunk.length > 0) {
+                    chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
+                }
+                // Start new chunk with this heading
+                currentHeading = headingMatch[2];
+                currentChunk = [line];
+            }
+            else {
+                currentChunk.push(line);
+                // Check if chunk is getting too large
+                const tokenCount = this.estimateTokens(currentChunk.join('\n'));
+                if (tokenCount >= this.options.maxChunkSize) {
+                    // Split chunk
+                    const chunkText = currentChunk.join('\n');
+                    chunks.push(this.createChunk(chunkText, doc, currentHeading));
+                    // Create overlap for next chunk
+                    const overlapLines = this.getOverlapLines(currentChunk, this.options.chunkOverlap);
+                    currentChunk = overlapLines;
+                }
+            }
+        }
+        // Add final chunk
+        if (currentChunk.length > 0) {
+            chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
+        }
+        return chunks.filter((chunk) => chunk.text.trim().length > 0);
+    }
+    /**
+     * Create a chunk with metadata
+     */
+    createChunk(text, doc, heading) {
+        return {
+            text: text.trim(),
+            metadata: {
+                sourceFile: doc.relativePath,
+                title: doc.metadata.title || 'Untitled',
+                heading,
+                headingId: heading ? this.slugify(heading) : undefined,
+                url: doc.url,
+            },
+        };
+    }
+    /**
+     * Convert heading text to anchor ID (matches markdown-it-anchor behavior)
+     */
+    slugify(text) {
+        return text
+            .toLowerCase()
+            .trim()
+            .replace(/[\s+]/g, '-')
+            .replace(/[^\w\-]+/g, '')
+            .replace(/\-\-+/g, '-')
+            .replace(/^-+/, '')
+            .replace(/-+$/, '');
+    }
+    /**
+     * Estimate token count (rough approximation: ~4 chars = 1 token)
+     */
+    estimateTokens(text) {
+        return Math.ceil(text.length / 4);
+    }
+    /**
+     * Get last N tokens worth of lines for overlap
+     */
+    getOverlapLines(lines, targetTokens) {
+        const result = [];
+        let tokenCount = 0;
+        // Work backwards from end
+        for (let i = lines.length - 1; i >= 0; i--) {
+            const line = lines[i];
+            const lineTokens = this.estimateTokens(line);
+            if (tokenCount + lineTokens > targetTokens) {
+                break;
+            }
+            result.unshift(line);
+            tokenCount += lineTokens;
+        }
+        return result;
+    }
+    /**
+     * Chunk multiple documents
+     */
+    chunkDocuments(documents) {
+        const allChunks = [];
+        for (const doc of documents) {
+            const chunks = this.chunkDocument(doc);
+            allChunks.push(...chunks);
+        }
+        return allChunks;
+    }
+}

package/dist/src/builder/embedder.d.ts

@@ -0,0 +1,26 @@
+export declare class Embedder {
+    private model;
+    private modelName;
+    private dimension;
+    constructor(modelName?: string);
+    /**
+     * Initialize the embedding model
+     */
+    initialize(): Promise<void>;
+    /**
+     * Generate embedding for a single text
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Generate embeddings for multiple texts (batched for efficiency)
+     */
+    embedBatch(texts: string[], batchSize?: number, onProgress?: (current: number, total: number) => void): Promise<number[][]>;
+    /**
+     * Get embedding dimension
+     */
+    getDimension(): number;
+    /**
+     * Get model name
+     */
+    getModelName(): string;
+}

package/dist/src/builder/embedder.js

@@ -0,0 +1,75 @@
+import { pipeline, env } from '@xenova/transformers';
+// Disable local model loading - use Hugging Face
+env.allowLocalModels = false;
+export class Embedder {
+    model;
+    modelName;
+    dimension;
+    constructor(modelName = 'Xenova/e5-small-v2') {
+        this.modelName = modelName;
+        this.dimension = 384; // e5-small-v2 produces 384-dimensional embeddings
+    }
+    /**
+     * Initialize the embedding model
+     */
+    async initialize() {
+        console.log(`Loading embedding model: ${this.modelName}...`);
+        this.model = await pipeline('feature-extraction', this.modelName);
+        console.log('Embedding model loaded');
+    }
+    /**
+     * Generate embedding for a single text
+     */
+    async embed(text) {
+        if (!this.model) {
+            await this.initialize();
+        }
+        // Prepend "passage: " prefix for e5 models
+        const prefixedText = `passage: ${text}`;
+        // Generate embedding
+        const output = await this.model(prefixedText, {
+            pooling: 'mean',
+            normalize: true,
+        });
+        // Convert to array
+        const embedding = Array.from(output.data);
+        return embedding;
+    }
+    /**
+     * Generate embeddings for multiple texts (batched for efficiency)
+     */
+    async embedBatch(texts, batchSize = 10, onProgress) {
+        if (!this.model) {
+            await this.initialize();
+        }
+        const embeddings = [];
+        const total = texts.length;
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize);
+            // Process batch in parallel
+            const batchEmbeddings = await Promise.all(batch.map((text) => this.embed(text)));
+            embeddings.push(...batchEmbeddings);
+            // Report progress
+            const current = Math.min(i + batchSize, total);
+            if (onProgress) {
+                onProgress(current, total);
+            }
+            else {
+                console.log(`Embedded ${current}/${total} chunks`);
+            }
+        }
+        return embeddings;
+    }
+    /**
+     * Get embedding dimension
+     */
+    getDimension() {
+        return this.dimension;
+    }
+    /**
+     * Get model name
+     */
+    getModelName() {
+        return this.modelName;
+    }
+}

package/dist/src/builder/index.d.ts

@@ -0,0 +1,5 @@
+import { BuildOptions } from '../types/config.js';
+/**
+ * Main build orchestrator
+ */
+export declare function build(options: BuildOptions): Promise<void>;

package/dist/src/builder/index.js

@@ -0,0 +1,142 @@
+import { defaultConfig } from '../types/config.js';
+import { SiteGenerator } from './site-generator.js';
+import { VectorDBBuilder } from './vector-db-builder.js';
+import { existsSync, readFileSync, writeFileSync, copyFileSync } from 'fs';
+import fs from 'fs-extra';
+import { join, dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+const execAsync = promisify(exec);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+/**
+ * Main build orchestrator
+ */
+export async function build(options) {
+    const { inputDir, outputDir, chatEnabled, configPath, verbose } = options;
+    // Validate input directory
+    if (!existsSync(inputDir)) {
+        throw new Error(`Input directory does not exist: ${inputDir}`);
+    }
+    // Load config
+    const config = loadConfig(configPath, inputDir);
+    // Override chat enabled setting if specified in CLI
+    if (chatEnabled !== undefined) {
+        config.chat = config.chat || {};
+        config.chat.enabled = chatEnabled;
+    }
+    if (verbose) {
+        console.log('Configuration:', JSON.stringify(config, null, 2));
+    }
+    // Prepare output directory
+    fs.ensureDirSync(outputDir);
+    if (verbose) {
+        console.log(`Cleaning output directory: ${outputDir}`);
+    }
+    // Create site generator
+    const generator = new SiteGenerator();
+    // Generate site
+    const documents = await generator.generate(inputDir, outputDir, config);
+    // Create assets directories
+    const assetsDir = join(outputDir, 'assets');
+    fs.ensureDirSync(join(assetsDir, 'css'));
+    fs.ensureDirSync(join(assetsDir, 'js'));
+    if (verbose) {
+        console.log(`Processed ${documents.length} documents`);
+    }
+    // Phase 2: Generate vector database (if chat enabled)
+    if (config.chat?.enabled) {
+        const vectorDBBuilder = new VectorDBBuilder({
+            chunkSize: config.build?.chunkSize,
+            chunkOverlap: config.build?.chunkOverlap,
+        });
+        await vectorDBBuilder.build(documents, outputDir, verbose);
+    }
+    // Phase 3: Build client-side code with Vite
+    console.log('Building client-side code...');
+    // From dist/src/builder, go up 3 levels to project root
+    const projectRoot = resolve(__dirname, '../../..');
+    try {
+        // Skip rebuild if already built
+        const distClientDir = join(projectRoot, 'dist-client');
+        const bundleExists = existsSync(join(distClientDir, 'bundle.js'));
+        if (!bundleExists) {
+            await execAsync('npm run build:client', { cwd: projectRoot });
+        }
+        // Copy bundled JS files
+        if (existsSync(distClientDir)) {
+            const jsOutputDir = join(assetsDir, 'js');
+            if (verbose) {
+                console.log(`Copying client code from: ${distClientDir}`);
+                console.log(`Copying to: ${jsOutputDir}`);
+            }
+            // Copy bundle.js
+            const sourceBundlePath = join(distClientDir, 'bundle.js');
+            if (existsSync(sourceBundlePath)) {
+                copyFileSync(sourceBundlePath, join(jsOutputDir, 'bundle.js'));
+            }
+            // Copy bundle.js.map if exists
+            const sourceMapPath = join(distClientDir, 'bundle.js.map');
+            if (existsSync(sourceMapPath)) {
+                copyFileSync(sourceMapPath, join(jsOutputDir, 'bundle.js.map'));
+            }
+            // Copy assets folder (for code-split chunks)
+            const distAssetsDir = join(distClientDir, 'assets');
+            if (existsSync(distAssetsDir)) {
+                const outputAssetsDir = join(jsOutputDir, 'assets');
+                fs.copySync(distAssetsDir, outputAssetsDir);
+                if (verbose) {
+                    console.log('Copied code-split chunks');
+                }
+            }
+            if (verbose) {
+                console.log('Client code copy complete');
+            }
+        }
+        else if (verbose) {
+            console.log(`dist-client directory not found: ${distClientDir}`);
+        }
+    }
+    catch (error) {
+        console.warn('Client build skipped (run npm run build:client manually)');
+        if (verbose) {
+            console.error(error);
+        }
+    }
+    // Phase 4: Copy styles
+    console.log('Copying styles...');
+    // From dist/src/builder, go to project root, then to src/styles
+    const stylesDir = resolve(__dirname, '../../../src/styles');
+    const outputCssDir = join(assetsDir, 'css');
+    // Combine all CSS files into one bundle
+    const cssFiles = ['main.css', 'themes.css', 'chat.css'];
+    let bundledCss = '';
+    for (const cssFile of cssFiles) {
+        const cssPath = join(stylesDir, cssFile);
+        if (existsSync(cssPath)) {
+            bundledCss += readFileSync(cssPath, 'utf-8') + '\n\n';
+        }
+    }
+    writeFileSync(join(outputCssDir, 'bundle.css'), bundledCss, 'utf-8');
+    if (verbose) {
+        console.log('Styles copied');
+    }
+    console.log('Site generation complete!');
+}
+/**
+ * Load configuration from file or use defaults
+ */
+function loadConfig(configPath, inputDir) {
+    // Try explicit config path
+    if (configPath && existsSync(configPath)) {
+        return JSON.parse(readFileSync(configPath, 'utf-8'));
+    }
+    // Try default config in input directory
+    const defaultConfigPath = join(inputDir, 'botdocs.config.json');
+    if (existsSync(defaultConfigPath)) {
+        return JSON.parse(readFileSync(defaultConfigPath, 'utf-8'));
+    }
+    // Use defaults
+    return { ...defaultConfig };
+}

package/dist/src/builder/markdown-processor.d.ts

@@ -0,0 +1,26 @@
+import { ProcessedDocument } from '../types/document.js';
+export declare class MarkdownProcessor {
+    private md;
+    private shikiInitialized;
+    constructor();
+    private escapeHtml;
+    private setupShiki;
+    /**
+     * Process a markdown file and extract front matter
+     */
+    processFile(filePath: string, inputDir: string, content: string): Promise<ProcessedDocument>;
+    /**
+     * Extract title from markdown content (first h1) or filename
+     */
+    private extractTitle;
+    /**
+     * Generate URL from relative file path
+     * e.g., "getting-started.md" -> "/getting-started.html"
+     * e.g., "api/overview.md" -> "/api/overview.html"
+     */
+    private generateUrl;
+    /**
+     * Render markdown string to HTML
+     */
+    render(markdown: string): string;
+}