botdocs 0.1.0

package/dist/src/builder/markdown-processor.js

@@ -0,0 +1,161 @@
+import MarkdownIt from 'markdown-it';
+import anchor from 'markdown-it-anchor';
+import toc from 'markdown-it-toc-done-right';
+import alerts from 'markdown-it-github-alerts';
+import taskLists from 'markdown-it-task-lists';
+import footnote from 'markdown-it-footnote';
+import { full as emoji } from 'markdown-it-emoji';
+import sub from 'markdown-it-sub';
+import sup from 'markdown-it-sup';
+import { bundledLanguages, getHighlighter } from 'shiki';
+import matter from 'gray-matter';
+import { relative, basename } from 'path';
+export class MarkdownProcessor {
+    md;
+    shikiInitialized = false;
+    constructor() {
+        this.md = new MarkdownIt({
+            html: true,
+            linkify: true,
+            typographer: true,
+            breaks: false,
+            highlight: (code, lang, attrs) => {
+                // Fallback for when Shiki isn't initialized or lang not found
+                if (!lang) {
+                    return `<pre><code>${this.escapeHtml(code)}</code></pre>`;
+                }
+                return `<pre><code class="language-${lang}">${this.escapeHtml(code)}</code></pre>`;
+            },
+        })
+            // Enable strikethrough (built-in feature)
+            .enable('strikethrough');
+        // Add anchor plugin for heading links
+        this.md.use(anchor, {
+            permalink: anchor.permalink.linkInsideHeader({
+                symbol: '#',
+                placement: 'before',
+            }),
+        });
+        // Add table of contents plugin
+        this.md.use(toc, {
+            containerClass: 'toc',
+            listType: 'ul',
+        });
+        // Add GitHub alerts plugin for [!NOTE], [!WARNING], etc.
+        this.md.use(alerts);
+        // Add task lists plugin for - [ ] and - [x]
+        this.md.use(taskLists, {
+            enabled: true,
+            label: true,
+            labelAfter: true,
+        });
+        // Add footnotes plugin for [^1] style references
+        this.md.use(footnote);
+        // Add emoji shortcuts plugin for :smile: → 😄
+        this.md.use(emoji);
+        // Add subscript and superscript support
+        this.md.use(sub);
+        this.md.use(sup);
+    }
+    escapeHtml(text) {
+        return text
+            .replace(/&/g, '&amp;')
+            .replace(/</g, '&lt;')
+            .replace(/>/g, '&gt;')
+            .replace(/"/g, '&quot;')
+            .replace(/'/g, '&#039;');
+    }
+    async setupShiki() {
+        if (this.shikiInitialized)
+            return;
+        try {
+            const highlighter = await getHighlighter({
+                themes: ['github-light', 'github-dark'],
+                langs: Object.keys(bundledLanguages),
+            });
+            // Override markdown-it highlight with Shiki, but with error handling
+            const originalHighlight = this.md.options.highlight;
+            this.md.options.highlight = (code, lang, attrs) => {
+                try {
+                    if (!lang)
+                        return originalHighlight(code, lang, attrs);
+                    // Try to get the language, fall back to txt if not found
+                    const languages = highlighter.getLoadedLanguages();
+                    const safeLang = languages.includes(lang) ? lang : 'txt';
+                    return highlighter.codeToHtml(code, {
+                        lang: safeLang,
+                        themes: {
+                            light: 'github-light',
+                            dark: 'github-dark',
+                        },
+                    });
+                }
+                catch (error) {
+                    // Fall back to default highlighting if Shiki fails
+                    return originalHighlight(code, lang, attrs);
+                }
+            };
+            this.shikiInitialized = true;
+        }
+        catch (error) {
+            console.warn('Failed to initialize Shiki, falling back to default code rendering');
+        }
+    }
+    /**
+     * Process a markdown file and extract front matter
+     */
+    async processFile(filePath, inputDir, content) {
+        // Ensure Shiki is initialized
+        await this.setupShiki();
+        // Parse front matter
+        const { data: metadata, content: markdownContent } = matter(content);
+        // Convert markdown to HTML
+        const html = this.md.render(markdownContent);
+        // Generate relative path and URL
+        const relativePath = relative(inputDir, filePath);
+        const url = this.generateUrl(relativePath);
+        // Extract title from metadata or first h1
+        const title = metadata.title || this.extractTitle(markdownContent, relativePath);
+        return {
+            filePath,
+            relativePath,
+            content: markdownContent,
+            html,
+            metadata: {
+                ...metadata,
+                title,
+            },
+            url,
+        };
+    }
+    /**
+     * Extract title from markdown content (first h1) or filename
+     */
+    extractTitle(content, relativePath) {
+        const h1Match = content.match(/^#\s+(.+)$/m);
+        if (h1Match) {
+            return h1Match[1];
+        }
+        // Fallback to filename
+        return basename(relativePath, '.md')
+            .replace(/-/g, ' ')
+            .replace(/\b\w/g, (char) => char.toUpperCase());
+    }
+    /**
+     * Generate URL from relative file path
+     * e.g., "getting-started.md" -> "/getting-started.html"
+     * e.g., "api/overview.md" -> "/api/overview.html"
+     */
+    generateUrl(relativePath) {
+        const url = relativePath
+            .replace(/\.md$/, '.html')
+            .replace(/\\/g, '/');
+        return url === 'index.html' ? '/' : `/${url}`;
+    }
+    /**
+     * Render markdown string to HTML
+     */
+    render(markdown) {
+        return this.md.render(markdown);
+    }
+}

package/dist/src/builder/site-generator.d.ts

@@ -0,0 +1,24 @@
+import { ProcessedDocument } from '../types/document.js';
+import { BotdocsConfig } from '../types/config.js';
+export declare class SiteGenerator {
+    private processor;
+    private templateEngine;
+    private documents;
+    constructor();
+    /**
+     * Generate the complete site
+     */
+    generate(inputDir: string, outputDir: string, config: BotdocsConfig): Promise<ProcessedDocument[]>;
+    /**
+     * Build navigation structure from documents
+     */
+    private buildNavigation;
+    /**
+     * Render navigation HTML
+     */
+    private renderNavigation;
+    /**
+     * Get all processed documents
+     */
+    getDocuments(): ProcessedDocument[];
+}

package/dist/src/builder/site-generator.js

@@ -0,0 +1,152 @@
+import { glob } from 'glob';
+import { readFileSync, writeFileSync } from 'fs';
+import fs from 'fs-extra';
+import { join, dirname, resolve } from 'path';
+import { MarkdownProcessor } from './markdown-processor.js';
+import { TemplateEngine } from './template-engine.js';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+export class SiteGenerator {
+    processor;
+    templateEngine;
+    documents = [];
+    constructor() {
+        this.processor = new MarkdownProcessor();
+        this.templateEngine = new TemplateEngine();
+    }
+    /**
+     * Generate the complete site
+     */
+    async generate(inputDir, outputDir, config) {
+        console.log('Processing markdown files...');
+        // Find all markdown files
+        const markdownFiles = await glob('**/*.md', {
+            cwd: inputDir,
+            absolute: true,
+            ignore: ['node_modules/**', '**/node_modules/**'],
+        });
+        if (markdownFiles.length === 0) {
+            throw new Error(`No markdown files found in ${inputDir}`);
+        }
+        console.log(`Found ${markdownFiles.length} markdown files`);
+        // Process all markdown files
+        for (const filePath of markdownFiles) {
+            const content = readFileSync(filePath, 'utf-8');
+            const doc = await this.processor.processFile(filePath, inputDir, content);
+            this.documents.push(doc);
+        }
+        // Sort documents by path for consistent ordering
+        this.documents.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+        // Generate navigation
+        const navigation = this.buildNavigation(this.documents);
+        // Load templates from source directory (not dist)
+        // From dist/src/builder, go to project root, then to src/templates
+        const templatesDir = resolve(__dirname, '../../../src/templates');
+        const layoutTemplate = readFileSync(join(templatesDir, 'layout.html'), 'utf-8');
+        const docPageTemplate = readFileSync(join(templatesDir, 'doc-page.html'), 'utf-8');
+        const indexTemplate = readFileSync(join(templatesDir, 'index.html'), 'utf-8');
+        // Generate HTML pages
+        console.log('Generating HTML pages...');
+        for (let i = 0; i < this.documents.length; i++) {
+            const doc = this.documents[i];
+            const isIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
+            // Prepare navigation data
+            const prevDoc = i > 0 ? this.documents[i - 1] : null;
+            const nextDoc = i < this.documents.length - 1 ? this.documents[i + 1] : null;
+            // Render document content
+            const contentTemplate = isIndex ? indexTemplate : docPageTemplate;
+            const content = this.templateEngine.renderWithLoops(contentTemplate, {
+                html: doc.html,
+                metadata: doc.metadata,
+                title: doc.metadata.title,
+                description: doc.metadata.description,
+                pages: isIndex
+                    ? this.documents.filter(d => d !== doc).map(d => ({
+                        title: d.metadata.title,
+                        description: d.metadata.description,
+                        url: d.url,
+                    }))
+                    : undefined,
+                prevPage: prevDoc
+                    ? {
+                        title: prevDoc.metadata.title,
+                        url: prevDoc.url,
+                    }
+                    : undefined,
+                nextPage: nextDoc
+                    ? {
+                        title: nextDoc.metadata.title,
+                        url: nextDoc.url,
+                    }
+                    : undefined,
+            });
+            // Render full page with layout
+            const html = this.templateEngine.renderWithLoops(layoutTemplate, {
+                title: doc.metadata.title || 'Documentation',
+                description: doc.metadata.description || config.description || '',
+                siteTitle: config.title || 'Documentation',
+                siteDescription: config.description || '',
+                content,
+                navigation: this.renderNavigation(navigation, doc.url),
+                chatEnabled: config.chat?.enabled,
+            });
+            // Write HTML file
+            const outputPath = join(outputDir, doc.relativePath.replace(/\.md$/, '.html'));
+            fs.ensureDirSync(dirname(outputPath));
+            writeFileSync(outputPath, html, 'utf-8');
+        }
+        // Copy index.html if README.md exists
+        const readmeDoc = this.documents.find((d) => d.relativePath === 'README.md' || d.relativePath === 'index.md');
+        if (readmeDoc) {
+            const readmePath = join(outputDir, readmeDoc.relativePath.replace(/\.md$/, '.html'));
+            const indexPath = join(outputDir, 'index.html');
+            if (readmePath !== indexPath) {
+                fs.copySync(readmePath, indexPath);
+            }
+        }
+        console.log(`Generated ${this.documents.length} HTML pages`);
+        return this.documents;
+    }
+    /**
+     * Build navigation structure from documents
+     */
+    buildNavigation(documents) {
+        const nav = [];
+        for (const doc of documents) {
+            const parts = doc.relativePath.split('/');
+            const title = doc.metadata.title || parts[parts.length - 1].replace(/\.md$/, '');
+            nav.push({
+                title,
+                url: doc.url,
+            });
+        }
+        return nav;
+    }
+    /**
+     * Render navigation HTML
+     */
+    renderNavigation(items, currentUrl) {
+        if (items.length === 0)
+            return '';
+        let html = '<ul class="nav-list">';
+        for (const item of items) {
+            const isActive = item.url === currentUrl;
+            const activeClass = isActive ? ' class="active"' : '';
+            html += `<li${activeClass}>`;
+            html += `<a href="${item.url}"${activeClass}>${item.title}</a>`;
+            if (item.children && item.children.length > 0) {
+                html += this.renderNavigation(item.children, currentUrl);
+            }
+            html += '</li>';
+        }
+        html += '</ul>';
+        return html;
+    }
+    /**
+     * Get all processed documents
+     */
+    getDocuments() {
+        return this.documents;
+    }
+}

package/dist/src/builder/template-engine.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Simple template engine using string interpolation
+ */
+export declare class TemplateEngine {
+    /**
+     * Render a template with variables
+     * Replaces {{variableName}} with values from data object
+     */
+    render(template: string, data: Record<string, any>): string;
+    /**
+     * Render a template with nested variables
+     * Supports {{object.property}} syntax
+     */
+    renderAdvanced(template: string, data: Record<string, any>): string;
+    /**
+     * Get nested property from object using dot notation
+     */
+    private getNestedProperty;
+    /**
+     * Render template with conditional blocks
+     * Supports {{#if variable}}...{{/if}} syntax
+     * Handles nested conditionals recursively
+     */
+    renderWithConditionals(template: string, data: Record<string, any>): string;
+    /**
+     * Render template with loops
+     * Supports {{#each items}}...{{/each}} syntax
+     */
+    renderWithLoops(template: string, data: Record<string, any>): string;
+}

package/dist/src/builder/template-engine.js

@@ -0,0 +1,77 @@
+/**
+ * Simple template engine using string interpolation
+ */
+export class TemplateEngine {
+    /**
+     * Render a template with variables
+     * Replaces {{variableName}} with values from data object
+     */
+    render(template, data) {
+        return template.replace(/\{\{(\w+)\}\}/g, (match, key) => {
+            return data[key] !== undefined ? String(data[key]) : match;
+        });
+    }
+    /**
+     * Render a template with nested variables
+     * Supports {{object.property}} syntax
+     */
+    renderAdvanced(template, data) {
+        return template.replace(/\{\{([\w.]+)\}\}/g, (match, path) => {
+            const value = this.getNestedProperty(data, path);
+            return value !== undefined ? String(value) : match;
+        });
+    }
+    /**
+     * Get nested property from object using dot notation
+     */
+    getNestedProperty(obj, path) {
+        return path.split('.').reduce((current, key) => current?.[key], obj);
+    }
+    /**
+     * Render template with conditional blocks
+     * Supports {{#if variable}}...{{/if}} syntax
+     * Handles nested conditionals recursively
+     */
+    renderWithConditionals(template, data) {
+        let result = template;
+        let previousResult = '';
+        // Keep processing until no more conditionals are found (handles nesting)
+        while (result !== previousResult) {
+            previousResult = result;
+            // Match innermost if blocks first (non-greedy, no nested {{#if}} inside)
+            result = result.replace(/\{\{#if\s+([\w.]+)\}\}((?:(?!\{\{#if)[\s\S])*?)\{\{\/if\}\}/g, (match, key, content) => {
+                const value = this.getNestedProperty(data, key);
+                return value ? content : '';
+            });
+        }
+        // Handle variable substitution
+        result = this.renderAdvanced(result, data);
+        return result;
+    }
+    /**
+     * Render template with loops
+     * Supports {{#each items}}...{{/each}} syntax
+     */
+    renderWithLoops(template, data) {
+        // Handle each blocks
+        let result = template.replace(/\{\{#each\s+(\w+)\}\}([\s\S]*?)\{\{\/each\}\}/g, (match, key, content) => {
+            const items = data[key];
+            if (!Array.isArray(items))
+                return '';
+            return items
+                .map((item, index) => {
+                // Process conditionals AND variables for each item
+                return this.renderWithConditionals(content, {
+                    ...data,
+                    ...item,
+                    index,
+                    '@index': index,
+                });
+            })
+                .join('');
+        });
+        // Handle conditionals and variables outside loops
+        result = this.renderWithConditionals(result, data);
+        return result;
+    }
+}

package/dist/src/builder/vector-db-builder.d.ts

@@ -0,0 +1,20 @@
+import { ProcessedDocument } from '../types/document.js';
+import { VectorDatabase } from '../types/vector-db.js';
+export interface VectorDBBuilderOptions {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    modelName?: string;
+}
+export declare class VectorDBBuilder {
+    private chunker;
+    private embedder;
+    constructor(options?: VectorDBBuilderOptions);
+    /**
+     * Build vector database from documents
+     */
+    build(documents: ProcessedDocument[], outputDir: string, verbose?: boolean): Promise<VectorDatabase>;
+    /**
+     * Generate unique ID for a chunk
+     */
+    private generateChunkId;
+}

package/dist/src/builder/vector-db-builder.js

@@ -0,0 +1,63 @@
+import { Chunker } from './chunker.js';
+import { Embedder } from './embedder.js';
+import { writeFileSync } from 'fs';
+import { join } from 'path';
+import { createHash } from 'crypto';
+export class VectorDBBuilder {
+    chunker;
+    embedder;
+    constructor(options = {}) {
+        this.chunker = new Chunker({
+            maxChunkSize: options.chunkSize || 500,
+            chunkOverlap: options.chunkOverlap || 50,
+        });
+        this.embedder = new Embedder(options.modelName);
+    }
+    /**
+     * Build vector database from documents
+     */
+    async build(documents, outputDir, verbose = false) {
+        if (verbose) {
+            console.log('Building vector database...');
+        }
+        // Step 1: Chunk documents
+        console.log('Chunking documents...');
+        const chunks = this.chunker.chunkDocuments(documents);
+        console.log(`Created ${chunks.length} chunks`);
+        // Step 2: Generate embeddings
+        console.log('Generating embeddings...');
+        await this.embedder.initialize();
+        const texts = chunks.map((chunk) => chunk.text);
+        const embeddings = await this.embedder.embedBatch(texts, 10);
+        // Step 3: Create document chunks with embeddings
+        const documentChunks = chunks.map((chunk, index) => ({
+            id: this.generateChunkId(chunk.text, chunk.metadata.sourceFile, index),
+            text: chunk.text,
+            embedding: embeddings[index],
+            metadata: chunk.metadata,
+        }));
+        // Step 4: Create vector database
+        const vectorDB = {
+            version: '1.0',
+            model: this.embedder.getModelName(),
+            dimension: this.embedder.getDimension(),
+            chunks: documentChunks,
+        };
+        // Step 5: Write to file
+        const outputPath = join(outputDir, 'vector-db.json');
+        writeFileSync(outputPath, JSON.stringify(vectorDB, null, 2), 'utf-8');
+        console.log(`Vector database saved: ${outputPath}`);
+        console.log(`Total chunks: ${vectorDB.chunks.length}`);
+        console.log(`Embedding dimension: ${vectorDB.dimension}`);
+        return vectorDB;
+    }
+    /**
+     * Generate unique ID for a chunk
+     */
+    generateChunkId(text, sourceFile, index) {
+        const hash = createHash('md5')
+            .update(`${sourceFile}:${index}:${text.substring(0, 100)}`)
+            .digest('hex');
+        return hash.substring(0, 16);
+    }
+}

package/dist/src/cli/index.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/cli/index.js

@@ -0,0 +1,47 @@
+import { Command } from 'commander';
+import { build } from '../builder/index.js';
+import { defaultOptions } from './options.js';
+import { readFileSync } from 'fs';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+// Read package.json for version
+// From dist/src/cli/index.js, go up 3 levels to project root
+const packageJson = JSON.parse(readFileSync(resolve(__dirname, '../../../package.json'), 'utf-8'));
+const program = new Command();
+program
+    .name('botdocs')
+    .description('Convert markdown documentation into a static website with AI chatbot')
+    .version(packageJson.version)
+    .argument('<input>', 'Input directory containing markdown files')
+    .option('-o, --output <dir>', 'Output directory for generated site', defaultOptions.output)
+    .option('--no-chat', 'Disable AI chatbot functionality')
+    .option('-c, --config <file>', 'Path to config file (botdocs.config.json)')
+    .option('-v, --verbose', 'Enable verbose logging')
+    .action(async (input, options) => {
+    try {
+        const inputDir = resolve(process.cwd(), input);
+        const outputDir = resolve(process.cwd(), options.output || defaultOptions.output);
+        if (options.verbose) {
+            console.log('Botdocs starting...');
+            console.log(`Input: ${inputDir}`);
+            console.log(`Output: ${outputDir}`);
+            console.log(`Chat enabled: ${!options.noChat}`);
+        }
+        await build({
+            inputDir,
+            outputDir,
+            chatEnabled: !options.noChat,
+            configPath: options.config,
+            verbose: options.verbose || false,
+        });
+        console.log('Build complete!');
+        console.log(`Site generated at: ${outputDir}`);
+    }
+    catch (error) {
+        console.error('Build failed:', error);
+        process.exit(1);
+    }
+});
+program.parse();

package/dist/src/cli/options.d.ts

@@ -0,0 +1,7 @@
+export interface CliOptions {
+    output?: string;
+    noChat?: boolean;
+    config?: string;
+    verbose?: boolean;
+}
+export declare const defaultOptions: Partial<CliOptions>;

package/dist/src/cli/options.js

@@ -0,0 +1,5 @@
+export const defaultOptions = {
+    output: 'output',
+    noChat: false,
+    verbose: false,
+};

package/dist/src/types/config.d.ts

@@ -0,0 +1,24 @@
+export interface BotdocsConfig {
+    title?: string;
+    description?: string;
+    theme?: {
+        primaryColor?: string;
+    };
+    chat?: {
+        enabled?: boolean;
+        welcomeMessage?: string;
+    };
+    build?: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        topK?: number;
+    };
+}
+export interface BuildOptions {
+    inputDir: string;
+    outputDir: string;
+    chatEnabled: boolean;
+    configPath?: string;
+    verbose: boolean;
+}
+export declare const defaultConfig: BotdocsConfig;

package/dist/src/types/config.js

@@ -0,0 +1,16 @@
+export const defaultConfig = {
+    title: 'Documentation',
+    description: 'Project documentation',
+    theme: {
+        primaryColor: '#3b82f6',
+    },
+    chat: {
+        enabled: true,
+        welcomeMessage: 'Ask me anything about the docs!',
+    },
+    build: {
+        chunkSize: 500,
+        chunkOverlap: 50,
+        topK: 5,
+    },
+};

package/dist/src/types/document.d.ts

@@ -0,0 +1,20 @@
+export interface DocumentMetadata {
+    title?: string;
+    description?: string;
+    date?: string;
+    author?: string;
+    [key: string]: any;
+}
+export interface ProcessedDocument {
+    filePath: string;
+    relativePath: string;
+    content: string;
+    html: string;
+    metadata: DocumentMetadata;
+    url: string;
+}
+export interface NavigationItem {
+    title: string;
+    url: string;
+    children?: NavigationItem[];
+}

package/dist/src/types/document.js

@@ -0,0 +1 @@
+export {};