botdocs 0.3.0 → 0.5.0

package/dist/src/builder/site-generator.js

@@ -1,10 +1,18 @@
 import { readFileSync, writeFileSync, readdirSync, mkdirSync, cpSync } from 'fs';
-import { join, dirname, resolve } from 'path';
+import { join, dirname, basename } from 'path';
 import { MarkdownProcessor } from './markdown-processor.js';
 import { TemplateEngine } from './template-engine.js';
+import { underSrc } from './paths.js';
+import { rootPrefix, relativeUrl } from '../shared/site-root.js';
 import { fileURLToPath } from 'url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
+function stripHtml(value) {
+    return value.replace(/<[^>]*>/g, '');
+}
+export function absoluteUrl(baseUrl, urlPath) {
+    return `${baseUrl.replace(/\/+$/, '')}${urlPath.startsWith('/') ? urlPath : `/${urlPath}`}`;
+}
 export class SiteGenerator {
     processor;
     templateEngine;
@@ -42,9 +50,10 @@ export class SiteGenerator {
         this.documents.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
         // Generate navigation
         const navigation = this.buildNavigation(this.documents);
-        // Load templates from source directory (not dist)
-        // From dist/src/builder, go to project root, then to src/templates
-        const templatesDir = resolve(__dirname, '../../../src/templates');
+        const pageSequence = this.buildPageSequence(navigation);
+        // Load templates from the project's src/templates, whether running
+        // compiled from dist/ or directly from src/ via tsx.
+        const templatesDir = underSrc(__dirname, 'templates');
         const layoutTemplate = readFileSync(join(templatesDir, 'layout.html'), 'utf-8');
         const docPageTemplate = readFileSync(join(templatesDir, 'doc-page.html'), 'utf-8');
         const indexTemplate = readFileSync(join(templatesDir, 'index.html'), 'utf-8');
@@ -53,9 +62,11 @@ export class SiteGenerator {
         for (let i = 0; i < this.documents.length; i++) {
             const doc = this.documents[i];
             const isIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
+            const pageUrl = isIndex ? '/' : doc.url;
             // Prepare navigation data
-            const prevDoc = i > 0 ? this.documents[i - 1] : null;
-            const nextDoc = i < this.documents.length - 1 ? this.documents[i + 1] : null;
+            const adjacent = pageSequence.get(doc.url);
+            const root = rootPrefix(doc.url);
+            const relativeLink = (link) => link && { ...link, url: relativeUrl(root, link.url) };
             // Render document content
             const contentTemplate = isIndex ? indexTemplate : docPageTemplate;
             const content = this.templateEngine.renderWithLoops(contentTemplate, {
@@ -67,31 +78,31 @@ export class SiteGenerator {
                     ? this.documents.filter(d => d !== doc).map(d => ({
                         title: d.metadata.title,
                         description: d.metadata.description,
-                        url: d.url,
+                        url: relativeUrl(root, d.url),
                     }))
                     : undefined,
-                prevPage: prevDoc
-                    ? {
-                        title: prevDoc.metadata.title,
-                        url: prevDoc.url,
-                    }
-                    : undefined,
-                nextPage: nextDoc
-                    ? {
-                        title: nextDoc.metadata.title,
-                        url: nextDoc.url,
-                    }
-                    : undefined,
+                prevPage: relativeLink(adjacent?.prev),
+                nextPage: relativeLink(adjacent?.next),
             });
             // Render full page with layout
             const html = this.templateEngine.renderWithLoops(layoutTemplate, {
                 title: doc.metadata.title || 'Documentation',
-                description: doc.metadata.description || config.description || '',
+                // The template engine doesn't escape interpolated values, so
+                // config.description may contain markup (e.g. a hotlink) meant
+                // for the visible siteDescription below — strip it here since
+                // this one lands inside a <meta content="..."> attribute.
+                description: stripHtml(doc.metadata.description || config.description || ''),
                 siteTitle: config.title || 'Documentation',
                 siteDescription: config.description || '',
                 content,
-                navigation: this.renderNavigation(navigation, doc.url),
+                root,
+                navigation: this.renderNavigation(navigation, root, doc.url),
                 chatEnabled: config.chat?.enabled,
+                ogUrl: config.baseUrl ? absoluteUrl(config.baseUrl, pageUrl) : undefined,
+                searchConfigJson: JSON.stringify({
+                    topK: config.build?.topK ?? 3,
+                    minScore: config.build?.minScore ?? 0.75,
+                }),
                 attribution: config.attribution !== false, // defaults to true
             });
             // Write HTML file
@@ -109,43 +120,115 @@ export class SiteGenerator {
             }
         }
         console.log(`Generated ${this.documents.length} HTML pages`);
+        if (config.baseUrl) {
+            this.writeSitemap(outputDir, config.baseUrl);
+        }
         return this.documents;
     }
+    writeSitemap(outputDir, baseUrl) {
+        const urls = this.documents
+            .map((doc) => {
+            const isRootIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
+            return absoluteUrl(baseUrl, isRootIndex ? '/' : doc.url);
+        }).map((loc) => `  <url><loc>${loc}</loc></url>`)
+            .join('\n');
+        const sitemap = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`;
+        writeFileSync(join(outputDir, 'sitemap.xml'), sitemap, 'utf-8');
+        console.log('Generated sitemap.xml');
+    }
     /**
-     * Build navigation structure from documents
+     * Build navigation structure from documents, grouping by top-level
+     * folder so e.g. every doc under `path-of-python/` (its README plus
+     * anything in `path-of-python/docs/`) nests under one "Path of Python"
+     * entry instead of interleaving flat with every other folder's pages.
+     * Files at the root (no folder) stay flat, top-level entries.
      */
     buildNavigation(documents) {
         const nav = [];
+        const groups = new Map();
         for (const doc of documents) {
             const parts = doc.relativePath.split('/');
-            const title = doc.metadata.title || parts[parts.length - 1].replace(/\.md$/, '');
-            nav.push({
-                title,
-                url: doc.url,
-            });
+            const title = doc.metadata.title || basename(doc.relativePath, '.md');
+            const isRootIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
+            if (parts.length === 1) {
+                if (!isRootIndex) {
+                    nav.push({ title, url: doc.url });
+                }
+                continue;
+            }
+            const folder = parts[0];
+            const restOfPath = parts.slice(1).join('/');
+            const isFolderIndex = restOfPath === 'README.md' || restOfPath === 'index.md';
+            let group = groups.get(folder);
+            if (!group) {
+                group = {
+                    title: folder.replace(/[-_]/g, ' ').replace(/\b\w/g, (char) => char.toUpperCase()),
+                    url: doc.url,
+                    children: [],
+                };
+                groups.set(folder, group);
+                nav.push(group);
+            }
+            if (isFolderIndex) {
+                group.url = doc.url;
+            }
+            else {
+                group.children.push({ title, url: doc.url });
+            }
         }
         return nav;
     }
     /**
-     * Render navigation HTML
+     * Render navigation HTML. Folder groups only expand their children when
+     * the current page is the group's root or one of its children — every
+     * other page sees the group collapsed to a single link that leads to
+     * its root README.
      */
-    renderNavigation(items, currentUrl) {
+    renderNavigation(items, root, currentUrl) {
         if (items.length === 0)
             return '';
         let html = '<ul class="nav-list">';
         for (const item of items) {
             const isActive = item.url === currentUrl;
-            const activeClass = isActive ? ' class="active"' : '';
-            html += `<li${activeClass}>`;
-            html += `<a href="${item.url}"${activeClass}>${item.title}</a>`;
-            if (item.children && item.children.length > 0) {
-                html += this.renderNavigation(item.children, currentUrl);
+            const hasChildren = !!item.children && item.children.length > 0;
+            const isExpanded = hasChildren && (isActive || item.children.some((child) => child.url === currentUrl));
+            const liClasses = [isActive && 'active', hasChildren && 'nav-group']
+                .filter(Boolean)
+                .join(' ');
+            html += `<li${liClasses ? ` class="${liClasses}"` : ''}>`;
+            html += `<a href="${relativeUrl(root, item.url)}"${isActive ? ' class="active"' : ''}>${item.title}</a>`;
+            if (isExpanded) {
+                html += this.renderNavigation(item.children, root, currentUrl);
             }
             html += '</li>';
         }
         html += '</ul>';
         return html;
     }
+    /**
+     * Compute prev/next page links that walk the top-level navigation
+     * (root README to root README) rather than the flat, alphabetical
+     * document list — so paging from a folder's root never dips into that
+     * folder's children. Paging from within a group's children walks those
+     * siblings first, then rolls into the next top-level entry.
+     */
+    buildPageSequence(navigation) {
+        const positions = new Map();
+        const asLink = (item) => ({ title: item.title, url: item.url });
+        for (let i = 0; i < navigation.length; i++) {
+            const item = navigation[i];
+            const prevTop = i > 0 ? asLink(navigation[i - 1]) : undefined;
+            const nextTop = i < navigation.length - 1 ? asLink(navigation[i + 1]) : undefined;
+            positions.set(item.url, { prev: prevTop, next: nextTop });
+            const children = item.children ?? [];
+            for (let j = 0; j < children.length; j++) {
+                const prev = j === 0 ? asLink(item) : asLink(children[j - 1]);
+                const next = j < children.length - 1 ? asLink(children[j + 1]) : nextTop;
+                positions.set(children[j].url, { prev, next });
+            }
+        }
+        return positions;
+    }
     /**
      * Get all processed documents
      */

package/dist/src/builder/template-engine.d.ts

@@ -6,12 +6,12 @@ export declare class TemplateEngine {
      * Render a template with variables
      * Replaces {{variableName}} with values from data object
      */
-    render(template: string, data: Record<string, any>): string;
+    render(template: string, data: Record<string, unknown>): string;
     /**
      * Render a template with nested variables
      * Supports {{object.property}} syntax
      */
-    renderAdvanced(template: string, data: Record<string, any>): string;
+    renderAdvanced(template: string, data: Record<string, unknown>): string;
     /**
      * Get nested property from object using dot notation
      */
@@ -21,10 +21,10 @@ export declare class TemplateEngine {
      * Supports {{#if variable}}...{{/if}} syntax
      * Handles nested conditionals recursively
      */
-    renderWithConditionals(template: string, data: Record<string, any>): string;
+    renderWithConditionals(template: string, data: Record<string, unknown>): string;
     /**
      * Render template with loops
      * Supports {{#each items}}...{{/each}} syntax
      */
-    renderWithLoops(template: string, data: Record<string, any>): string;
+    renderWithLoops(template: string, data: Record<string, unknown>): string;
 }

package/dist/src/builder/template-engine.js

@@ -25,7 +25,11 @@ export class TemplateEngine {
      * Get nested property from object using dot notation
      */
     getNestedProperty(obj, path) {
-        return path.split('.').reduce((current, key) => current?.[key], obj);
+        return path
+            .split('.')
+            .reduce((current, key) => current && typeof current === 'object'
+            ? current[key]
+            : undefined, obj);
     }
     /**
      * Render template with conditional blocks

package/dist/src/builder/vector-db-builder.d.ts

@@ -3,6 +3,7 @@ import { VectorDatabase } from '../types/vector-db.js';
 export interface VectorDBBuilderOptions {
     chunkSize?: number;
     chunkOverlap?: number;
+    minChunkSize?: number;
     modelName?: string;
 }
 export declare class VectorDBBuilder {

package/dist/src/builder/vector-db-builder.js

@@ -10,6 +10,7 @@ export class VectorDBBuilder {
         this.chunker = new Chunker({
             maxChunkSize: options.chunkSize || 500,
             chunkOverlap: options.chunkOverlap || 50,
+            minChunkSize: options.minChunkSize,
         });
         this.embedder = new Embedder(options.modelName);
     }

package/dist/src/cli/index.js

@@ -1,14 +1,26 @@
 import { Command } from 'commander';
 import { build } from '../builder/index.js';
 import { defaultOptions } from './options.js';
-import { readFileSync } from 'fs';
+import { startServer } from './server.js';
+import { watchDocs } from './watcher.js';
+import { readFileSync, existsSync } from 'fs';
 import { resolve, dirname } from 'path';
 import { fileURLToPath } from 'url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-// Read package.json for version
-// From dist/src/cli/index.js, go up 3 levels to project root
-const packageJson = JSON.parse(readFileSync(resolve(__dirname, '../../../package.json'), 'utf-8'));
+// Read package.json for version. Walks up from this file so it resolves
+// both from dist/src/cli (installed) and src/cli (tsx).
+function findPackageJson(startDir) {
+    let dir = startDir;
+    while (dir !== dirname(dir)) {
+        const candidate = resolve(dir, 'package.json');
+        if (existsSync(candidate))
+            return candidate;
+        dir = dirname(dir);
+    }
+    throw new Error('package.json not found');
+}
+const packageJson = JSON.parse(readFileSync(findPackageJson(__dirname), 'utf-8'));
 const program = new Command();
 program
     .name('botdocs')
@@ -18,29 +30,60 @@ program
     .option('-o, --output <dir>', 'Output directory for generated site', defaultOptions.output)
     .option('--no-chat', 'Disable AI chatbot functionality')
     .option('-c, --config <file>', 'Path to config file (botdocs.config.json)')
-    .option('-t, --theme <theme>', 'Theme to use (classic, material, minimal, slate, modern)', defaultOptions.theme)
+    .option('-t, --theme <theme>', 'Theme to use (classic, material, minimal, slate, modern); overrides config file if set')
     .option('-v, --verbose', 'Enable verbose logging')
+    .option('-w, --watch', 'Rebuild on changes and serve the site for live preview')
+    .option('-p, --port <number>', 'Port for the preview server (with --watch)', parseInt)
     .action(async (input, options) => {
-    try {
-        const inputDir = resolve(process.cwd(), input);
-        const outputDir = resolve(process.cwd(), options.output || defaultOptions.output);
-        if (options.verbose) {
-            console.log('Botdocs starting...');
-            console.log(`Input: ${inputDir}`);
-            console.log(`Output: ${outputDir}`);
-            console.log(`Chat enabled: ${!options.noChat}`);
-            console.log(`Theme: ${options.theme || defaultOptions.theme}`);
-        }
+    const inputDir = resolve(process.cwd(), input);
+    const outputDir = resolve(process.cwd(), options.output || defaultOptions.output);
+    const verbose = options.verbose || false;
+    const runBuild = async () => {
         await build({
             inputDir,
             outputDir,
             chatEnabled: !options.noChat,
             configPath: options.config,
-            verbose: options.verbose || false,
+            verbose,
             theme: options.theme,
         });
-        console.log('Build complete!');
-        console.log(`Site generated at: ${outputDir}`);
+    };
+    try {
+        if (verbose) {
+            console.log('Botdocs starting...');
+            console.log(`Input: ${inputDir}`);
+            console.log(`Output: ${outputDir}`);
+            console.log(`Chat enabled: ${!options.noChat}`);
+            console.log(`Theme override: ${options.theme || '(none — using config file or default)'}`);
+        }
+        await runBuild();
+        if (options.watch) {
+            const server = await startServer(outputDir, options.port ?? defaultOptions.port);
+            console.log(`Site generated at: ${outputDir}`);
+            console.log(`Preview at: ${server.url}`);
+            console.log('Watching for changes... Press Ctrl+C to stop.');
+            const watcher = watchDocs(inputDir, (changes) => {
+                const names = [...new Set(changes.map((c) => c.path.split('/').pop()))].join(', ');
+                console.log(`Changed: ${names} — rebuilding...`);
+                runBuild().catch((error) => {
+                    console.error('Rebuild failed:', error);
+                });
+            });
+            let shuttingDown = false;
+            const shutdown = () => {
+                if (shuttingDown)
+                    return;
+                shuttingDown = true;
+                watcher.close();
+                server.close().finally(() => process.exit(0));
+            };
+            process.on('SIGINT', shutdown);
+            process.on('SIGTERM', shutdown);
+        }
+        else {
+            console.log('Build complete!');
+            console.log(`Site generated at: ${outputDir}`);
+        }
     }
     catch (error) {
         console.error('Build failed:', error);

package/dist/src/cli/options.d.ts

@@ -5,5 +5,7 @@ export interface CliOptions {
     config?: string;
     verbose?: boolean;
     theme?: Theme;
+    watch?: boolean;
+    port?: number;
 }
 export declare const defaultOptions: Partial<CliOptions>;

package/dist/src/cli/options.js

@@ -2,5 +2,6 @@ export const defaultOptions = {
     output: 'output',
     noChat: false,
     verbose: false,
-    theme: 'classic',
+    watch: false,
+    port: 3000,
 };

package/dist/src/cli/server.d.ts

@@ -0,0 +1,6 @@
+export interface RunningServer {
+    url: string;
+    port: number;
+    close(): Promise<void>;
+}
+export declare function startServer(rootDir: string, port?: number): Promise<RunningServer>;

package/dist/src/cli/server.js

@@ -0,0 +1,79 @@
+import { createServer } from 'http';
+import { promises as fs } from 'fs';
+import { join, resolve, sep, extname } from 'path';
+const MIME_TYPES = {
+    '.html': 'text/html; charset=utf-8',
+    '.css': 'text/css; charset=utf-8',
+    '.js': 'text/javascript; charset=utf-8',
+    '.mjs': 'text/javascript; charset=utf-8',
+    '.json': 'application/json; charset=utf-8',
+    '.map': 'application/json; charset=utf-8',
+    '.svg': 'image/svg+xml',
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.webp': 'image/webp',
+    '.ico': 'image/x-icon',
+    '.xml': 'application/xml; charset=utf-8',
+    '.txt': 'text/plain; charset=utf-8',
+    '.woff': 'font/woff',
+    '.woff2': 'font/woff2',
+};
+export async function startServer(rootDir, port = 0) {
+    const root = resolve(rootDir);
+    const server = createServer((req, res) => {
+        handleRequest(req, res, root).catch(() => {
+            respond(res, 500, 'text/plain; charset=utf-8', 'Internal server error');
+        });
+    });
+    return new Promise((resolveStart, rejectStart) => {
+        server.once('error', rejectStart);
+        server.listen(port, '127.0.0.1', () => {
+            const address = server.address();
+            const actualPort = typeof address === 'object' && address ? address.port : port;
+            resolveStart({
+                url: `http://127.0.0.1:${actualPort}`,
+                port: actualPort,
+                close: () => new Promise((done) => server.close(() => done())),
+            });
+        });
+    });
+}
+async function handleRequest(req, res, root) {
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+        respond(res, 405, 'text/plain; charset=utf-8', 'Method not allowed');
+        return;
+    }
+    let urlPath;
+    try {
+        urlPath = decodeURIComponent(new URL(req.url ?? '/', 'http://localhost').pathname);
+    }
+    catch {
+        respond(res, 400, 'text/plain; charset=utf-8', 'Bad request');
+        return;
+    }
+    // Complete mediation: every request resolves inside the site root or it
+    // never touches the filesystem.
+    const filePath = resolve(root, `.${sep}${urlPath}`);
+    if (filePath !== root && !filePath.startsWith(root + sep)) {
+        respond(res, 403, 'text/plain; charset=utf-8', 'Forbidden');
+        return;
+    }
+    let target = filePath;
+    try {
+        const stat = await fs.stat(target);
+        if (stat.isDirectory()) {
+            target = join(target, 'index.html');
+        }
+        const body = await fs.readFile(target);
+        respond(res, 200, MIME_TYPES[extname(target).toLowerCase()] ?? 'application/octet-stream', body);
+    }
+    catch {
+        respond(res, 404, 'text/plain; charset=utf-8', 'Not found');
+    }
+}
+function respond(res, status, contentType, body) {
+    res.writeHead(status, { 'Content-Type': contentType });
+    res.end(body);
+}

package/dist/src/cli/watcher.d.ts

@@ -0,0 +1,7 @@
+export interface WatchedChange {
+    path: string;
+}
+export interface DocWatcher {
+    close(): void;
+}
+export declare function watchDocs(inputDir: string, onChange: (changes: WatchedChange[]) => void, debounceMs?: number): DocWatcher;

package/dist/src/cli/watcher.js

@@ -0,0 +1,40 @@
+import { watch } from 'fs';
+const WATCHED_EXTENSIONS = ['.md', '.markdown'];
+const WATCHED_FILES = ['botdocs.config.json'];
+function isWatchedPath(path) {
+    if (WATCHED_EXTENSIONS.some((ext) => path.endsWith(ext)))
+        return true;
+    return WATCHED_FILES.some((file) => path === file || path.endsWith(`/${file}`));
+}
+export function watchDocs(inputDir, onChange, debounceMs = 300) {
+    let pending = new Map();
+    let timer;
+    const flush = () => {
+        timer = undefined;
+        const changes = [...pending.values()];
+        pending = new Map();
+        if (changes.length > 0) {
+            onChange(changes);
+        }
+    };
+    const schedule = (path) => {
+        pending.set(path, { path });
+        if (!timer) {
+            timer = setTimeout(flush, debounceMs);
+        }
+    };
+    const watcher = watch(inputDir, { recursive: true }, (_eventType, filename) => {
+        const path = filename ?? '';
+        // Some platforms report no filename — treat as a full rebuild trigger.
+        if (!path || isWatchedPath(path)) {
+            schedule(path);
+        }
+    });
+    return {
+        close: () => {
+            if (timer)
+                clearTimeout(timer);
+            watcher.close();
+        },
+    };
+}

package/dist/src/shared/site-root.d.ts

@@ -0,0 +1,2 @@
+export declare function rootPrefix(pageUrl: string): string;
+export declare function relativeUrl(root: string, url: string): string;

package/dist/src/shared/site-root.js

@@ -0,0 +1,9 @@
+export function rootPrefix(pageUrl) {
+    const depth = pageUrl.split('/').length - 2;
+    return depth <= 0 ? './' : '../'.repeat(depth);
+}
+export function relativeUrl(root, url) {
+    if (!url.startsWith('/') || url.startsWith('//'))
+        return url;
+    return root + (url === '/' ? 'index.html' : url.slice(1));
+}

package/dist/src/types/config.d.ts

@@ -3,7 +3,9 @@ export interface BotdocsConfig {
     title?: string;
     description?: string;
     theme?: Theme;
+    customCss?: string;
     attribution?: boolean;
+    baseUrl?: string;
     chat?: {
         enabled?: boolean;
         welcomeMessage?: string;
@@ -11,7 +13,9 @@ export interface BotdocsConfig {
     build?: {
         chunkSize?: number;
         chunkOverlap?: number;
+        minChunkSize?: number;
         topK?: number;
+        minScore?: number;
     };
 }
 export interface BuildOptions {

package/dist/src/types/config.js

@@ -10,6 +10,8 @@ export const defaultConfig = {
     build: {
         chunkSize: 500,
         chunkOverlap: 50,
+        minChunkSize: 15,
         topK: 3,
+        minScore: 0.75,
     },
 };

package/dist/src/types/document.d.ts

@@ -3,7 +3,7 @@ export interface DocumentMetadata {
     description?: string;
     date?: string;
     author?: string;
-    [key: string]: any;
+    [key: string]: unknown;
 }
 export interface ProcessedDocument {
     filePath: string;