botdocs 0.2.0 → 0.3.0

package/dist/src/builder/chunker.d.ts

@@ -17,7 +17,7 @@ export declare class Chunker {
     /**
      * Chunk a document into semantically meaningful pieces
      */
-    chunkDocument(doc: ProcessedDocument): TextChunk[];
+    chunkDocument(doc: ProcessedDocument, fileHash?: string): TextChunk[];
     /**
      * Create a chunk with metadata
      */
@@ -37,5 +37,5 @@ export declare class Chunker {
     /**
      * Chunk multiple documents
      */
-    chunkDocuments(documents: ProcessedDocument[]): TextChunk[];
+    chunkDocuments(documents: ProcessedDocument[], fileHashes?: Map<string, string>): TextChunk[];
 }

package/dist/src/builder/chunker.js

@@ -12,7 +12,7 @@ export class Chunker {
     /**
      * Chunk a document into semantically meaningful pieces
      */
-    chunkDocument(doc) {
+    chunkDocument(doc, fileHash) {
         const chunks = [];
         const lines = doc.content.split('\n');
         let currentChunk = [];
@@ -47,7 +47,7 @@ export class Chunker {
             if (headingMatch) {
                 // Save current chunk if it exists
                 if (currentChunk.length > 0) {
-                    chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
+                    chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading, fileHash));
                 }
                 // Start new chunk with this heading
                 currentHeading = headingMatch[2];
@@ -60,7 +60,7 @@ export class Chunker {
                 if (tokenCount >= this.options.maxChunkSize) {
                     // Split chunk
                     const chunkText = currentChunk.join('\n');
-                    chunks.push(this.createChunk(chunkText, doc, currentHeading));
+                    chunks.push(this.createChunk(chunkText, doc, currentHeading, fileHash));
                     // Create overlap for next chunk
                     const overlapLines = this.getOverlapLines(currentChunk, this.options.chunkOverlap);
                     currentChunk = overlapLines;
@@ -69,14 +69,14 @@ export class Chunker {
         }
         // Add final chunk
         if (currentChunk.length > 0) {
-            chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
+            chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading, fileHash));
         }
         return chunks.filter((chunk) => chunk.text.trim().length > 0);
     }
     /**
      * Create a chunk with metadata
      */
-    createChunk(text, doc, heading) {
+    createChunk(text, doc, heading, fileHash) {
         return {
             text: text.trim(),
             metadata: {
@@ -85,6 +85,7 @@ export class Chunker {
                 heading,
                 headingId: heading ? this.slugify(heading) : undefined,
                 url: doc.url,
+                fileHash,
             },
         };
     }
@@ -128,10 +129,11 @@ export class Chunker {
     /**
      * Chunk multiple documents
      */
-    chunkDocuments(documents) {
+    chunkDocuments(documents, fileHashes) {
         const allChunks = [];
         for (const doc of documents) {
-            const chunks = this.chunkDocument(doc);
+            const fileHash = fileHashes?.get(doc.relativePath);
+            const chunks = this.chunkDocument(doc, fileHash);
             allChunks.push(...chunks);
         }
         return allChunks;

package/dist/src/builder/vector-db-builder.d.ts

@@ -10,9 +10,21 @@ export declare class VectorDBBuilder {
     private embedder;
     constructor(options?: VectorDBBuilderOptions);
     /**
-     * Build vector database from documents
+     * Build vector database from documents (with incremental build support)
      */
     build(documents: ProcessedDocument[], outputDir: string, verbose?: boolean): Promise<VectorDatabase>;
+    /**
+     * Load existing vector database from disk
+     */
+    private loadExistingDatabase;
+    /**
+     * Build a map of sourceFile -> chunks for quick lookup
+     */
+    private buildCacheMap;
+    /**
+     * Hash file content for change detection
+     */
+    private hashContent;
     /**
      * Generate unique ID for a chunk
      */

package/dist/src/builder/vector-db-builder.js

@@ -1,6 +1,6 @@
 import { Chunker } from './chunker.js';
 import { Embedder } from './embedder.js';
-import { writeFileSync } from 'fs';
+import { writeFileSync, existsSync, readFileSync } from 'fs';
 import { join } from 'path';
 import { createHash } from 'crypto';
 export class VectorDBBuilder {
@@ -14,43 +14,130 @@ export class VectorDBBuilder {
         this.embedder = new Embedder(options.modelName);
     }
     /**
-     * Build vector database from documents
+     * Build vector database from documents (with incremental build support)
      */
     async build(documents, outputDir, verbose = false) {
         if (verbose) {
             console.log('Building vector database...');
         }
-        // Step 1: Chunk documents
-        console.log('Chunking documents...');
-        const chunks = this.chunker.chunkDocuments(documents);
-        console.log(`Created ${chunks.length} chunks`);
-        // Step 2: Generate embeddings
-        console.log('Generating embeddings...');
-        await this.embedder.initialize();
-        const texts = chunks.map((chunk) => chunk.text);
-        const embeddings = await this.embedder.embedBatch(texts, 10);
-        // Step 3: Create document chunks with embeddings
-        const documentChunks = chunks.map((chunk, index) => ({
-            id: this.generateChunkId(chunk.text, chunk.metadata.sourceFile, index),
-            text: chunk.text,
-            embedding: embeddings[index],
-            metadata: chunk.metadata,
-        }));
-        // Step 4: Create vector database
+        const outputPath = join(outputDir, 'vector-db.json');
+        // Step 1: Load existing vector database for incremental builds
+        const existingDB = this.loadExistingDatabase(outputPath, verbose);
+        const cachedChunks = this.buildCacheMap(existingDB);
+        // Step 2: Hash documents and determine what needs rebuilding
+        const fileHashes = new Map();
+        const changedDocs = [];
+        const unchangedDocs = [];
+        for (const doc of documents) {
+            const hash = this.hashContent(doc.content);
+            fileHashes.set(doc.relativePath, hash);
+            const cachedDocChunks = cachedChunks.get(doc.relativePath);
+            if (cachedDocChunks && cachedDocChunks[0]?.metadata.fileHash === hash) {
+                unchangedDocs.push(doc);
+                if (verbose) {
+                    console.log(`  ✓ Cached: ${doc.relativePath}`);
+                }
+            }
+            else {
+                changedDocs.push(doc);
+                if (verbose) {
+                    console.log(`  ⟳ Changed: ${doc.relativePath}`);
+                }
+            }
+        }
+        console.log(`Unchanged: ${unchangedDocs.length}, Changed: ${changedDocs.length}, Total: ${documents.length}`);
+        // Step 3: Reuse cached chunks for unchanged documents
+        const reusedChunks = [];
+        for (const doc of unchangedDocs) {
+            const chunks = cachedChunks.get(doc.relativePath);
+            if (chunks) {
+                reusedChunks.push(...chunks);
+            }
+        }
+        // Step 4: Process changed documents only
+        let newChunks = [];
+        if (changedDocs.length > 0) {
+            console.log('Chunking changed documents...');
+            const chunks = this.chunker.chunkDocuments(changedDocs, fileHashes);
+            console.log(`Created ${chunks.length} chunks from ${changedDocs.length} changed documents`);
+            // Generate embeddings only for changed documents
+            console.log('Generating embeddings for changed documents...');
+            await this.embedder.initialize();
+            const texts = chunks.map((chunk) => chunk.text);
+            const embeddings = await this.embedder.embedBatch(texts, 10);
+            // Create document chunks with embeddings
+            newChunks = chunks.map((chunk, index) => ({
+                id: this.generateChunkId(chunk.text, chunk.metadata.sourceFile, index),
+                text: chunk.text,
+                embedding: embeddings[index],
+                metadata: chunk.metadata,
+            }));
+        }
+        else {
+            console.log('No changes detected - reusing all cached embeddings');
+        }
+        // Step 5: Merge cached and new chunks
+        const allChunks = [...reusedChunks, ...newChunks];
+        // Step 6: Create vector database
         const vectorDB = {
             version: '1.0',
             model: this.embedder.getModelName(),
             dimension: this.embedder.getDimension(),
-            chunks: documentChunks,
+            chunks: allChunks,
         };
-        // Step 5: Write to file
-        const outputPath = join(outputDir, 'vector-db.json');
+        // Step 7: Write to file
         writeFileSync(outputPath, JSON.stringify(vectorDB, null, 2), 'utf-8');
         console.log(`Vector database saved: ${outputPath}`);
-        console.log(`Total chunks: ${vectorDB.chunks.length}`);
+        console.log(`Total chunks: ${vectorDB.chunks.length} (${reusedChunks.length} cached, ${newChunks.length} new)`);
         console.log(`Embedding dimension: ${vectorDB.dimension}`);
         return vectorDB;
     }
+    /**
+     * Load existing vector database from disk
+     */
+    loadExistingDatabase(outputPath, verbose) {
+        if (!existsSync(outputPath)) {
+            if (verbose) {
+                console.log('No existing vector database found - performing full build');
+            }
+            return null;
+        }
+        try {
+            const content = readFileSync(outputPath, 'utf-8');
+            const db = JSON.parse(content);
+            if (verbose) {
+                console.log(`Loaded existing vector database with ${db.chunks.length} chunks`);
+            }
+            return db;
+        }
+        catch (error) {
+            console.warn('Failed to load existing vector database - performing full build');
+            return null;
+        }
+    }
+    /**
+     * Build a map of sourceFile -> chunks for quick lookup
+     */
+    buildCacheMap(db) {
+        const cache = new Map();
+        if (!db) {
+            return cache;
+        }
+        for (const chunk of db.chunks) {
+            const sourceFile = chunk.metadata.sourceFile;
+            if (!cache.has(sourceFile)) {
+                cache.set(sourceFile, []);
+            }
+            cache.get(sourceFile).push(chunk);
+        }
+        return cache;
+    }
+    /**
+     * Hash file content for change detection
+     */
+    hashContent(content) {
+        return createHash('md5').update(content).digest('hex');
+    }
     /**
      * Generate unique ID for a chunk
      */

package/dist/src/types/vector-db.d.ts

@@ -6,6 +6,7 @@ export interface ChunkMetadata {
     url: string;
     startLine?: number;
     endLine?: number;
+    fileHash?: string;
 }
 export interface DocumentChunk {
     id: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "botdocs",
-  "version": "0.2.0",
-  "description": "An npm CLI tool that converts markdown documentation into a static website with an optional client-side AI chatbot",
+  "version": "0.3.0",
+  "description": "CLI tool that converts markdown documentation into a static website with an optional client-side AI chatbot",
   "author": "usr-wwelsh <https://wwel.sh>",
   "license": "MIT",
   "bin": {

package/src/styles/themes/minimal.css

@@ -258,7 +258,7 @@ blockquote {
 }
 
 .markdown-alert-note .markdown-alert-title::before {
-  content: 'Note: ';
+  content: '';
   font-weight: 700;
 }
 
@@ -272,7 +272,7 @@ blockquote {
 }
 
 .markdown-alert-tip .markdown-alert-title::before {
-  content: 'Tip: ';
+  content: '';
   font-weight: 700;
 }
 
@@ -286,7 +286,7 @@ blockquote {
 }
 
 .markdown-alert-important .markdown-alert-title::before {
-  content: 'Important: ';
+  content: '';
   font-weight: 700;
 }
 
@@ -300,7 +300,7 @@ blockquote {
 }
 
 .markdown-alert-warning .markdown-alert-title::before {
-  content: 'Warning: ';
+  content: '';
   font-weight: 700;
 }
 
@@ -314,7 +314,7 @@ blockquote {
 }
 
 .markdown-alert-caution .markdown-alert-title::before {
-  content: 'Caution: ';
+  content: '';
   font-weight: 700;
 }