botdocs 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,7 +17,7 @@ export declare class Chunker {
17
17
  /**
18
18
  * Chunk a document into semantically meaningful pieces
19
19
  */
20
- chunkDocument(doc: ProcessedDocument): TextChunk[];
20
+ chunkDocument(doc: ProcessedDocument, fileHash?: string): TextChunk[];
21
21
  /**
22
22
  * Create a chunk with metadata
23
23
  */
@@ -37,5 +37,5 @@ export declare class Chunker {
37
37
  /**
38
38
  * Chunk multiple documents
39
39
  */
40
- chunkDocuments(documents: ProcessedDocument[]): TextChunk[];
40
+ chunkDocuments(documents: ProcessedDocument[], fileHashes?: Map<string, string>): TextChunk[];
41
41
  }
@@ -12,7 +12,7 @@ export class Chunker {
12
12
  /**
13
13
  * Chunk a document into semantically meaningful pieces
14
14
  */
15
- chunkDocument(doc) {
15
+ chunkDocument(doc, fileHash) {
16
16
  const chunks = [];
17
17
  const lines = doc.content.split('\n');
18
18
  let currentChunk = [];
@@ -47,7 +47,7 @@ export class Chunker {
47
47
  if (headingMatch) {
48
48
  // Save current chunk if it exists
49
49
  if (currentChunk.length > 0) {
50
- chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
50
+ chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading, fileHash));
51
51
  }
52
52
  // Start new chunk with this heading
53
53
  currentHeading = headingMatch[2];
@@ -60,7 +60,7 @@ export class Chunker {
60
60
  if (tokenCount >= this.options.maxChunkSize) {
61
61
  // Split chunk
62
62
  const chunkText = currentChunk.join('\n');
63
- chunks.push(this.createChunk(chunkText, doc, currentHeading));
63
+ chunks.push(this.createChunk(chunkText, doc, currentHeading, fileHash));
64
64
  // Create overlap for next chunk
65
65
  const overlapLines = this.getOverlapLines(currentChunk, this.options.chunkOverlap);
66
66
  currentChunk = overlapLines;
@@ -69,14 +69,14 @@ export class Chunker {
69
69
  }
70
70
  // Add final chunk
71
71
  if (currentChunk.length > 0) {
72
- chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
72
+ chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading, fileHash));
73
73
  }
74
74
  return chunks.filter((chunk) => chunk.text.trim().length > 0);
75
75
  }
76
76
  /**
77
77
  * Create a chunk with metadata
78
78
  */
79
- createChunk(text, doc, heading) {
79
+ createChunk(text, doc, heading, fileHash) {
80
80
  return {
81
81
  text: text.trim(),
82
82
  metadata: {
@@ -85,6 +85,7 @@ export class Chunker {
85
85
  heading,
86
86
  headingId: heading ? this.slugify(heading) : undefined,
87
87
  url: doc.url,
88
+ fileHash,
88
89
  },
89
90
  };
90
91
  }
@@ -128,10 +129,11 @@ export class Chunker {
128
129
  /**
129
130
  * Chunk multiple documents
130
131
  */
131
- chunkDocuments(documents) {
132
+ chunkDocuments(documents, fileHashes) {
132
133
  const allChunks = [];
133
134
  for (const doc of documents) {
134
- const chunks = this.chunkDocument(doc);
135
+ const fileHash = fileHashes?.get(doc.relativePath);
136
+ const chunks = this.chunkDocument(doc, fileHash);
135
137
  allChunks.push(...chunks);
136
138
  }
137
139
  return allChunks;
@@ -1,4 +1,4 @@
1
- import { pipeline, env } from '@xenova/transformers';
1
+ import { pipeline, env } from '@huggingface/transformers';
2
2
  // Disable local model loading - use Hugging Face
3
3
  env.allowLocalModels = false;
4
4
  export class Embedder {
@@ -20,6 +20,11 @@ export async function build(options) {
20
20
  }
21
21
  // Load config
22
22
  const config = loadConfig(configPath, inputDir);
23
+ // customCss resolves relative to wherever the config file actually lives,
24
+ // not inputDir — inputDir is often a regenerated staging directory.
25
+ const configDir = configPath && existsSync(configPath)
26
+ ? dirname(resolve(configPath))
27
+ : inputDir;
23
28
  // Override chat enabled setting if specified in CLI
24
29
  if (chatEnabled !== undefined) {
25
30
  config.chat = config.chat || {};
@@ -139,6 +144,19 @@ export async function build(options) {
139
144
  bundledCss += readFileSync(fallbackPath, 'utf-8') + '\n\n';
140
145
  }
141
146
  }
147
+ // Append user-supplied custom CSS last so it overrides theme rules of equal specificity
148
+ if (config.customCss) {
149
+ const customCssPath = resolve(configDir, config.customCss);
150
+ if (existsSync(customCssPath)) {
151
+ bundledCss += readFileSync(customCssPath, 'utf-8') + '\n';
152
+ if (verbose) {
153
+ console.log(`Appended custom CSS: ${customCssPath}`);
154
+ }
155
+ }
156
+ else {
157
+ console.warn(`customCss file not found: ${customCssPath}`);
158
+ }
159
+ }
142
160
  writeFileSync(join(outputCssDir, 'bundle.css'), bundledCss, 'utf-8');
143
161
  if (verbose) {
144
162
  console.log('Styles copied');
@@ -4,13 +4,20 @@ export declare class MarkdownProcessor {
4
4
  private shikiInitialized;
5
5
  constructor();
6
6
  private escapeHtml;
7
+ private decodeHtmlEntities;
7
8
  private setupShiki;
8
9
  /**
9
10
  * Process a markdown file and extract front matter
10
11
  */
11
12
  processFile(filePath: string, inputDir: string, content: string): Promise<ProcessedDocument>;
12
13
  /**
13
- * Extract title from markdown content (first h1) or filename
14
+ * Extract title from markdown content (first h1) or filename.
15
+ *
16
+ * Checks, in order: a markdown `# ` heading, an HTML `<h1>` tag (common
17
+ * when the heading wraps a logo image), and a lone banner image's alt
18
+ * text (READMEs that open with `![Project Name](banner.svg)` instead of
19
+ * a text heading). All matching skips fenced code blocks so shell
20
+ * comments like `# Start the server:` aren't mistaken for headings.
14
21
  */
15
22
  private extractTitle;
16
23
  /**
@@ -9,7 +9,7 @@ import sub from 'markdown-it-sub';
9
9
  import sup from 'markdown-it-sup';
10
10
  import { bundledLanguages, getHighlighter } from 'shiki';
11
11
  import matter from 'gray-matter';
12
- import { relative, basename } from 'path';
12
+ import { relative, basename, dirname } from 'path';
13
13
  export class MarkdownProcessor {
14
14
  md;
15
15
  shikiInitialized = false;
@@ -65,6 +65,15 @@ export class MarkdownProcessor {
65
65
  .replace(/"/g, '&quot;')
66
66
  .replace(/'/g, '&#039;');
67
67
  }
68
+ decodeHtmlEntities(text) {
69
+ return text
70
+ .replace(/&nbsp;/g, ' ')
71
+ .replace(/&amp;/g, '&')
72
+ .replace(/&lt;/g, '<')
73
+ .replace(/&gt;/g, '>')
74
+ .replace(/&quot;/g, '"')
75
+ .replace(/&#0?39;/g, "'");
76
+ }
68
77
  async setupShiki() {
69
78
  if (this.shikiInitialized)
70
79
  return;
@@ -129,16 +138,40 @@ export class MarkdownProcessor {
129
138
  };
130
139
  }
131
140
  /**
132
- * Extract title from markdown content (first h1) or filename
141
+ * Extract title from markdown content (first h1) or filename.
142
+ *
143
+ * Checks, in order: a markdown `# ` heading, an HTML `<h1>` tag (common
144
+ * when the heading wraps a logo image), and a lone banner image's alt
145
+ * text (READMEs that open with `![Project Name](banner.svg)` instead of
146
+ * a text heading). All matching skips fenced code blocks so shell
147
+ * comments like `# Start the server:` aren't mistaken for headings.
133
148
  */
134
149
  extractTitle(content, relativePath) {
135
- const h1Match = content.match(/^#\s+(.+)$/m);
150
+ const withoutCodeFences = content.replace(/^```[\s\S]*?^```/gm, '');
151
+ const h1Match = withoutCodeFences.match(/^#\s+(.+)$/m);
136
152
  if (h1Match) {
137
- return h1Match[1];
153
+ return h1Match[1].trim();
154
+ }
155
+ const htmlH1Match = withoutCodeFences.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
156
+ if (htmlH1Match) {
157
+ const text = this.decodeHtmlEntities(htmlH1Match[1].replace(/<[^>]+>/g, ''))
158
+ .replace(/\s+/g, ' ')
159
+ .trim();
160
+ if (text) {
161
+ return text;
162
+ }
163
+ }
164
+ const bannerImageMatch = withoutCodeFences.match(/^!\[([^\]]+)\]\([^)]*\)\s*$/m);
165
+ if (bannerImageMatch) {
166
+ return bannerImageMatch[1].trim();
138
167
  }
139
- // Fallback to filename
140
- return basename(relativePath, '.md')
141
- .replace(/-/g, ' ')
168
+ // Fallback to filename, or the parent directory name for README/index
169
+ // files where the filename itself carries no useful title.
170
+ const base = basename(relativePath, '.md');
171
+ const parentDir = basename(dirname(relativePath));
172
+ const name = /^(readme|index)$/i.test(base) && parentDir !== '.' ? parentDir : base;
173
+ return name
174
+ .replace(/[-_]/g, ' ')
142
175
  .replace(/\b\w/g, (char) => char.toUpperCase());
143
176
  }
144
177
  /**
@@ -10,13 +10,28 @@ export declare class SiteGenerator {
10
10
  */
11
11
  generate(inputDir: string, outputDir: string, config: BotdocsConfig): Promise<ProcessedDocument[]>;
12
12
  /**
13
- * Build navigation structure from documents
13
+ * Build navigation structure from documents, grouping by top-level
14
+ * folder so e.g. every doc under `path-of-python/` (its README plus
15
+ * anything in `path-of-python/docs/`) nests under one "Path of Python"
16
+ * entry instead of interleaving flat with every other folder's pages.
17
+ * Files at the root (no folder) stay flat, top-level entries.
14
18
  */
15
19
  private buildNavigation;
16
20
  /**
17
- * Render navigation HTML
21
+ * Render navigation HTML. Folder groups only expand their children when
22
+ * the current page is the group's root or one of its children — every
23
+ * other page sees the group collapsed to a single link that leads to
24
+ * its root README.
18
25
  */
19
26
  private renderNavigation;
27
+ /**
28
+ * Compute prev/next page links that walk the top-level navigation
29
+ * (root README to root README) rather than the flat, alphabetical
30
+ * document list — so paging from a folder's root never dips into that
31
+ * folder's children. Paging from within a group's children walks those
32
+ * siblings first, then rolls into the next top-level entry.
33
+ */
34
+ private buildPageSequence;
20
35
  /**
21
36
  * Get all processed documents
22
37
  */
@@ -1,10 +1,13 @@
1
1
  import { readFileSync, writeFileSync, readdirSync, mkdirSync, cpSync } from 'fs';
2
- import { join, dirname, resolve } from 'path';
2
+ import { join, dirname, resolve, basename } from 'path';
3
3
  import { MarkdownProcessor } from './markdown-processor.js';
4
4
  import { TemplateEngine } from './template-engine.js';
5
5
  import { fileURLToPath } from 'url';
6
6
  const __filename = fileURLToPath(import.meta.url);
7
7
  const __dirname = dirname(__filename);
8
+ function stripHtml(value) {
9
+ return value.replace(/<[^>]*>/g, '');
10
+ }
8
11
  export class SiteGenerator {
9
12
  processor;
10
13
  templateEngine;
@@ -42,6 +45,7 @@ export class SiteGenerator {
42
45
  this.documents.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
43
46
  // Generate navigation
44
47
  const navigation = this.buildNavigation(this.documents);
48
+ const pageSequence = this.buildPageSequence(navigation);
45
49
  // Load templates from source directory (not dist)
46
50
  // From dist/src/builder, go to project root, then to src/templates
47
51
  const templatesDir = resolve(__dirname, '../../../src/templates');
@@ -54,8 +58,7 @@ export class SiteGenerator {
54
58
  const doc = this.documents[i];
55
59
  const isIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
56
60
  // Prepare navigation data
57
- const prevDoc = i > 0 ? this.documents[i - 1] : null;
58
- const nextDoc = i < this.documents.length - 1 ? this.documents[i + 1] : null;
61
+ const adjacent = pageSequence.get(doc.url);
59
62
  // Render document content
60
63
  const contentTemplate = isIndex ? indexTemplate : docPageTemplate;
61
64
  const content = this.templateEngine.renderWithLoops(contentTemplate, {
@@ -70,23 +73,17 @@ export class SiteGenerator {
70
73
  url: d.url,
71
74
  }))
72
75
  : undefined,
73
- prevPage: prevDoc
74
- ? {
75
- title: prevDoc.metadata.title,
76
- url: prevDoc.url,
77
- }
78
- : undefined,
79
- nextPage: nextDoc
80
- ? {
81
- title: nextDoc.metadata.title,
82
- url: nextDoc.url,
83
- }
84
- : undefined,
76
+ prevPage: adjacent?.prev,
77
+ nextPage: adjacent?.next,
85
78
  });
86
79
  // Render full page with layout
87
80
  const html = this.templateEngine.renderWithLoops(layoutTemplate, {
88
81
  title: doc.metadata.title || 'Documentation',
89
- description: doc.metadata.description || config.description || '',
82
+ // The template engine doesn't escape interpolated values, so
83
+ // config.description may contain markup (e.g. a hotlink) meant
84
+ // for the visible siteDescription below — strip it here since
85
+ // this one lands inside a <meta content="..."> attribute.
86
+ description: stripHtml(doc.metadata.description || config.description || ''),
90
87
  siteTitle: config.title || 'Documentation',
91
88
  siteDescription: config.description || '',
92
89
  content,
@@ -112,22 +109,52 @@ export class SiteGenerator {
112
109
  return this.documents;
113
110
  }
114
111
  /**
115
- * Build navigation structure from documents
112
+ * Build navigation structure from documents, grouping by top-level
113
+ * folder so e.g. every doc under `path-of-python/` (its README plus
114
+ * anything in `path-of-python/docs/`) nests under one "Path of Python"
115
+ * entry instead of interleaving flat with every other folder's pages.
116
+ * Files at the root (no folder) stay flat, top-level entries.
116
117
  */
117
118
  buildNavigation(documents) {
118
119
  const nav = [];
120
+ const groups = new Map();
119
121
  for (const doc of documents) {
120
122
  const parts = doc.relativePath.split('/');
121
- const title = doc.metadata.title || parts[parts.length - 1].replace(/\.md$/, '');
122
- nav.push({
123
- title,
124
- url: doc.url,
125
- });
123
+ const title = doc.metadata.title || basename(doc.relativePath, '.md');
124
+ const isRootIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
125
+ if (parts.length === 1) {
126
+ if (!isRootIndex) {
127
+ nav.push({ title, url: doc.url });
128
+ }
129
+ continue;
130
+ }
131
+ const folder = parts[0];
132
+ const restOfPath = parts.slice(1).join('/');
133
+ const isFolderIndex = restOfPath === 'README.md' || restOfPath === 'index.md';
134
+ let group = groups.get(folder);
135
+ if (!group) {
136
+ group = {
137
+ title: folder.replace(/[-_]/g, ' ').replace(/\b\w/g, (char) => char.toUpperCase()),
138
+ url: doc.url,
139
+ children: [],
140
+ };
141
+ groups.set(folder, group);
142
+ nav.push(group);
143
+ }
144
+ if (isFolderIndex) {
145
+ group.url = doc.url;
146
+ }
147
+ else {
148
+ group.children.push({ title, url: doc.url });
149
+ }
126
150
  }
127
151
  return nav;
128
152
  }
129
153
  /**
130
- * Render navigation HTML
154
+ * Render navigation HTML. Folder groups only expand their children when
155
+ * the current page is the group's root or one of its children — every
156
+ * other page sees the group collapsed to a single link that leads to
157
+ * its root README.
131
158
  */
132
159
  renderNavigation(items, currentUrl) {
133
160
  if (items.length === 0)
@@ -135,10 +162,14 @@ export class SiteGenerator {
135
162
  let html = '<ul class="nav-list">';
136
163
  for (const item of items) {
137
164
  const isActive = item.url === currentUrl;
138
- const activeClass = isActive ? ' class="active"' : '';
139
- html += `<li${activeClass}>`;
140
- html += `<a href="${item.url}"${activeClass}>${item.title}</a>`;
141
- if (item.children && item.children.length > 0) {
165
+ const hasChildren = !!item.children && item.children.length > 0;
166
+ const isExpanded = hasChildren && (isActive || item.children.some((child) => child.url === currentUrl));
167
+ const liClasses = [isActive && 'active', hasChildren && 'nav-group']
168
+ .filter(Boolean)
169
+ .join(' ');
170
+ html += `<li${liClasses ? ` class="${liClasses}"` : ''}>`;
171
+ html += `<a href="${item.url}"${isActive ? ' class="active"' : ''}>${item.title}</a>`;
172
+ if (isExpanded) {
142
173
  html += this.renderNavigation(item.children, currentUrl);
143
174
  }
144
175
  html += '</li>';
@@ -146,6 +177,30 @@ export class SiteGenerator {
146
177
  html += '</ul>';
147
178
  return html;
148
179
  }
180
+ /**
181
+ * Compute prev/next page links that walk the top-level navigation
182
+ * (root README to root README) rather than the flat, alphabetical
183
+ * document list — so paging from a folder's root never dips into that
184
+ * folder's children. Paging from within a group's children walks those
185
+ * siblings first, then rolls into the next top-level entry.
186
+ */
187
+ buildPageSequence(navigation) {
188
+ const positions = new Map();
189
+ const asLink = (item) => ({ title: item.title, url: item.url });
190
+ for (let i = 0; i < navigation.length; i++) {
191
+ const item = navigation[i];
192
+ const prevTop = i > 0 ? asLink(navigation[i - 1]) : undefined;
193
+ const nextTop = i < navigation.length - 1 ? asLink(navigation[i + 1]) : undefined;
194
+ positions.set(item.url, { prev: prevTop, next: nextTop });
195
+ const children = item.children ?? [];
196
+ for (let j = 0; j < children.length; j++) {
197
+ const prev = j === 0 ? asLink(item) : asLink(children[j - 1]);
198
+ const next = j < children.length - 1 ? asLink(children[j + 1]) : nextTop;
199
+ positions.set(children[j].url, { prev, next });
200
+ }
201
+ }
202
+ return positions;
203
+ }
149
204
  /**
150
205
  * Get all processed documents
151
206
  */
@@ -10,9 +10,21 @@ export declare class VectorDBBuilder {
10
10
  private embedder;
11
11
  constructor(options?: VectorDBBuilderOptions);
12
12
  /**
13
- * Build vector database from documents
13
+ * Build vector database from documents (with incremental build support)
14
14
  */
15
15
  build(documents: ProcessedDocument[], outputDir: string, verbose?: boolean): Promise<VectorDatabase>;
16
+ /**
17
+ * Load existing vector database from disk
18
+ */
19
+ private loadExistingDatabase;
20
+ /**
21
+ * Build a map of sourceFile -> chunks for quick lookup
22
+ */
23
+ private buildCacheMap;
24
+ /**
25
+ * Hash file content for change detection
26
+ */
27
+ private hashContent;
16
28
  /**
17
29
  * Generate unique ID for a chunk
18
30
  */
@@ -1,6 +1,6 @@
1
1
  import { Chunker } from './chunker.js';
2
2
  import { Embedder } from './embedder.js';
3
- import { writeFileSync } from 'fs';
3
+ import { writeFileSync, existsSync, readFileSync } from 'fs';
4
4
  import { join } from 'path';
5
5
  import { createHash } from 'crypto';
6
6
  export class VectorDBBuilder {
@@ -14,43 +14,130 @@ export class VectorDBBuilder {
14
14
  this.embedder = new Embedder(options.modelName);
15
15
  }
16
16
  /**
17
- * Build vector database from documents
17
+ * Build vector database from documents (with incremental build support)
18
18
  */
19
19
  async build(documents, outputDir, verbose = false) {
20
20
  if (verbose) {
21
21
  console.log('Building vector database...');
22
22
  }
23
- // Step 1: Chunk documents
24
- console.log('Chunking documents...');
25
- const chunks = this.chunker.chunkDocuments(documents);
26
- console.log(`Created ${chunks.length} chunks`);
27
- // Step 2: Generate embeddings
28
- console.log('Generating embeddings...');
29
- await this.embedder.initialize();
30
- const texts = chunks.map((chunk) => chunk.text);
31
- const embeddings = await this.embedder.embedBatch(texts, 10);
32
- // Step 3: Create document chunks with embeddings
33
- const documentChunks = chunks.map((chunk, index) => ({
34
- id: this.generateChunkId(chunk.text, chunk.metadata.sourceFile, index),
35
- text: chunk.text,
36
- embedding: embeddings[index],
37
- metadata: chunk.metadata,
38
- }));
39
- // Step 4: Create vector database
23
+ const outputPath = join(outputDir, 'vector-db.json');
24
+ // Step 1: Load existing vector database for incremental builds
25
+ const existingDB = this.loadExistingDatabase(outputPath, verbose);
26
+ const cachedChunks = this.buildCacheMap(existingDB);
27
+ // Step 2: Hash documents and determine what needs rebuilding
28
+ const fileHashes = new Map();
29
+ const changedDocs = [];
30
+ const unchangedDocs = [];
31
+ for (const doc of documents) {
32
+ const hash = this.hashContent(doc.content);
33
+ fileHashes.set(doc.relativePath, hash);
34
+ const cachedDocChunks = cachedChunks.get(doc.relativePath);
35
+ if (cachedDocChunks && cachedDocChunks[0]?.metadata.fileHash === hash) {
36
+ unchangedDocs.push(doc);
37
+ if (verbose) {
38
+ console.log(` ✓ Cached: ${doc.relativePath}`);
39
+ }
40
+ }
41
+ else {
42
+ changedDocs.push(doc);
43
+ if (verbose) {
44
+ console.log(` ⟳ Changed: ${doc.relativePath}`);
45
+ }
46
+ }
47
+ }
48
+ console.log(`Unchanged: ${unchangedDocs.length}, Changed: ${changedDocs.length}, Total: ${documents.length}`);
49
+ // Step 3: Reuse cached chunks for unchanged documents
50
+ const reusedChunks = [];
51
+ for (const doc of unchangedDocs) {
52
+ const chunks = cachedChunks.get(doc.relativePath);
53
+ if (chunks) {
54
+ reusedChunks.push(...chunks);
55
+ }
56
+ }
57
+ // Step 4: Process changed documents only
58
+ let newChunks = [];
59
+ if (changedDocs.length > 0) {
60
+ console.log('Chunking changed documents...');
61
+ const chunks = this.chunker.chunkDocuments(changedDocs, fileHashes);
62
+ console.log(`Created ${chunks.length} chunks from ${changedDocs.length} changed documents`);
63
+ // Generate embeddings only for changed documents
64
+ console.log('Generating embeddings for changed documents...');
65
+ await this.embedder.initialize();
66
+ const texts = chunks.map((chunk) => chunk.text);
67
+ const embeddings = await this.embedder.embedBatch(texts, 10);
68
+ // Create document chunks with embeddings
69
+ newChunks = chunks.map((chunk, index) => ({
70
+ id: this.generateChunkId(chunk.text, chunk.metadata.sourceFile, index),
71
+ text: chunk.text,
72
+ embedding: embeddings[index],
73
+ metadata: chunk.metadata,
74
+ }));
75
+ }
76
+ else {
77
+ console.log('No changes detected - reusing all cached embeddings');
78
+ }
79
+ // Step 5: Merge cached and new chunks
80
+ const allChunks = [...reusedChunks, ...newChunks];
81
+ // Step 6: Create vector database
40
82
  const vectorDB = {
41
83
  version: '1.0',
42
84
  model: this.embedder.getModelName(),
43
85
  dimension: this.embedder.getDimension(),
44
- chunks: documentChunks,
86
+ chunks: allChunks,
45
87
  };
46
- // Step 5: Write to file
47
- const outputPath = join(outputDir, 'vector-db.json');
88
+ // Step 7: Write to file
48
89
  writeFileSync(outputPath, JSON.stringify(vectorDB, null, 2), 'utf-8');
49
90
  console.log(`Vector database saved: ${outputPath}`);
50
- console.log(`Total chunks: ${vectorDB.chunks.length}`);
91
+ console.log(`Total chunks: ${vectorDB.chunks.length} (${reusedChunks.length} cached, ${newChunks.length} new)`);
51
92
  console.log(`Embedding dimension: ${vectorDB.dimension}`);
52
93
  return vectorDB;
53
94
  }
95
+ /**
96
+ * Load existing vector database from disk
97
+ */
98
+ loadExistingDatabase(outputPath, verbose) {
99
+ if (!existsSync(outputPath)) {
100
+ if (verbose) {
101
+ console.log('No existing vector database found - performing full build');
102
+ }
103
+ return null;
104
+ }
105
+ try {
106
+ const content = readFileSync(outputPath, 'utf-8');
107
+ const db = JSON.parse(content);
108
+ if (verbose) {
109
+ console.log(`Loaded existing vector database with ${db.chunks.length} chunks`);
110
+ }
111
+ return db;
112
+ }
113
+ catch (error) {
114
+ console.warn('Failed to load existing vector database - performing full build');
115
+ return null;
116
+ }
117
+ }
118
+ /**
119
+ * Build a map of sourceFile -> chunks for quick lookup
120
+ */
121
+ buildCacheMap(db) {
122
+ const cache = new Map();
123
+ if (!db) {
124
+ return cache;
125
+ }
126
+ for (const chunk of db.chunks) {
127
+ const sourceFile = chunk.metadata.sourceFile;
128
+ if (!cache.has(sourceFile)) {
129
+ cache.set(sourceFile, []);
130
+ }
131
+ cache.get(sourceFile).push(chunk);
132
+ }
133
+ return cache;
134
+ }
135
+ /**
136
+ * Hash file content for change detection
137
+ */
138
+ hashContent(content) {
139
+ return createHash('md5').update(content).digest('hex');
140
+ }
54
141
  /**
55
142
  * Generate unique ID for a chunk
56
143
  */
@@ -18,7 +18,7 @@ program
18
18
  .option('-o, --output <dir>', 'Output directory for generated site', defaultOptions.output)
19
19
  .option('--no-chat', 'Disable AI chatbot functionality')
20
20
  .option('-c, --config <file>', 'Path to config file (botdocs.config.json)')
21
- .option('-t, --theme <theme>', 'Theme to use (classic, material, minimal, slate, modern)', defaultOptions.theme)
21
+ .option('-t, --theme <theme>', 'Theme to use (classic, material, minimal, slate, modern); overrides config file if set')
22
22
  .option('-v, --verbose', 'Enable verbose logging')
23
23
  .action(async (input, options) => {
24
24
  try {
@@ -29,7 +29,7 @@ program
29
29
  console.log(`Input: ${inputDir}`);
30
30
  console.log(`Output: ${outputDir}`);
31
31
  console.log(`Chat enabled: ${!options.noChat}`);
32
- console.log(`Theme: ${options.theme || defaultOptions.theme}`);
32
+ console.log(`Theme override: ${options.theme || '(none — using config file or default)'}`);
33
33
  }
34
34
  await build({
35
35
  inputDir,
@@ -2,5 +2,4 @@ export const defaultOptions = {
2
2
  output: 'output',
3
3
  noChat: false,
4
4
  verbose: false,
5
- theme: 'classic',
6
5
  };
@@ -3,6 +3,7 @@ export interface BotdocsConfig {
3
3
  title?: string;
4
4
  description?: string;
5
5
  theme?: Theme;
6
+ customCss?: string;
6
7
  attribution?: boolean;
7
8
  chat?: {
8
9
  enabled?: boolean;
@@ -6,6 +6,7 @@ export interface ChunkMetadata {
6
6
  url: string;
7
7
  startLine?: number;
8
8
  endLine?: number;
9
+ fileHash?: string;
9
10
  }
10
11
  export interface DocumentChunk {
11
12
  id: string;