botdocs 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/builder/chunker.d.ts +2 -2
- package/dist/src/builder/chunker.js +9 -7
- package/dist/src/builder/embedder.js +1 -1
- package/dist/src/builder/index.js +18 -0
- package/dist/src/builder/markdown-processor.d.ts +8 -1
- package/dist/src/builder/markdown-processor.js +40 -7
- package/dist/src/builder/site-generator.d.ts +17 -2
- package/dist/src/builder/site-generator.js +82 -27
- package/dist/src/builder/vector-db-builder.d.ts +13 -1
- package/dist/src/builder/vector-db-builder.js +110 -23
- package/dist/src/cli/index.js +2 -2
- package/dist/src/cli/options.js +0 -1
- package/dist/src/types/config.d.ts +1 -0
- package/dist/src/types/vector-db.d.ts +1 -0
- package/dist-client/assets/chatbox-P1j6YP1y.js +7 -0
- package/dist-client/assets/rag-engine-CU9vjCAg.js +1 -0
- package/dist-client/bundle.js +1 -1
- package/dist-client/wasm/ort-wasm-simd-threaded.asyncify.wasm +0 -0
- package/man/botdocs.1 +122 -0
- package/package.json +10 -3
- package/src/styles/chat.css +131 -0
- package/src/styles/themes/classic.css +29 -0
- package/src/styles/themes/material.css +29 -0
- package/src/styles/themes/minimal.css +39 -5
- package/src/styles/themes/modern.css +34 -0
- package/src/styles/themes/slate.css +32 -36
- package/src/templates/layout.html +7 -0
- package/dist-client/assets/chatbox-CmTTiB2Z.js +0 -1
- package/dist-client/assets/rag-engine-L2vj3Y0G.js +0 -7
|
@@ -17,7 +17,7 @@ export declare class Chunker {
|
|
|
17
17
|
/**
|
|
18
18
|
* Chunk a document into semantically meaningful pieces
|
|
19
19
|
*/
|
|
20
|
-
chunkDocument(doc: ProcessedDocument): TextChunk[];
|
|
20
|
+
chunkDocument(doc: ProcessedDocument, fileHash?: string): TextChunk[];
|
|
21
21
|
/**
|
|
22
22
|
* Create a chunk with metadata
|
|
23
23
|
*/
|
|
@@ -37,5 +37,5 @@ export declare class Chunker {
|
|
|
37
37
|
/**
|
|
38
38
|
* Chunk multiple documents
|
|
39
39
|
*/
|
|
40
|
-
chunkDocuments(documents: ProcessedDocument[]): TextChunk[];
|
|
40
|
+
chunkDocuments(documents: ProcessedDocument[], fileHashes?: Map<string, string>): TextChunk[];
|
|
41
41
|
}
|
|
@@ -12,7 +12,7 @@ export class Chunker {
|
|
|
12
12
|
/**
|
|
13
13
|
* Chunk a document into semantically meaningful pieces
|
|
14
14
|
*/
|
|
15
|
-
chunkDocument(doc) {
|
|
15
|
+
chunkDocument(doc, fileHash) {
|
|
16
16
|
const chunks = [];
|
|
17
17
|
const lines = doc.content.split('\n');
|
|
18
18
|
let currentChunk = [];
|
|
@@ -47,7 +47,7 @@ export class Chunker {
|
|
|
47
47
|
if (headingMatch) {
|
|
48
48
|
// Save current chunk if it exists
|
|
49
49
|
if (currentChunk.length > 0) {
|
|
50
|
-
chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
|
|
50
|
+
chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading, fileHash));
|
|
51
51
|
}
|
|
52
52
|
// Start new chunk with this heading
|
|
53
53
|
currentHeading = headingMatch[2];
|
|
@@ -60,7 +60,7 @@ export class Chunker {
|
|
|
60
60
|
if (tokenCount >= this.options.maxChunkSize) {
|
|
61
61
|
// Split chunk
|
|
62
62
|
const chunkText = currentChunk.join('\n');
|
|
63
|
-
chunks.push(this.createChunk(chunkText, doc, currentHeading));
|
|
63
|
+
chunks.push(this.createChunk(chunkText, doc, currentHeading, fileHash));
|
|
64
64
|
// Create overlap for next chunk
|
|
65
65
|
const overlapLines = this.getOverlapLines(currentChunk, this.options.chunkOverlap);
|
|
66
66
|
currentChunk = overlapLines;
|
|
@@ -69,14 +69,14 @@ export class Chunker {
|
|
|
69
69
|
}
|
|
70
70
|
// Add final chunk
|
|
71
71
|
if (currentChunk.length > 0) {
|
|
72
|
-
chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading));
|
|
72
|
+
chunks.push(this.createChunk(currentChunk.join('\n'), doc, currentHeading, fileHash));
|
|
73
73
|
}
|
|
74
74
|
return chunks.filter((chunk) => chunk.text.trim().length > 0);
|
|
75
75
|
}
|
|
76
76
|
/**
|
|
77
77
|
* Create a chunk with metadata
|
|
78
78
|
*/
|
|
79
|
-
createChunk(text, doc, heading) {
|
|
79
|
+
createChunk(text, doc, heading, fileHash) {
|
|
80
80
|
return {
|
|
81
81
|
text: text.trim(),
|
|
82
82
|
metadata: {
|
|
@@ -85,6 +85,7 @@ export class Chunker {
|
|
|
85
85
|
heading,
|
|
86
86
|
headingId: heading ? this.slugify(heading) : undefined,
|
|
87
87
|
url: doc.url,
|
|
88
|
+
fileHash,
|
|
88
89
|
},
|
|
89
90
|
};
|
|
90
91
|
}
|
|
@@ -128,10 +129,11 @@ export class Chunker {
|
|
|
128
129
|
/**
|
|
129
130
|
* Chunk multiple documents
|
|
130
131
|
*/
|
|
131
|
-
chunkDocuments(documents) {
|
|
132
|
+
chunkDocuments(documents, fileHashes) {
|
|
132
133
|
const allChunks = [];
|
|
133
134
|
for (const doc of documents) {
|
|
134
|
-
const
|
|
135
|
+
const fileHash = fileHashes?.get(doc.relativePath);
|
|
136
|
+
const chunks = this.chunkDocument(doc, fileHash);
|
|
135
137
|
allChunks.push(...chunks);
|
|
136
138
|
}
|
|
137
139
|
return allChunks;
|
|
@@ -20,6 +20,11 @@ export async function build(options) {
|
|
|
20
20
|
}
|
|
21
21
|
// Load config
|
|
22
22
|
const config = loadConfig(configPath, inputDir);
|
|
23
|
+
// customCss resolves relative to wherever the config file actually lives,
|
|
24
|
+
// not inputDir — inputDir is often a regenerated staging directory.
|
|
25
|
+
const configDir = configPath && existsSync(configPath)
|
|
26
|
+
? dirname(resolve(configPath))
|
|
27
|
+
: inputDir;
|
|
23
28
|
// Override chat enabled setting if specified in CLI
|
|
24
29
|
if (chatEnabled !== undefined) {
|
|
25
30
|
config.chat = config.chat || {};
|
|
@@ -139,6 +144,19 @@ export async function build(options) {
|
|
|
139
144
|
bundledCss += readFileSync(fallbackPath, 'utf-8') + '\n\n';
|
|
140
145
|
}
|
|
141
146
|
}
|
|
147
|
+
// Append user-supplied custom CSS last so it overrides theme rules of equal specificity
|
|
148
|
+
if (config.customCss) {
|
|
149
|
+
const customCssPath = resolve(configDir, config.customCss);
|
|
150
|
+
if (existsSync(customCssPath)) {
|
|
151
|
+
bundledCss += readFileSync(customCssPath, 'utf-8') + '\n';
|
|
152
|
+
if (verbose) {
|
|
153
|
+
console.log(`Appended custom CSS: ${customCssPath}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
console.warn(`customCss file not found: ${customCssPath}`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
142
160
|
writeFileSync(join(outputCssDir, 'bundle.css'), bundledCss, 'utf-8');
|
|
143
161
|
if (verbose) {
|
|
144
162
|
console.log('Styles copied');
|
|
@@ -4,13 +4,20 @@ export declare class MarkdownProcessor {
|
|
|
4
4
|
private shikiInitialized;
|
|
5
5
|
constructor();
|
|
6
6
|
private escapeHtml;
|
|
7
|
+
private decodeHtmlEntities;
|
|
7
8
|
private setupShiki;
|
|
8
9
|
/**
|
|
9
10
|
* Process a markdown file and extract front matter
|
|
10
11
|
*/
|
|
11
12
|
processFile(filePath: string, inputDir: string, content: string): Promise<ProcessedDocument>;
|
|
12
13
|
/**
|
|
13
|
-
* Extract title from markdown content (first h1) or filename
|
|
14
|
+
* Extract title from markdown content (first h1) or filename.
|
|
15
|
+
*
|
|
16
|
+
* Checks, in order: a markdown `# ` heading, an HTML `<h1>` tag (common
|
|
17
|
+
* when the heading wraps a logo image), and a lone banner image's alt
|
|
18
|
+
* text (READMEs that open with `` instead of
|
|
19
|
+
* a text heading). All matching skips fenced code blocks so shell
|
|
20
|
+
* comments like `# Start the server:` aren't mistaken for headings.
|
|
14
21
|
*/
|
|
15
22
|
private extractTitle;
|
|
16
23
|
/**
|
|
@@ -9,7 +9,7 @@ import sub from 'markdown-it-sub';
|
|
|
9
9
|
import sup from 'markdown-it-sup';
|
|
10
10
|
import { bundledLanguages, getHighlighter } from 'shiki';
|
|
11
11
|
import matter from 'gray-matter';
|
|
12
|
-
import { relative, basename } from 'path';
|
|
12
|
+
import { relative, basename, dirname } from 'path';
|
|
13
13
|
export class MarkdownProcessor {
|
|
14
14
|
md;
|
|
15
15
|
shikiInitialized = false;
|
|
@@ -65,6 +65,15 @@ export class MarkdownProcessor {
|
|
|
65
65
|
.replace(/"/g, '"')
|
|
66
66
|
.replace(/'/g, ''');
|
|
67
67
|
}
|
|
68
|
+
decodeHtmlEntities(text) {
|
|
69
|
+
return text
|
|
70
|
+
.replace(/ /g, ' ')
|
|
71
|
+
.replace(/&/g, '&')
|
|
72
|
+
.replace(/</g, '<')
|
|
73
|
+
.replace(/>/g, '>')
|
|
74
|
+
.replace(/"/g, '"')
|
|
75
|
+
.replace(/�?39;/g, "'");
|
|
76
|
+
}
|
|
68
77
|
async setupShiki() {
|
|
69
78
|
if (this.shikiInitialized)
|
|
70
79
|
return;
|
|
@@ -129,16 +138,40 @@ export class MarkdownProcessor {
|
|
|
129
138
|
};
|
|
130
139
|
}
|
|
131
140
|
/**
|
|
132
|
-
* Extract title from markdown content (first h1) or filename
|
|
141
|
+
* Extract title from markdown content (first h1) or filename.
|
|
142
|
+
*
|
|
143
|
+
* Checks, in order: a markdown `# ` heading, an HTML `<h1>` tag (common
|
|
144
|
+
* when the heading wraps a logo image), and a lone banner image's alt
|
|
145
|
+
* text (READMEs that open with `` instead of
|
|
146
|
+
* a text heading). All matching skips fenced code blocks so shell
|
|
147
|
+
* comments like `# Start the server:` aren't mistaken for headings.
|
|
133
148
|
*/
|
|
134
149
|
extractTitle(content, relativePath) {
|
|
135
|
-
const
|
|
150
|
+
const withoutCodeFences = content.replace(/^```[\s\S]*?^```/gm, '');
|
|
151
|
+
const h1Match = withoutCodeFences.match(/^#\s+(.+)$/m);
|
|
136
152
|
if (h1Match) {
|
|
137
|
-
return h1Match[1];
|
|
153
|
+
return h1Match[1].trim();
|
|
154
|
+
}
|
|
155
|
+
const htmlH1Match = withoutCodeFences.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
|
|
156
|
+
if (htmlH1Match) {
|
|
157
|
+
const text = this.decodeHtmlEntities(htmlH1Match[1].replace(/<[^>]+>/g, ''))
|
|
158
|
+
.replace(/\s+/g, ' ')
|
|
159
|
+
.trim();
|
|
160
|
+
if (text) {
|
|
161
|
+
return text;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const bannerImageMatch = withoutCodeFences.match(/^!\[([^\]]+)\]\([^)]*\)\s*$/m);
|
|
165
|
+
if (bannerImageMatch) {
|
|
166
|
+
return bannerImageMatch[1].trim();
|
|
138
167
|
}
|
|
139
|
-
// Fallback to filename
|
|
140
|
-
|
|
141
|
-
|
|
168
|
+
// Fallback to filename, or the parent directory name for README/index
|
|
169
|
+
// files where the filename itself carries no useful title.
|
|
170
|
+
const base = basename(relativePath, '.md');
|
|
171
|
+
const parentDir = basename(dirname(relativePath));
|
|
172
|
+
const name = /^(readme|index)$/i.test(base) && parentDir !== '.' ? parentDir : base;
|
|
173
|
+
return name
|
|
174
|
+
.replace(/[-_]/g, ' ')
|
|
142
175
|
.replace(/\b\w/g, (char) => char.toUpperCase());
|
|
143
176
|
}
|
|
144
177
|
/**
|
|
@@ -10,13 +10,28 @@ export declare class SiteGenerator {
|
|
|
10
10
|
*/
|
|
11
11
|
generate(inputDir: string, outputDir: string, config: BotdocsConfig): Promise<ProcessedDocument[]>;
|
|
12
12
|
/**
|
|
13
|
-
* Build navigation structure from documents
|
|
13
|
+
* Build navigation structure from documents, grouping by top-level
|
|
14
|
+
* folder so e.g. every doc under `path-of-python/` (its README plus
|
|
15
|
+
* anything in `path-of-python/docs/`) nests under one "Path of Python"
|
|
16
|
+
* entry instead of interleaving flat with every other folder's pages.
|
|
17
|
+
* Files at the root (no folder) stay flat, top-level entries.
|
|
14
18
|
*/
|
|
15
19
|
private buildNavigation;
|
|
16
20
|
/**
|
|
17
|
-
* Render navigation HTML
|
|
21
|
+
* Render navigation HTML. Folder groups only expand their children when
|
|
22
|
+
* the current page is the group's root or one of its children — every
|
|
23
|
+
* other page sees the group collapsed to a single link that leads to
|
|
24
|
+
* its root README.
|
|
18
25
|
*/
|
|
19
26
|
private renderNavigation;
|
|
27
|
+
/**
|
|
28
|
+
* Compute prev/next page links that walk the top-level navigation
|
|
29
|
+
* (root README to root README) rather than the flat, alphabetical
|
|
30
|
+
* document list — so paging from a folder's root never dips into that
|
|
31
|
+
* folder's children. Paging from within a group's children walks those
|
|
32
|
+
* siblings first, then rolls into the next top-level entry.
|
|
33
|
+
*/
|
|
34
|
+
private buildPageSequence;
|
|
20
35
|
/**
|
|
21
36
|
* Get all processed documents
|
|
22
37
|
*/
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { readFileSync, writeFileSync, readdirSync, mkdirSync, cpSync } from 'fs';
|
|
2
|
-
import { join, dirname, resolve } from 'path';
|
|
2
|
+
import { join, dirname, resolve, basename } from 'path';
|
|
3
3
|
import { MarkdownProcessor } from './markdown-processor.js';
|
|
4
4
|
import { TemplateEngine } from './template-engine.js';
|
|
5
5
|
import { fileURLToPath } from 'url';
|
|
6
6
|
const __filename = fileURLToPath(import.meta.url);
|
|
7
7
|
const __dirname = dirname(__filename);
|
|
8
|
+
function stripHtml(value) {
|
|
9
|
+
return value.replace(/<[^>]*>/g, '');
|
|
10
|
+
}
|
|
8
11
|
export class SiteGenerator {
|
|
9
12
|
processor;
|
|
10
13
|
templateEngine;
|
|
@@ -42,6 +45,7 @@ export class SiteGenerator {
|
|
|
42
45
|
this.documents.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
|
|
43
46
|
// Generate navigation
|
|
44
47
|
const navigation = this.buildNavigation(this.documents);
|
|
48
|
+
const pageSequence = this.buildPageSequence(navigation);
|
|
45
49
|
// Load templates from source directory (not dist)
|
|
46
50
|
// From dist/src/builder, go to project root, then to src/templates
|
|
47
51
|
const templatesDir = resolve(__dirname, '../../../src/templates');
|
|
@@ -54,8 +58,7 @@ export class SiteGenerator {
|
|
|
54
58
|
const doc = this.documents[i];
|
|
55
59
|
const isIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
|
|
56
60
|
// Prepare navigation data
|
|
57
|
-
const
|
|
58
|
-
const nextDoc = i < this.documents.length - 1 ? this.documents[i + 1] : null;
|
|
61
|
+
const adjacent = pageSequence.get(doc.url);
|
|
59
62
|
// Render document content
|
|
60
63
|
const contentTemplate = isIndex ? indexTemplate : docPageTemplate;
|
|
61
64
|
const content = this.templateEngine.renderWithLoops(contentTemplate, {
|
|
@@ -70,23 +73,17 @@ export class SiteGenerator {
|
|
|
70
73
|
url: d.url,
|
|
71
74
|
}))
|
|
72
75
|
: undefined,
|
|
73
|
-
prevPage:
|
|
74
|
-
|
|
75
|
-
title: prevDoc.metadata.title,
|
|
76
|
-
url: prevDoc.url,
|
|
77
|
-
}
|
|
78
|
-
: undefined,
|
|
79
|
-
nextPage: nextDoc
|
|
80
|
-
? {
|
|
81
|
-
title: nextDoc.metadata.title,
|
|
82
|
-
url: nextDoc.url,
|
|
83
|
-
}
|
|
84
|
-
: undefined,
|
|
76
|
+
prevPage: adjacent?.prev,
|
|
77
|
+
nextPage: adjacent?.next,
|
|
85
78
|
});
|
|
86
79
|
// Render full page with layout
|
|
87
80
|
const html = this.templateEngine.renderWithLoops(layoutTemplate, {
|
|
88
81
|
title: doc.metadata.title || 'Documentation',
|
|
89
|
-
|
|
82
|
+
// The template engine doesn't escape interpolated values, so
|
|
83
|
+
// config.description may contain markup (e.g. a hotlink) meant
|
|
84
|
+
// for the visible siteDescription below — strip it here since
|
|
85
|
+
// this one lands inside a <meta content="..."> attribute.
|
|
86
|
+
description: stripHtml(doc.metadata.description || config.description || ''),
|
|
90
87
|
siteTitle: config.title || 'Documentation',
|
|
91
88
|
siteDescription: config.description || '',
|
|
92
89
|
content,
|
|
@@ -112,22 +109,52 @@ export class SiteGenerator {
|
|
|
112
109
|
return this.documents;
|
|
113
110
|
}
|
|
114
111
|
/**
|
|
115
|
-
* Build navigation structure from documents
|
|
112
|
+
* Build navigation structure from documents, grouping by top-level
|
|
113
|
+
* folder so e.g. every doc under `path-of-python/` (its README plus
|
|
114
|
+
* anything in `path-of-python/docs/`) nests under one "Path of Python"
|
|
115
|
+
* entry instead of interleaving flat with every other folder's pages.
|
|
116
|
+
* Files at the root (no folder) stay flat, top-level entries.
|
|
116
117
|
*/
|
|
117
118
|
buildNavigation(documents) {
|
|
118
119
|
const nav = [];
|
|
120
|
+
const groups = new Map();
|
|
119
121
|
for (const doc of documents) {
|
|
120
122
|
const parts = doc.relativePath.split('/');
|
|
121
|
-
const title = doc.metadata.title ||
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
123
|
+
const title = doc.metadata.title || basename(doc.relativePath, '.md');
|
|
124
|
+
const isRootIndex = doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
|
|
125
|
+
if (parts.length === 1) {
|
|
126
|
+
if (!isRootIndex) {
|
|
127
|
+
nav.push({ title, url: doc.url });
|
|
128
|
+
}
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const folder = parts[0];
|
|
132
|
+
const restOfPath = parts.slice(1).join('/');
|
|
133
|
+
const isFolderIndex = restOfPath === 'README.md' || restOfPath === 'index.md';
|
|
134
|
+
let group = groups.get(folder);
|
|
135
|
+
if (!group) {
|
|
136
|
+
group = {
|
|
137
|
+
title: folder.replace(/[-_]/g, ' ').replace(/\b\w/g, (char) => char.toUpperCase()),
|
|
138
|
+
url: doc.url,
|
|
139
|
+
children: [],
|
|
140
|
+
};
|
|
141
|
+
groups.set(folder, group);
|
|
142
|
+
nav.push(group);
|
|
143
|
+
}
|
|
144
|
+
if (isFolderIndex) {
|
|
145
|
+
group.url = doc.url;
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
group.children.push({ title, url: doc.url });
|
|
149
|
+
}
|
|
126
150
|
}
|
|
127
151
|
return nav;
|
|
128
152
|
}
|
|
129
153
|
/**
|
|
130
|
-
* Render navigation HTML
|
|
154
|
+
* Render navigation HTML. Folder groups only expand their children when
|
|
155
|
+
* the current page is the group's root or one of its children — every
|
|
156
|
+
* other page sees the group collapsed to a single link that leads to
|
|
157
|
+
* its root README.
|
|
131
158
|
*/
|
|
132
159
|
renderNavigation(items, currentUrl) {
|
|
133
160
|
if (items.length === 0)
|
|
@@ -135,10 +162,14 @@ export class SiteGenerator {
|
|
|
135
162
|
let html = '<ul class="nav-list">';
|
|
136
163
|
for (const item of items) {
|
|
137
164
|
const isActive = item.url === currentUrl;
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
165
|
+
const hasChildren = !!item.children && item.children.length > 0;
|
|
166
|
+
const isExpanded = hasChildren && (isActive || item.children.some((child) => child.url === currentUrl));
|
|
167
|
+
const liClasses = [isActive && 'active', hasChildren && 'nav-group']
|
|
168
|
+
.filter(Boolean)
|
|
169
|
+
.join(' ');
|
|
170
|
+
html += `<li${liClasses ? ` class="${liClasses}"` : ''}>`;
|
|
171
|
+
html += `<a href="${item.url}"${isActive ? ' class="active"' : ''}>${item.title}</a>`;
|
|
172
|
+
if (isExpanded) {
|
|
142
173
|
html += this.renderNavigation(item.children, currentUrl);
|
|
143
174
|
}
|
|
144
175
|
html += '</li>';
|
|
@@ -146,6 +177,30 @@ export class SiteGenerator {
|
|
|
146
177
|
html += '</ul>';
|
|
147
178
|
return html;
|
|
148
179
|
}
|
|
180
|
+
/**
|
|
181
|
+
* Compute prev/next page links that walk the top-level navigation
|
|
182
|
+
* (root README to root README) rather than the flat, alphabetical
|
|
183
|
+
* document list — so paging from a folder's root never dips into that
|
|
184
|
+
* folder's children. Paging from within a group's children walks those
|
|
185
|
+
* siblings first, then rolls into the next top-level entry.
|
|
186
|
+
*/
|
|
187
|
+
buildPageSequence(navigation) {
|
|
188
|
+
const positions = new Map();
|
|
189
|
+
const asLink = (item) => ({ title: item.title, url: item.url });
|
|
190
|
+
for (let i = 0; i < navigation.length; i++) {
|
|
191
|
+
const item = navigation[i];
|
|
192
|
+
const prevTop = i > 0 ? asLink(navigation[i - 1]) : undefined;
|
|
193
|
+
const nextTop = i < navigation.length - 1 ? asLink(navigation[i + 1]) : undefined;
|
|
194
|
+
positions.set(item.url, { prev: prevTop, next: nextTop });
|
|
195
|
+
const children = item.children ?? [];
|
|
196
|
+
for (let j = 0; j < children.length; j++) {
|
|
197
|
+
const prev = j === 0 ? asLink(item) : asLink(children[j - 1]);
|
|
198
|
+
const next = j < children.length - 1 ? asLink(children[j + 1]) : nextTop;
|
|
199
|
+
positions.set(children[j].url, { prev, next });
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return positions;
|
|
203
|
+
}
|
|
149
204
|
/**
|
|
150
205
|
* Get all processed documents
|
|
151
206
|
*/
|
|
@@ -10,9 +10,21 @@ export declare class VectorDBBuilder {
|
|
|
10
10
|
private embedder;
|
|
11
11
|
constructor(options?: VectorDBBuilderOptions);
|
|
12
12
|
/**
|
|
13
|
-
* Build vector database from documents
|
|
13
|
+
* Build vector database from documents (with incremental build support)
|
|
14
14
|
*/
|
|
15
15
|
build(documents: ProcessedDocument[], outputDir: string, verbose?: boolean): Promise<VectorDatabase>;
|
|
16
|
+
/**
|
|
17
|
+
* Load existing vector database from disk
|
|
18
|
+
*/
|
|
19
|
+
private loadExistingDatabase;
|
|
20
|
+
/**
|
|
21
|
+
* Build a map of sourceFile -> chunks for quick lookup
|
|
22
|
+
*/
|
|
23
|
+
private buildCacheMap;
|
|
24
|
+
/**
|
|
25
|
+
* Hash file content for change detection
|
|
26
|
+
*/
|
|
27
|
+
private hashContent;
|
|
16
28
|
/**
|
|
17
29
|
* Generate unique ID for a chunk
|
|
18
30
|
*/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Chunker } from './chunker.js';
|
|
2
2
|
import { Embedder } from './embedder.js';
|
|
3
|
-
import { writeFileSync } from 'fs';
|
|
3
|
+
import { writeFileSync, existsSync, readFileSync } from 'fs';
|
|
4
4
|
import { join } from 'path';
|
|
5
5
|
import { createHash } from 'crypto';
|
|
6
6
|
export class VectorDBBuilder {
|
|
@@ -14,43 +14,130 @@ export class VectorDBBuilder {
|
|
|
14
14
|
this.embedder = new Embedder(options.modelName);
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
|
-
* Build vector database from documents
|
|
17
|
+
* Build vector database from documents (with incremental build support)
|
|
18
18
|
*/
|
|
19
19
|
async build(documents, outputDir, verbose = false) {
|
|
20
20
|
if (verbose) {
|
|
21
21
|
console.log('Building vector database...');
|
|
22
22
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
// Step 2:
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
const
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
23
|
+
const outputPath = join(outputDir, 'vector-db.json');
|
|
24
|
+
// Step 1: Load existing vector database for incremental builds
|
|
25
|
+
const existingDB = this.loadExistingDatabase(outputPath, verbose);
|
|
26
|
+
const cachedChunks = this.buildCacheMap(existingDB);
|
|
27
|
+
// Step 2: Hash documents and determine what needs rebuilding
|
|
28
|
+
const fileHashes = new Map();
|
|
29
|
+
const changedDocs = [];
|
|
30
|
+
const unchangedDocs = [];
|
|
31
|
+
for (const doc of documents) {
|
|
32
|
+
const hash = this.hashContent(doc.content);
|
|
33
|
+
fileHashes.set(doc.relativePath, hash);
|
|
34
|
+
const cachedDocChunks = cachedChunks.get(doc.relativePath);
|
|
35
|
+
if (cachedDocChunks && cachedDocChunks[0]?.metadata.fileHash === hash) {
|
|
36
|
+
unchangedDocs.push(doc);
|
|
37
|
+
if (verbose) {
|
|
38
|
+
console.log(` ✓ Cached: ${doc.relativePath}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
changedDocs.push(doc);
|
|
43
|
+
if (verbose) {
|
|
44
|
+
console.log(` ⟳ Changed: ${doc.relativePath}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
console.log(`Unchanged: ${unchangedDocs.length}, Changed: ${changedDocs.length}, Total: ${documents.length}`);
|
|
49
|
+
// Step 3: Reuse cached chunks for unchanged documents
|
|
50
|
+
const reusedChunks = [];
|
|
51
|
+
for (const doc of unchangedDocs) {
|
|
52
|
+
const chunks = cachedChunks.get(doc.relativePath);
|
|
53
|
+
if (chunks) {
|
|
54
|
+
reusedChunks.push(...chunks);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
// Step 4: Process changed documents only
|
|
58
|
+
let newChunks = [];
|
|
59
|
+
if (changedDocs.length > 0) {
|
|
60
|
+
console.log('Chunking changed documents...');
|
|
61
|
+
const chunks = this.chunker.chunkDocuments(changedDocs, fileHashes);
|
|
62
|
+
console.log(`Created ${chunks.length} chunks from ${changedDocs.length} changed documents`);
|
|
63
|
+
// Generate embeddings only for changed documents
|
|
64
|
+
console.log('Generating embeddings for changed documents...');
|
|
65
|
+
await this.embedder.initialize();
|
|
66
|
+
const texts = chunks.map((chunk) => chunk.text);
|
|
67
|
+
const embeddings = await this.embedder.embedBatch(texts, 10);
|
|
68
|
+
// Create document chunks with embeddings
|
|
69
|
+
newChunks = chunks.map((chunk, index) => ({
|
|
70
|
+
id: this.generateChunkId(chunk.text, chunk.metadata.sourceFile, index),
|
|
71
|
+
text: chunk.text,
|
|
72
|
+
embedding: embeddings[index],
|
|
73
|
+
metadata: chunk.metadata,
|
|
74
|
+
}));
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
console.log('No changes detected - reusing all cached embeddings');
|
|
78
|
+
}
|
|
79
|
+
// Step 5: Merge cached and new chunks
|
|
80
|
+
const allChunks = [...reusedChunks, ...newChunks];
|
|
81
|
+
// Step 6: Create vector database
|
|
40
82
|
const vectorDB = {
|
|
41
83
|
version: '1.0',
|
|
42
84
|
model: this.embedder.getModelName(),
|
|
43
85
|
dimension: this.embedder.getDimension(),
|
|
44
|
-
chunks:
|
|
86
|
+
chunks: allChunks,
|
|
45
87
|
};
|
|
46
|
-
// Step
|
|
47
|
-
const outputPath = join(outputDir, 'vector-db.json');
|
|
88
|
+
// Step 7: Write to file
|
|
48
89
|
writeFileSync(outputPath, JSON.stringify(vectorDB, null, 2), 'utf-8');
|
|
49
90
|
console.log(`Vector database saved: ${outputPath}`);
|
|
50
|
-
console.log(`Total chunks: ${vectorDB.chunks.length}`);
|
|
91
|
+
console.log(`Total chunks: ${vectorDB.chunks.length} (${reusedChunks.length} cached, ${newChunks.length} new)`);
|
|
51
92
|
console.log(`Embedding dimension: ${vectorDB.dimension}`);
|
|
52
93
|
return vectorDB;
|
|
53
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Load existing vector database from disk
|
|
97
|
+
*/
|
|
98
|
+
loadExistingDatabase(outputPath, verbose) {
|
|
99
|
+
if (!existsSync(outputPath)) {
|
|
100
|
+
if (verbose) {
|
|
101
|
+
console.log('No existing vector database found - performing full build');
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
try {
|
|
106
|
+
const content = readFileSync(outputPath, 'utf-8');
|
|
107
|
+
const db = JSON.parse(content);
|
|
108
|
+
if (verbose) {
|
|
109
|
+
console.log(`Loaded existing vector database with ${db.chunks.length} chunks`);
|
|
110
|
+
}
|
|
111
|
+
return db;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
console.warn('Failed to load existing vector database - performing full build');
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Build a map of sourceFile -> chunks for quick lookup
|
|
120
|
+
*/
|
|
121
|
+
buildCacheMap(db) {
|
|
122
|
+
const cache = new Map();
|
|
123
|
+
if (!db) {
|
|
124
|
+
return cache;
|
|
125
|
+
}
|
|
126
|
+
for (const chunk of db.chunks) {
|
|
127
|
+
const sourceFile = chunk.metadata.sourceFile;
|
|
128
|
+
if (!cache.has(sourceFile)) {
|
|
129
|
+
cache.set(sourceFile, []);
|
|
130
|
+
}
|
|
131
|
+
cache.get(sourceFile).push(chunk);
|
|
132
|
+
}
|
|
133
|
+
return cache;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Hash file content for change detection
|
|
137
|
+
*/
|
|
138
|
+
hashContent(content) {
|
|
139
|
+
return createHash('md5').update(content).digest('hex');
|
|
140
|
+
}
|
|
54
141
|
/**
|
|
55
142
|
* Generate unique ID for a chunk
|
|
56
143
|
*/
|
package/dist/src/cli/index.js
CHANGED
|
@@ -18,7 +18,7 @@ program
|
|
|
18
18
|
.option('-o, --output <dir>', 'Output directory for generated site', defaultOptions.output)
|
|
19
19
|
.option('--no-chat', 'Disable AI chatbot functionality')
|
|
20
20
|
.option('-c, --config <file>', 'Path to config file (botdocs.config.json)')
|
|
21
|
-
.option('-t, --theme <theme>', 'Theme to use (classic, material, minimal, slate, modern)'
|
|
21
|
+
.option('-t, --theme <theme>', 'Theme to use (classic, material, minimal, slate, modern); overrides config file if set')
|
|
22
22
|
.option('-v, --verbose', 'Enable verbose logging')
|
|
23
23
|
.action(async (input, options) => {
|
|
24
24
|
try {
|
|
@@ -29,7 +29,7 @@ program
|
|
|
29
29
|
console.log(`Input: ${inputDir}`);
|
|
30
30
|
console.log(`Output: ${outputDir}`);
|
|
31
31
|
console.log(`Chat enabled: ${!options.noChat}`);
|
|
32
|
-
console.log(`Theme: ${options.theme ||
|
|
32
|
+
console.log(`Theme override: ${options.theme || '(none — using config file or default)'}`);
|
|
33
33
|
}
|
|
34
34
|
await build({
|
|
35
35
|
inputDir,
|
package/dist/src/cli/options.js
CHANGED