@customize-agent/knowledge 1.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/text-chunker.d.ts +12 -1
- package/dist/chunking/text-chunker.js +219 -41
- package/dist/core/change-tracker.d.ts +1 -0
- package/dist/core/change-tracker.js +21 -0
- package/dist/core/index-state-store.d.ts +37 -1
- package/dist/core/index-state-store.js +300 -28
- package/dist/core/knowledge-base-manager.d.ts +70 -3
- package/dist/core/knowledge-base-manager.js +547 -125
- package/dist/core/multi-project-manager.d.ts +7 -4
- package/dist/core/multi-project-manager.js +35 -23
- package/dist/dedup/dedup-engine.d.ts +3 -0
- package/dist/dedup/dedup-engine.js +12 -2
- package/dist/embedding/embedding-provider.d.ts +1 -0
- package/dist/embedding/embedding-provider.js +16 -1
- package/dist/extraction/content-extractor.d.ts +2 -0
- package/dist/extraction/content-extractor.js +209 -42
- package/dist/extraction/module-resolver.d.ts +17 -0
- package/dist/extraction/module-resolver.js +113 -0
- package/dist/index.d.ts +2 -4
- package/dist/index.js +2 -3
- package/dist/llm/llm-search-provider.d.ts +23 -0
- package/dist/llm/llm-search-provider.js +1 -0
- package/dist/search/federation-search.d.ts +29 -0
- package/dist/search/federation-search.js +8 -1
- package/dist/vector/chroma-store.d.ts +2 -0
- package/dist/vector/chroma-store.js +53 -22
- package/dist/vector/vector-indexer.d.ts +3 -0
- package/dist/vector/vector-indexer.js +23 -0
- package/package.json +11 -3
- package/dist/server/dashboard-client.d.ts +0 -2
- package/dist/server/dashboard-client.js +0 -396
- package/dist/server/dashboard-i18n.d.ts +0 -112
- package/dist/server/dashboard-i18n.js +0 -220
- package/dist/server/dashboard-page.d.ts +0 -6
- package/dist/server/dashboard-page.js +0 -138
- package/dist/server/dashboard-server.d.ts +0 -13
- package/dist/server/dashboard-server.js +0 -225
- package/dist/server/dashboard-styles.d.ts +0 -1
- package/dist/server/dashboard-styles.js +0 -152
|
@@ -3,6 +3,7 @@ import * as fs from 'node:fs';
|
|
|
3
3
|
import * as path from 'node:path';
|
|
4
4
|
export class IndexStateStore {
|
|
5
5
|
db;
|
|
6
|
+
ftsEnabled = false;
|
|
6
7
|
constructor(dbPath) {
|
|
7
8
|
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
8
9
|
this.db = new Database(dbPath);
|
|
@@ -50,7 +51,7 @@ export class IndexStateStore {
|
|
|
50
51
|
listRecords() {
|
|
51
52
|
const rows = this.db.prepare(`
|
|
52
53
|
SELECT * FROM kb_index_state
|
|
53
|
-
WHERE status
|
|
54
|
+
WHERE status != 'deleted'
|
|
54
55
|
ORDER BY category, relative_path
|
|
55
56
|
`).all();
|
|
56
57
|
return rows.map(row => this.rowToRecord(row));
|
|
@@ -59,14 +60,40 @@ export class IndexStateStore {
|
|
|
59
60
|
const now = Date.now();
|
|
60
61
|
const transaction = this.db.transaction(() => {
|
|
61
62
|
this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
|
|
63
|
+
this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
|
|
64
|
+
if (this.ftsEnabled)
|
|
65
|
+
this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
|
|
62
66
|
const insert = this.db.prepare(`
|
|
63
67
|
INSERT INTO kb_chunks (
|
|
64
68
|
id, relative_path, chunk_index, content, category, format,
|
|
65
69
|
collection_name, token_count, section_title, metadata_json, created_at
|
|
66
70
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
67
71
|
`);
|
|
68
|
-
|
|
69
|
-
|
|
72
|
+
const insertFts = this.ftsEnabled ? this.db.prepare(`
|
|
73
|
+
INSERT INTO kb_chunks_fts (id, relative_path, category, format, section_title, content)
|
|
74
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
75
|
+
`) : undefined;
|
|
76
|
+
const insertParent = this.db.prepare(`
|
|
77
|
+
INSERT INTO kb_parent_chunks (
|
|
78
|
+
id, relative_path, parent_id, content, category, format,
|
|
79
|
+
collection_name, section_title, chunk_count, metadata_json, created_at
|
|
80
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
81
|
+
`);
|
|
82
|
+
const parentGroups = new Map();
|
|
83
|
+
const groupedChunks = this.splitParentGroups(relativePath, chunks);
|
|
84
|
+
for (const chunk of groupedChunks) {
|
|
85
|
+
const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
|
|
86
|
+
const group = parentGroups.get(parentId) ?? [];
|
|
87
|
+
group.push(chunk);
|
|
88
|
+
parentGroups.set(parentId, group);
|
|
89
|
+
}
|
|
90
|
+
for (const [parentId, group] of parentGroups.entries()) {
|
|
91
|
+
insertParent.run(parentId, relativePath, parentId, group.map(chunk => chunk.text).join('\n\n---\n\n'), file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind) }), now);
|
|
92
|
+
}
|
|
93
|
+
for (const chunk of groupedChunks) {
|
|
94
|
+
const chunkId = `${relativePath}#${chunk.index}`;
|
|
95
|
+
insert.run(chunkId, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
|
|
96
|
+
insertFts?.run(chunkId, relativePath, file.category, file.format, chunk.sectionTitle ?? '', chunk.text);
|
|
70
97
|
}
|
|
71
98
|
});
|
|
72
99
|
transaction();
|
|
@@ -94,10 +121,78 @@ export class IndexStateStore {
|
|
|
94
121
|
`).all(...params);
|
|
95
122
|
return rows.map(row => this.rowToChunk(row, 0));
|
|
96
123
|
}
|
|
124
|
+
getContextChunks(relativePath, chunkIndex, window = 1) {
|
|
125
|
+
const rows = this.db.prepare(`
|
|
126
|
+
SELECT * FROM kb_chunks
|
|
127
|
+
WHERE relative_path = ? AND chunk_index BETWEEN ? AND ?
|
|
128
|
+
ORDER BY chunk_index
|
|
129
|
+
`).all(relativePath, Math.max(0, chunkIndex - window), chunkIndex + window);
|
|
130
|
+
return rows.map(row => this.rowToChunk(row, 0));
|
|
131
|
+
}
|
|
132
|
+
listParentChunks(relativePath) {
|
|
133
|
+
const rows = this.db.prepare(`
|
|
134
|
+
SELECT * FROM kb_parent_chunks
|
|
135
|
+
WHERE relative_path = ?
|
|
136
|
+
ORDER BY parent_id
|
|
137
|
+
`).all(relativePath);
|
|
138
|
+
return rows.map(row => this.rowToParentChunk(row));
|
|
139
|
+
}
|
|
140
|
+
getParentChunk(relativePath, parentId) {
|
|
141
|
+
const row = this.db.prepare(`
|
|
142
|
+
SELECT * FROM kb_parent_chunks
|
|
143
|
+
WHERE relative_path = ? AND parent_id = ?
|
|
144
|
+
LIMIT 1
|
|
145
|
+
`).get(relativePath, parentId);
|
|
146
|
+
return row ? this.rowToParentChunk(row) : undefined;
|
|
147
|
+
}
|
|
148
|
+
getChunksByParent(relativePath, parentId, limit = 6) {
|
|
149
|
+
const rows = this.db.prepare(`
|
|
150
|
+
SELECT * FROM kb_chunks
|
|
151
|
+
WHERE relative_path = ? AND metadata_json LIKE ?
|
|
152
|
+
ORDER BY chunk_index
|
|
153
|
+
LIMIT ?
|
|
154
|
+
`).all(relativePath, `%"parentId":"${parentId.replace(/[%_]/gu, '')}"%`, limit);
|
|
155
|
+
return rows.map(row => this.rowToChunk(row, 0));
|
|
156
|
+
}
|
|
97
157
|
searchChunks(query, limit = 10) {
|
|
98
158
|
const terms = this.expandSearchTerms(query);
|
|
99
159
|
if (terms.length === 0)
|
|
100
160
|
return [];
|
|
161
|
+
if (this.ftsEnabled) {
|
|
162
|
+
const ftsResults = this.searchChunksFts(terms, limit);
|
|
163
|
+
if (ftsResults.length > 0)
|
|
164
|
+
return ftsResults;
|
|
165
|
+
}
|
|
166
|
+
return this.searchChunksLike(terms, limit);
|
|
167
|
+
}
|
|
168
|
+
searchChunksFts(terms, limit) {
|
|
169
|
+
try {
|
|
170
|
+
const matchQuery = this.toFtsQuery(terms);
|
|
171
|
+
if (!matchQuery)
|
|
172
|
+
return [];
|
|
173
|
+
const rows = this.db.prepare(`
|
|
174
|
+
SELECT c.*, bm25(kb_chunks_fts, 1.2, 0.8, 0.6, 1.0, 2.0) as bm25_score
|
|
175
|
+
FROM kb_chunks_fts
|
|
176
|
+
INNER JOIN kb_chunks c ON c.id = kb_chunks_fts.id
|
|
177
|
+
WHERE kb_chunks_fts MATCH ?
|
|
178
|
+
ORDER BY bm25_score ASC
|
|
179
|
+
LIMIT ?
|
|
180
|
+
`).all(matchQuery, limit * 8);
|
|
181
|
+
return rows
|
|
182
|
+
.map(row => {
|
|
183
|
+
const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
|
|
184
|
+
const bm25Score = this.bm25ToPositiveScore(Number(row.bm25_score));
|
|
185
|
+
return this.rowToChunk(row, keyword.keywordScore + bm25Score, { ...keyword, bm25Score });
|
|
186
|
+
})
|
|
187
|
+
.filter(row => row.score > 0)
|
|
188
|
+
.sort((a, b) => b.score - a.score)
|
|
189
|
+
.slice(0, limit);
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return [];
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
searchChunksLike(terms, limit) {
|
|
101
196
|
const rows = this.db.prepare(`
|
|
102
197
|
SELECT * FROM kb_chunks
|
|
103
198
|
WHERE ${terms.map(() => '(LOWER(content) LIKE ? OR LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ?)').join(' OR ')}
|
|
@@ -105,7 +200,10 @@ export class IndexStateStore {
|
|
|
105
200
|
LIMIT ?
|
|
106
201
|
`).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), limit * 6);
|
|
107
202
|
return rows
|
|
108
|
-
.map(row =>
|
|
203
|
+
.map(row => {
|
|
204
|
+
const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
|
|
205
|
+
return this.rowToChunk(row, keyword.keywordScore, keyword);
|
|
206
|
+
})
|
|
109
207
|
.filter(row => row.score > 0)
|
|
110
208
|
.sort((a, b) => b.score - a.score)
|
|
111
209
|
.slice(0, limit);
|
|
@@ -140,14 +238,22 @@ export class IndexStateStore {
|
|
|
140
238
|
`).run(record.contentHash, record.filePath, record.fileSize, record.category, record.normalizedHash ?? null, now, now);
|
|
141
239
|
}
|
|
142
240
|
upsertMinHash(record) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
241
|
+
const now = Date.now();
|
|
242
|
+
const tx = this.db.transaction(() => {
|
|
243
|
+
this.db.prepare(`
|
|
244
|
+
INSERT INTO kb_minhash (file_path, signature, shingle_count, created_at)
|
|
245
|
+
VALUES (?, ?, ?, ?)
|
|
246
|
+
ON CONFLICT(file_path) DO UPDATE SET
|
|
247
|
+
signature = excluded.signature,
|
|
248
|
+
shingle_count = excluded.shingle_count,
|
|
249
|
+
created_at = excluded.created_at
|
|
250
|
+
`).run(record.filePath, Buffer.from(JSON.stringify(record.signature), 'utf8'), record.shingleCount, now);
|
|
251
|
+
this.db.prepare('DELETE FROM kb_lsh_buckets WHERE file_path = ?').run(record.filePath);
|
|
252
|
+
const insertBucket = this.db.prepare('INSERT OR IGNORE INTO kb_lsh_buckets (bucket_key, file_path, created_at) VALUES (?, ?, ?)');
|
|
253
|
+
for (const bucket of record.buckets)
|
|
254
|
+
insertBucket.run(bucket, record.filePath, now);
|
|
255
|
+
});
|
|
256
|
+
tx();
|
|
151
257
|
}
|
|
152
258
|
listMinHashes(excludePath) {
|
|
153
259
|
const rows = excludePath
|
|
@@ -155,6 +261,24 @@ export class IndexStateStore {
|
|
|
155
261
|
: this.db.prepare('SELECT * FROM kb_minhash').all();
|
|
156
262
|
return rows.map(row => this.rowToMinHash(row));
|
|
157
263
|
}
|
|
264
|
+
listMinHashesByBuckets(buckets, excludePath) {
|
|
265
|
+
if (buckets.length === 0)
|
|
266
|
+
return [];
|
|
267
|
+
const placeholders = buckets.map(() => '?').join(',');
|
|
268
|
+
const params = [...buckets];
|
|
269
|
+
let sql = `
|
|
270
|
+
SELECT DISTINCT m.*
|
|
271
|
+
FROM kb_minhash m
|
|
272
|
+
INNER JOIN kb_lsh_buckets b ON b.file_path = m.file_path
|
|
273
|
+
WHERE b.bucket_key IN (${placeholders})
|
|
274
|
+
`;
|
|
275
|
+
if (excludePath) {
|
|
276
|
+
sql += ' AND m.file_path != ?';
|
|
277
|
+
params.push(excludePath);
|
|
278
|
+
}
|
|
279
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
280
|
+
return rows.map(row => this.rowToMinHash(row));
|
|
281
|
+
}
|
|
158
282
|
addRelationship(relationship) {
|
|
159
283
|
this.db.prepare(`
|
|
160
284
|
INSERT INTO kb_relationships (
|
|
@@ -206,9 +330,13 @@ export class IndexStateStore {
|
|
|
206
330
|
}
|
|
207
331
|
deleteRecord(relativePath) {
|
|
208
332
|
this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
|
|
333
|
+
this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
|
|
334
|
+
if (this.ftsEnabled)
|
|
335
|
+
this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
|
|
209
336
|
this.db.prepare('DELETE FROM kb_index_state WHERE relative_path = ?').run(relativePath);
|
|
210
337
|
this.db.prepare('DELETE FROM kb_file_hashes WHERE file_path = ?').run(relativePath);
|
|
211
338
|
this.db.prepare('DELETE FROM kb_minhash WHERE file_path = ?').run(relativePath);
|
|
339
|
+
this.db.prepare('DELETE FROM kb_lsh_buckets WHERE file_path = ?').run(relativePath);
|
|
212
340
|
this.db.prepare('DELETE FROM kb_tags WHERE file_path = ?').run(relativePath);
|
|
213
341
|
this.db.prepare('DELETE FROM kb_relationships WHERE source_file = ? OR target_file = ?').run(relativePath, relativePath);
|
|
214
342
|
}
|
|
@@ -218,15 +346,19 @@ export class IndexStateStore {
|
|
|
218
346
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
219
347
|
`).run(key, value);
|
|
220
348
|
}
|
|
349
|
+
getMetadata(key) {
|
|
350
|
+
const row = this.db.prepare('SELECT value FROM kb_metadata WHERE key = ?').get(key);
|
|
351
|
+
return row?.value;
|
|
352
|
+
}
|
|
221
353
|
getStats() {
|
|
222
354
|
const stats = this.db.prepare(`
|
|
223
355
|
SELECT
|
|
224
356
|
COUNT(*) as file_count,
|
|
225
|
-
|
|
357
|
+
(SELECT COUNT(*) FROM kb_chunks) as chunk_count,
|
|
226
358
|
COALESCE(SUM(file_size), 0) as total_size_bytes,
|
|
227
359
|
COALESCE(MAX(indexed_at), 0) as last_indexed_at
|
|
228
360
|
FROM kb_index_state
|
|
229
|
-
WHERE status
|
|
361
|
+
WHERE status != 'deleted'
|
|
230
362
|
`).get();
|
|
231
363
|
return {
|
|
232
364
|
fileCount: Number(stats.file_count ?? 0),
|
|
@@ -282,6 +414,22 @@ export class IndexStateStore {
|
|
|
282
414
|
CREATE INDEX IF NOT EXISTS idx_kb_chunks_category ON kb_chunks(category);
|
|
283
415
|
CREATE INDEX IF NOT EXISTS idx_kb_chunks_collection ON kb_chunks(collection_name);
|
|
284
416
|
|
|
417
|
+
CREATE TABLE IF NOT EXISTS kb_parent_chunks (
|
|
418
|
+
id TEXT PRIMARY KEY,
|
|
419
|
+
relative_path TEXT NOT NULL,
|
|
420
|
+
parent_id TEXT NOT NULL,
|
|
421
|
+
content TEXT NOT NULL,
|
|
422
|
+
category TEXT NOT NULL,
|
|
423
|
+
format TEXT NOT NULL,
|
|
424
|
+
collection_name TEXT NOT NULL,
|
|
425
|
+
section_title TEXT,
|
|
426
|
+
chunk_count INTEGER NOT NULL,
|
|
427
|
+
metadata_json TEXT,
|
|
428
|
+
created_at INTEGER NOT NULL
|
|
429
|
+
);
|
|
430
|
+
CREATE INDEX IF NOT EXISTS idx_kb_parent_path ON kb_parent_chunks(relative_path);
|
|
431
|
+
CREATE INDEX IF NOT EXISTS idx_kb_parent_id ON kb_parent_chunks(parent_id);
|
|
432
|
+
|
|
285
433
|
CREATE TABLE IF NOT EXISTS kb_file_hashes (
|
|
286
434
|
content_hash TEXT NOT NULL,
|
|
287
435
|
file_path TEXT PRIMARY KEY,
|
|
@@ -301,6 +449,15 @@ export class IndexStateStore {
|
|
|
301
449
|
created_at INTEGER NOT NULL
|
|
302
450
|
);
|
|
303
451
|
|
|
452
|
+
CREATE TABLE IF NOT EXISTS kb_lsh_buckets (
|
|
453
|
+
bucket_key TEXT NOT NULL,
|
|
454
|
+
file_path TEXT NOT NULL,
|
|
455
|
+
created_at INTEGER NOT NULL,
|
|
456
|
+
PRIMARY KEY (bucket_key, file_path)
|
|
457
|
+
);
|
|
458
|
+
CREATE INDEX IF NOT EXISTS idx_lsh_bucket ON kb_lsh_buckets(bucket_key);
|
|
459
|
+
CREATE INDEX IF NOT EXISTS idx_lsh_file ON kb_lsh_buckets(file_path);
|
|
460
|
+
|
|
304
461
|
CREATE TABLE IF NOT EXISTS kb_relationships (
|
|
305
462
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
306
463
|
source_file TEXT NOT NULL,
|
|
@@ -336,15 +493,50 @@ export class IndexStateStore {
|
|
|
336
493
|
created_at INTEGER NOT NULL
|
|
337
494
|
);
|
|
338
495
|
`);
|
|
339
|
-
this.
|
|
496
|
+
this.initFts();
|
|
497
|
+
this.setMetadata('schema_version', '2');
|
|
498
|
+
}
|
|
499
|
+
initFts() {
|
|
500
|
+
try {
|
|
501
|
+
this.db.exec(`
|
|
502
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(
|
|
503
|
+
id UNINDEXED,
|
|
504
|
+
relative_path,
|
|
505
|
+
category,
|
|
506
|
+
format,
|
|
507
|
+
section_title,
|
|
508
|
+
content,
|
|
509
|
+
tokenize = 'unicode61 remove_diacritics 2'
|
|
510
|
+
);
|
|
511
|
+
`);
|
|
512
|
+
this.ftsEnabled = true;
|
|
513
|
+
this.rebuildFtsIfNeeded();
|
|
514
|
+
}
|
|
515
|
+
catch {
|
|
516
|
+
this.ftsEnabled = false;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
rebuildFtsIfNeeded() {
|
|
520
|
+
if (!this.ftsEnabled)
|
|
521
|
+
return;
|
|
522
|
+
const row = this.db.prepare('SELECT COUNT(*) as count FROM kb_chunks_fts').get();
|
|
523
|
+
if (Number(row.count ?? 0) > 0)
|
|
524
|
+
return;
|
|
525
|
+
this.db.prepare(`
|
|
526
|
+
INSERT INTO kb_chunks_fts (id, relative_path, category, format, section_title, content)
|
|
527
|
+
SELECT id, relative_path, category, format, COALESCE(section_title, ''), content FROM kb_chunks
|
|
528
|
+
`).run();
|
|
340
529
|
}
|
|
341
530
|
rowToMinHash(row) {
|
|
342
531
|
const raw = row.signature;
|
|
343
532
|
const json = Buffer.isBuffer(raw) ? raw.toString('utf8') : String(raw);
|
|
533
|
+
const filePath = String(row.file_path);
|
|
534
|
+
const bucketRows = this.db.prepare('SELECT bucket_key FROM kb_lsh_buckets WHERE file_path = ?').all(filePath);
|
|
344
535
|
return {
|
|
345
|
-
filePath
|
|
536
|
+
filePath,
|
|
346
537
|
signature: JSON.parse(json),
|
|
347
538
|
shingleCount: Number(row.shingle_count),
|
|
539
|
+
buckets: bucketRows.map(bucket => bucket.bucket_key),
|
|
348
540
|
createdAt: Number(row.created_at),
|
|
349
541
|
};
|
|
350
542
|
}
|
|
@@ -371,7 +563,59 @@ export class IndexStateStore {
|
|
|
371
563
|
createdAt: Number(row.created_at),
|
|
372
564
|
};
|
|
373
565
|
}
|
|
374
|
-
|
|
566
|
+
rowToParentChunk(row) {
|
|
567
|
+
return {
|
|
568
|
+
id: String(row.id),
|
|
569
|
+
relativePath: String(row.relative_path),
|
|
570
|
+
parentId: String(row.parent_id),
|
|
571
|
+
content: String(row.content),
|
|
572
|
+
category: String(row.category),
|
|
573
|
+
format: String(row.format),
|
|
574
|
+
collectionName: String(row.collection_name),
|
|
575
|
+
sectionTitle: row.section_title == null ? undefined : String(row.section_title),
|
|
576
|
+
chunkCount: Number(row.chunk_count),
|
|
577
|
+
metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
|
|
578
|
+
createdAt: Number(row.created_at),
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
splitParentGroups(relativePath, chunks) {
|
|
582
|
+
const maxChildrenPerParent = 12;
|
|
583
|
+
const maxTokensPerParent = 4_000;
|
|
584
|
+
const grouped = new Map();
|
|
585
|
+
for (const chunk of chunks) {
|
|
586
|
+
const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
|
|
587
|
+
const list = grouped.get(parentId) ?? [];
|
|
588
|
+
list.push(chunk);
|
|
589
|
+
grouped.set(parentId, list);
|
|
590
|
+
}
|
|
591
|
+
const result = [];
|
|
592
|
+
for (const [parentId, list] of grouped.entries()) {
|
|
593
|
+
let batch = [];
|
|
594
|
+
let tokenCount = 0;
|
|
595
|
+
let batchIndex = 0;
|
|
596
|
+
const flush = () => {
|
|
597
|
+
if (batch.length === 0)
|
|
598
|
+
return;
|
|
599
|
+
const nextParentId = list.length <= maxChildrenPerParent && tokenCount <= maxTokensPerParent ? parentId : `${parentId}@${batchIndex + 1}`;
|
|
600
|
+
result.push(...batch.map(chunk => ({ ...chunk, metadata: { ...chunk.metadata, parentId: nextParentId, parentGroupIndex: batchIndex } })));
|
|
601
|
+
batch = [];
|
|
602
|
+
tokenCount = 0;
|
|
603
|
+
batchIndex += 1;
|
|
604
|
+
};
|
|
605
|
+
for (const chunk of list) {
|
|
606
|
+
if (batch.length > 0 && (batch.length >= maxChildrenPerParent || tokenCount + chunk.tokenCount > maxTokensPerParent))
|
|
607
|
+
flush();
|
|
608
|
+
batch.push(chunk);
|
|
609
|
+
tokenCount += chunk.tokenCount;
|
|
610
|
+
}
|
|
611
|
+
flush();
|
|
612
|
+
}
|
|
613
|
+
return result.sort((a, b) => a.index - b.index);
|
|
614
|
+
}
|
|
615
|
+
metadataString(value) {
|
|
616
|
+
return typeof value === 'string' ? value : undefined;
|
|
617
|
+
}
|
|
618
|
+
rowToChunk(row, score, scoreDetails) {
|
|
375
619
|
return {
|
|
376
620
|
id: String(row.id),
|
|
377
621
|
relativePath: String(row.relative_path),
|
|
@@ -385,13 +629,14 @@ export class IndexStateStore {
|
|
|
385
629
|
metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
|
|
386
630
|
createdAt: Number(row.created_at),
|
|
387
631
|
score,
|
|
632
|
+
scoreDetails,
|
|
388
633
|
};
|
|
389
634
|
}
|
|
390
635
|
expandSearchTerms(query) {
|
|
391
636
|
const normalized = query.toLowerCase().trim();
|
|
392
|
-
const terms = new Set(normalized
|
|
393
|
-
|
|
394
|
-
terms.add(
|
|
637
|
+
const terms = new Set(normalized ? [normalized] : []);
|
|
638
|
+
for (const term of normalized.split(/[\s,,。;;::、]+/u).filter(Boolean))
|
|
639
|
+
terms.add(term);
|
|
395
640
|
const synonyms = {
|
|
396
641
|
招标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
|
|
397
642
|
投标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
|
|
@@ -407,17 +652,44 @@ export class IndexStateStore {
|
|
|
407
652
|
}
|
|
408
653
|
return [...terms];
|
|
409
654
|
}
|
|
410
|
-
|
|
655
|
+
toFtsQuery(terms) {
|
|
656
|
+
const normalized = terms.map(term => term.replace(/["*^:(){}\]\\[]/gu, ' ').trim()).filter(term => term.length > 0);
|
|
657
|
+
const exact = normalized[0];
|
|
658
|
+
const weak = normalized.slice(1).filter(term => term.length >= 2).slice(0, 12);
|
|
659
|
+
return [exact ? `"${exact}"` : '', ...weak.map(term => `"${term}"`)].filter(Boolean).join(' OR ');
|
|
660
|
+
}
|
|
661
|
+
bm25ToPositiveScore(score) {
|
|
662
|
+
if (!Number.isFinite(score))
|
|
663
|
+
return 0;
|
|
664
|
+
return 1 / (1 + Math.max(0, score));
|
|
665
|
+
}
|
|
666
|
+
scoreChunkDetailed(content, terms) {
|
|
411
667
|
const lower = content.toLowerCase();
|
|
412
|
-
let
|
|
668
|
+
let raw = 0;
|
|
669
|
+
let exactPhraseBoost = 0;
|
|
670
|
+
const exactPhrase = terms[0] ?? '';
|
|
671
|
+
const exactHits = exactPhrase ? this.countOccurrences(lower, exactPhrase) : 0;
|
|
672
|
+
if (exactHits > 0)
|
|
673
|
+
exactPhraseBoost = 1000 + exactHits * 20;
|
|
674
|
+
raw += exactPhraseBoost;
|
|
413
675
|
for (const term of terms) {
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
676
|
+
if (term === exactPhrase)
|
|
677
|
+
continue;
|
|
678
|
+
raw += this.countOccurrences(lower, term) * 0.2;
|
|
679
|
+
}
|
|
680
|
+
return {
|
|
681
|
+
keywordScore: raw / Math.max(1, content.length / 1000),
|
|
682
|
+
exactPhraseBoost,
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
countOccurrences(content, term) {
|
|
686
|
+
let count = 0;
|
|
687
|
+
let index = content.indexOf(term);
|
|
688
|
+
while (index !== -1) {
|
|
689
|
+
count += 1;
|
|
690
|
+
index = content.indexOf(term, index + term.length);
|
|
419
691
|
}
|
|
420
|
-
return
|
|
692
|
+
return count;
|
|
421
693
|
}
|
|
422
694
|
rowToRecord(row) {
|
|
423
695
|
return {
|
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
import { type EmbeddingProvider } from '../embedding/embedding-provider.js';
|
|
2
2
|
import type { ExternalExtractorRegistry } from '../extraction/external-extractor.js';
|
|
3
|
-
import
|
|
3
|
+
import type { LLMSearchProvider } from '../llm/llm-search-provider.js';
|
|
4
|
+
import { type FederatedResult, type FederatedSearchItem, type RetrievalWeights, type SearchFilters } from '../search/federation-search.js';
|
|
4
5
|
import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
|
|
5
6
|
import type { VectorStoreInterface } from '../vector/types.js';
|
|
6
7
|
import { type VectorIndexResult } from '../vector/vector-indexer.js';
|
|
7
8
|
import { IndexStateStore, type ChunkSearchResult, type FileRelationship } from './index-state-store.js';
|
|
9
|
+
export type KnowledgeIndexStage = 'scanning' | 'parsing' | 'chunking' | 'vectorizing' | 'done' | 'error';
|
|
10
|
+
export interface KnowledgeIndexProgress {
|
|
11
|
+
stage: KnowledgeIndexStage;
|
|
12
|
+
percent: number;
|
|
13
|
+
message: string;
|
|
14
|
+
filePath?: string;
|
|
15
|
+
chunkCount?: number;
|
|
16
|
+
vectorStatus?: ReturnType<KnowledgeBaseManager['getVectorStatus']>;
|
|
17
|
+
}
|
|
8
18
|
export interface KnowledgeBaseManagerOptions {
|
|
9
19
|
scope: Exclude<KBScope, 'session'>;
|
|
10
20
|
projectRoot?: string;
|
|
@@ -14,6 +24,9 @@ export interface KnowledgeBaseManagerOptions {
|
|
|
14
24
|
embeddingProvider?: EmbeddingProvider;
|
|
15
25
|
vectorStores?: Map<string, VectorStoreInterface>;
|
|
16
26
|
externalExtractors?: ExternalExtractorRegistry;
|
|
27
|
+
onProgress?: (progress: KnowledgeIndexProgress) => void;
|
|
28
|
+
/** 可选的 LLM Provider,用于查询扩展和语义重排序 */
|
|
29
|
+
llmProvider?: LLMSearchProvider;
|
|
17
30
|
}
|
|
18
31
|
export declare class KnowledgeBaseManager {
|
|
19
32
|
readonly scope: Exclude<KBScope, 'session'>;
|
|
@@ -21,6 +34,7 @@ export declare class KnowledgeBaseManager {
|
|
|
21
34
|
readonly projectId?: string;
|
|
22
35
|
readonly kbPath: string;
|
|
23
36
|
readonly store: IndexStateStore;
|
|
37
|
+
private readonly chromaClient;
|
|
24
38
|
private readonly classifier;
|
|
25
39
|
private readonly scanner;
|
|
26
40
|
private readonly collections;
|
|
@@ -33,10 +47,23 @@ export declare class KnowledgeBaseManager {
|
|
|
33
47
|
private readonly configManager;
|
|
34
48
|
private projectConfig?;
|
|
35
49
|
private lastSkippedFiles;
|
|
50
|
+
private readonly llmProvider?;
|
|
51
|
+
private onProgress?;
|
|
36
52
|
constructor(options: KnowledgeBaseManagerOptions);
|
|
37
53
|
initialize(): void;
|
|
38
|
-
incrementalIndex(
|
|
54
|
+
incrementalIndex(options?: {
|
|
55
|
+
onProgress?: (progress: KnowledgeIndexProgress) => void;
|
|
56
|
+
vectorMode?: 'sync' | 'defer';
|
|
57
|
+
}): Promise<DiffResult>;
|
|
39
58
|
search(query: string, limit?: number): ChunkSearchResult[];
|
|
59
|
+
keywordSearchItems(query: string, limit?: number): FederatedSearchItem[];
|
|
60
|
+
expandContext(item: FederatedSearchItem): FederatedSearchItem;
|
|
61
|
+
hybridSearch(query: string, options?: {
|
|
62
|
+
limit?: number;
|
|
63
|
+
filters?: SearchFilters;
|
|
64
|
+
collections?: string[];
|
|
65
|
+
weights?: RetrievalWeights;
|
|
66
|
+
}): Promise<FederatedResult>;
|
|
40
67
|
semanticSearch(query: string, options?: {
|
|
41
68
|
limit?: number;
|
|
42
69
|
filters?: SearchFilters;
|
|
@@ -44,8 +71,25 @@ export declare class KnowledgeBaseManager {
|
|
|
44
71
|
}): Promise<FederatedResult>;
|
|
45
72
|
listRelationships(filePath?: string): FileRelationship[];
|
|
46
73
|
listFiles(): IndexStateRecord[];
|
|
74
|
+
getFileDetail(relativePath: string): {
|
|
75
|
+
file: IndexStateRecord;
|
|
76
|
+
absolutePath: string;
|
|
77
|
+
directory: string;
|
|
78
|
+
chunks: import("./index-state-store.js").StoredChunk[];
|
|
79
|
+
parents: import("./index-state-store.js").StoredParentChunk[];
|
|
80
|
+
relationships: FileRelationship[];
|
|
81
|
+
tags: {
|
|
82
|
+
filePath: string;
|
|
83
|
+
tag: string;
|
|
84
|
+
createdAt: number;
|
|
85
|
+
}[];
|
|
86
|
+
} | undefined;
|
|
87
|
+
reindexFile(relativePath: string): Promise<DiffResult>;
|
|
47
88
|
addFile(sourcePath: string, targetRelativePath?: string): Promise<DiffResult>;
|
|
48
|
-
|
|
89
|
+
getUploadRelativePath(fileName: string, targetRelativePath?: string): string;
|
|
90
|
+
uploadFile(fileName: string, content: Buffer, targetRelativePath?: string, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
|
|
91
|
+
vectorMode?: 'sync' | 'defer';
|
|
92
|
+
}): Promise<DiffResult>;
|
|
49
93
|
listFailedFiles(): DiffResult['skippedFiles'];
|
|
50
94
|
removeFile(relativePath: string): Promise<void>;
|
|
51
95
|
tagFile(relativePath: string, tags: string[]): void;
|
|
@@ -68,7 +112,30 @@ export declare class KnowledgeBaseManager {
|
|
|
68
112
|
}): Promise<VectorIndexResult[]>;
|
|
69
113
|
getProjectConfig(): ProjectConfig | undefined;
|
|
70
114
|
getStats(): KnowledgeBaseStats;
|
|
115
|
+
getVectorStatus(): {
|
|
116
|
+
status: string;
|
|
117
|
+
error?: string;
|
|
118
|
+
indexedChunks: number;
|
|
119
|
+
lastIndexedAt: number;
|
|
120
|
+
backend: string;
|
|
121
|
+
};
|
|
122
|
+
private rewriteQueries;
|
|
123
|
+
private llmExpandQueries;
|
|
124
|
+
private retrievalWeights;
|
|
125
|
+
private heuristicRerank;
|
|
126
|
+
private llmRerank;
|
|
127
|
+
private toFederatedItem;
|
|
128
|
+
private mergeHybridItems;
|
|
129
|
+
private parseChunkIndex;
|
|
130
|
+
private parseMetadataString;
|
|
131
|
+
private parseMetadata;
|
|
132
|
+
private metadataString;
|
|
133
|
+
private metadataFacets;
|
|
71
134
|
close(): void;
|
|
135
|
+
private reportProgress;
|
|
136
|
+
private ensureVectorStore;
|
|
137
|
+
private deleteVectorFile;
|
|
138
|
+
private ensureVectorIndexFresh;
|
|
72
139
|
private hasUsableContent;
|
|
73
140
|
private defaultUploadRelativePath;
|
|
74
141
|
private resolveKbRelativePath;
|