@customize-agent/knowledge 1.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/chunking/text-chunker.d.ts +12 -1
  2. package/dist/chunking/text-chunker.js +219 -41
  3. package/dist/core/change-tracker.d.ts +1 -0
  4. package/dist/core/change-tracker.js +21 -0
  5. package/dist/core/index-state-store.d.ts +37 -1
  6. package/dist/core/index-state-store.js +300 -28
  7. package/dist/core/knowledge-base-manager.d.ts +70 -3
  8. package/dist/core/knowledge-base-manager.js +547 -125
  9. package/dist/core/multi-project-manager.d.ts +7 -4
  10. package/dist/core/multi-project-manager.js +35 -23
  11. package/dist/dedup/dedup-engine.d.ts +3 -0
  12. package/dist/dedup/dedup-engine.js +12 -2
  13. package/dist/embedding/embedding-provider.d.ts +1 -0
  14. package/dist/embedding/embedding-provider.js +16 -1
  15. package/dist/extraction/content-extractor.d.ts +2 -0
  16. package/dist/extraction/content-extractor.js +209 -42
  17. package/dist/extraction/module-resolver.d.ts +17 -0
  18. package/dist/extraction/module-resolver.js +113 -0
  19. package/dist/index.d.ts +2 -4
  20. package/dist/index.js +2 -3
  21. package/dist/llm/llm-search-provider.d.ts +23 -0
  22. package/dist/llm/llm-search-provider.js +1 -0
  23. package/dist/search/federation-search.d.ts +29 -0
  24. package/dist/search/federation-search.js +8 -1
  25. package/dist/vector/chroma-store.d.ts +2 -0
  26. package/dist/vector/chroma-store.js +53 -22
  27. package/dist/vector/vector-indexer.d.ts +3 -0
  28. package/dist/vector/vector-indexer.js +23 -0
  29. package/package.json +11 -3
  30. package/dist/server/dashboard-client.d.ts +0 -2
  31. package/dist/server/dashboard-client.js +0 -396
  32. package/dist/server/dashboard-i18n.d.ts +0 -112
  33. package/dist/server/dashboard-i18n.js +0 -220
  34. package/dist/server/dashboard-page.d.ts +0 -6
  35. package/dist/server/dashboard-page.js +0 -138
  36. package/dist/server/dashboard-server.d.ts +0 -13
  37. package/dist/server/dashboard-server.js +0 -225
  38. package/dist/server/dashboard-styles.d.ts +0 -1
  39. package/dist/server/dashboard-styles.js +0 -152
@@ -3,6 +3,7 @@ import * as fs from 'node:fs';
3
3
  import * as path from 'node:path';
4
4
  export class IndexStateStore {
5
5
  db;
6
+ ftsEnabled = false;
6
7
  constructor(dbPath) {
7
8
  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
8
9
  this.db = new Database(dbPath);
@@ -50,7 +51,7 @@ export class IndexStateStore {
50
51
  listRecords() {
51
52
  const rows = this.db.prepare(`
52
53
  SELECT * FROM kb_index_state
53
- WHERE status = 'active'
54
+ WHERE status != 'deleted'
54
55
  ORDER BY category, relative_path
55
56
  `).all();
56
57
  return rows.map(row => this.rowToRecord(row));
@@ -59,14 +60,40 @@ export class IndexStateStore {
59
60
  const now = Date.now();
60
61
  const transaction = this.db.transaction(() => {
61
62
  this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
63
+ this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
64
+ if (this.ftsEnabled)
65
+ this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
62
66
  const insert = this.db.prepare(`
63
67
  INSERT INTO kb_chunks (
64
68
  id, relative_path, chunk_index, content, category, format,
65
69
  collection_name, token_count, section_title, metadata_json, created_at
66
70
  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
67
71
  `);
68
- for (const chunk of chunks) {
69
- insert.run(`${relativePath}#${chunk.index}`, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
72
+ const insertFts = this.ftsEnabled ? this.db.prepare(`
73
+ INSERT INTO kb_chunks_fts (id, relative_path, category, format, section_title, content)
74
+ VALUES (?, ?, ?, ?, ?, ?)
75
+ `) : undefined;
76
+ const insertParent = this.db.prepare(`
77
+ INSERT INTO kb_parent_chunks (
78
+ id, relative_path, parent_id, content, category, format,
79
+ collection_name, section_title, chunk_count, metadata_json, created_at
80
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
81
+ `);
82
+ const parentGroups = new Map();
83
+ const groupedChunks = this.splitParentGroups(relativePath, chunks);
84
+ for (const chunk of groupedChunks) {
85
+ const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
86
+ const group = parentGroups.get(parentId) ?? [];
87
+ group.push(chunk);
88
+ parentGroups.set(parentId, group);
89
+ }
90
+ for (const [parentId, group] of parentGroups.entries()) {
91
+ insertParent.run(parentId, relativePath, parentId, group.map(chunk => chunk.text).join('\n\n---\n\n'), file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind) }), now);
92
+ }
93
+ for (const chunk of groupedChunks) {
94
+ const chunkId = `${relativePath}#${chunk.index}`;
95
+ insert.run(chunkId, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
96
+ insertFts?.run(chunkId, relativePath, file.category, file.format, chunk.sectionTitle ?? '', chunk.text);
70
97
  }
71
98
  });
72
99
  transaction();
@@ -94,10 +121,78 @@ export class IndexStateStore {
94
121
  `).all(...params);
95
122
  return rows.map(row => this.rowToChunk(row, 0));
96
123
  }
124
+ getContextChunks(relativePath, chunkIndex, window = 1) {
125
+ const rows = this.db.prepare(`
126
+ SELECT * FROM kb_chunks
127
+ WHERE relative_path = ? AND chunk_index BETWEEN ? AND ?
128
+ ORDER BY chunk_index
129
+ `).all(relativePath, Math.max(0, chunkIndex - window), chunkIndex + window);
130
+ return rows.map(row => this.rowToChunk(row, 0));
131
+ }
132
+ listParentChunks(relativePath) {
133
+ const rows = this.db.prepare(`
134
+ SELECT * FROM kb_parent_chunks
135
+ WHERE relative_path = ?
136
+ ORDER BY parent_id
137
+ `).all(relativePath);
138
+ return rows.map(row => this.rowToParentChunk(row));
139
+ }
140
+ getParentChunk(relativePath, parentId) {
141
+ const row = this.db.prepare(`
142
+ SELECT * FROM kb_parent_chunks
143
+ WHERE relative_path = ? AND parent_id = ?
144
+ LIMIT 1
145
+ `).get(relativePath, parentId);
146
+ return row ? this.rowToParentChunk(row) : undefined;
147
+ }
148
+ getChunksByParent(relativePath, parentId, limit = 6) {
149
+ const rows = this.db.prepare(`
150
+ SELECT * FROM kb_chunks
151
+ WHERE relative_path = ? AND metadata_json LIKE ?
152
+ ORDER BY chunk_index
153
+ LIMIT ?
154
+ `).all(relativePath, `%"parentId":"${parentId.replace(/[%_]/gu, '')}"%`, limit);
155
+ return rows.map(row => this.rowToChunk(row, 0));
156
+ }
97
157
  searchChunks(query, limit = 10) {
98
158
  const terms = this.expandSearchTerms(query);
99
159
  if (terms.length === 0)
100
160
  return [];
161
+ if (this.ftsEnabled) {
162
+ const ftsResults = this.searchChunksFts(terms, limit);
163
+ if (ftsResults.length > 0)
164
+ return ftsResults;
165
+ }
166
+ return this.searchChunksLike(terms, limit);
167
+ }
168
+ searchChunksFts(terms, limit) {
169
+ try {
170
+ const matchQuery = this.toFtsQuery(terms);
171
+ if (!matchQuery)
172
+ return [];
173
+ const rows = this.db.prepare(`
174
+ SELECT c.*, bm25(kb_chunks_fts, 1.2, 0.8, 0.6, 1.0, 2.0) as bm25_score
175
+ FROM kb_chunks_fts
176
+ INNER JOIN kb_chunks c ON c.id = kb_chunks_fts.id
177
+ WHERE kb_chunks_fts MATCH ?
178
+ ORDER BY bm25_score ASC
179
+ LIMIT ?
180
+ `).all(matchQuery, limit * 8);
181
+ return rows
182
+ .map(row => {
183
+ const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
184
+ const bm25Score = this.bm25ToPositiveScore(Number(row.bm25_score));
185
+ return this.rowToChunk(row, keyword.keywordScore + bm25Score, { ...keyword, bm25Score });
186
+ })
187
+ .filter(row => row.score > 0)
188
+ .sort((a, b) => b.score - a.score)
189
+ .slice(0, limit);
190
+ }
191
+ catch {
192
+ return [];
193
+ }
194
+ }
195
+ searchChunksLike(terms, limit) {
101
196
  const rows = this.db.prepare(`
102
197
  SELECT * FROM kb_chunks
103
198
  WHERE ${terms.map(() => '(LOWER(content) LIKE ? OR LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ?)').join(' OR ')}
@@ -105,7 +200,10 @@ export class IndexStateStore {
105
200
  LIMIT ?
106
201
  `).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), limit * 6);
107
202
  return rows
108
- .map(row => this.rowToChunk(row, this.scoreChunk(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms)))
203
+ .map(row => {
204
+ const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
205
+ return this.rowToChunk(row, keyword.keywordScore, keyword);
206
+ })
109
207
  .filter(row => row.score > 0)
110
208
  .sort((a, b) => b.score - a.score)
111
209
  .slice(0, limit);
@@ -140,14 +238,22 @@ export class IndexStateStore {
140
238
  `).run(record.contentHash, record.filePath, record.fileSize, record.category, record.normalizedHash ?? null, now, now);
141
239
  }
142
240
  upsertMinHash(record) {
143
- this.db.prepare(`
144
- INSERT INTO kb_minhash (file_path, signature, shingle_count, created_at)
145
- VALUES (?, ?, ?, ?)
146
- ON CONFLICT(file_path) DO UPDATE SET
147
- signature = excluded.signature,
148
- shingle_count = excluded.shingle_count,
149
- created_at = excluded.created_at
150
- `).run(record.filePath, Buffer.from(JSON.stringify(record.signature), 'utf8'), record.shingleCount, Date.now());
241
+ const now = Date.now();
242
+ const tx = this.db.transaction(() => {
243
+ this.db.prepare(`
244
+ INSERT INTO kb_minhash (file_path, signature, shingle_count, created_at)
245
+ VALUES (?, ?, ?, ?)
246
+ ON CONFLICT(file_path) DO UPDATE SET
247
+ signature = excluded.signature,
248
+ shingle_count = excluded.shingle_count,
249
+ created_at = excluded.created_at
250
+ `).run(record.filePath, Buffer.from(JSON.stringify(record.signature), 'utf8'), record.shingleCount, now);
251
+ this.db.prepare('DELETE FROM kb_lsh_buckets WHERE file_path = ?').run(record.filePath);
252
+ const insertBucket = this.db.prepare('INSERT OR IGNORE INTO kb_lsh_buckets (bucket_key, file_path, created_at) VALUES (?, ?, ?)');
253
+ for (const bucket of record.buckets)
254
+ insertBucket.run(bucket, record.filePath, now);
255
+ });
256
+ tx();
151
257
  }
152
258
  listMinHashes(excludePath) {
153
259
  const rows = excludePath
@@ -155,6 +261,24 @@ export class IndexStateStore {
155
261
  : this.db.prepare('SELECT * FROM kb_minhash').all();
156
262
  return rows.map(row => this.rowToMinHash(row));
157
263
  }
264
+ listMinHashesByBuckets(buckets, excludePath) {
265
+ if (buckets.length === 0)
266
+ return [];
267
+ const placeholders = buckets.map(() => '?').join(',');
268
+ const params = [...buckets];
269
+ let sql = `
270
+ SELECT DISTINCT m.*
271
+ FROM kb_minhash m
272
+ INNER JOIN kb_lsh_buckets b ON b.file_path = m.file_path
273
+ WHERE b.bucket_key IN (${placeholders})
274
+ `;
275
+ if (excludePath) {
276
+ sql += ' AND m.file_path != ?';
277
+ params.push(excludePath);
278
+ }
279
+ const rows = this.db.prepare(sql).all(...params);
280
+ return rows.map(row => this.rowToMinHash(row));
281
+ }
158
282
  addRelationship(relationship) {
159
283
  this.db.prepare(`
160
284
  INSERT INTO kb_relationships (
@@ -206,9 +330,13 @@ export class IndexStateStore {
206
330
  }
207
331
  deleteRecord(relativePath) {
208
332
  this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
333
+ this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
334
+ if (this.ftsEnabled)
335
+ this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
209
336
  this.db.prepare('DELETE FROM kb_index_state WHERE relative_path = ?').run(relativePath);
210
337
  this.db.prepare('DELETE FROM kb_file_hashes WHERE file_path = ?').run(relativePath);
211
338
  this.db.prepare('DELETE FROM kb_minhash WHERE file_path = ?').run(relativePath);
339
+ this.db.prepare('DELETE FROM kb_lsh_buckets WHERE file_path = ?').run(relativePath);
212
340
  this.db.prepare('DELETE FROM kb_tags WHERE file_path = ?').run(relativePath);
213
341
  this.db.prepare('DELETE FROM kb_relationships WHERE source_file = ? OR target_file = ?').run(relativePath, relativePath);
214
342
  }
@@ -218,15 +346,19 @@ export class IndexStateStore {
218
346
  ON CONFLICT(key) DO UPDATE SET value = excluded.value
219
347
  `).run(key, value);
220
348
  }
349
+ getMetadata(key) {
350
+ const row = this.db.prepare('SELECT value FROM kb_metadata WHERE key = ?').get(key);
351
+ return row?.value;
352
+ }
221
353
  getStats() {
222
354
  const stats = this.db.prepare(`
223
355
  SELECT
224
356
  COUNT(*) as file_count,
225
- COALESCE(SUM(chunk_count), 0) as chunk_count,
357
+ (SELECT COUNT(*) FROM kb_chunks) as chunk_count,
226
358
  COALESCE(SUM(file_size), 0) as total_size_bytes,
227
359
  COALESCE(MAX(indexed_at), 0) as last_indexed_at
228
360
  FROM kb_index_state
229
- WHERE status = 'active'
361
+ WHERE status != 'deleted'
230
362
  `).get();
231
363
  return {
232
364
  fileCount: Number(stats.file_count ?? 0),
@@ -282,6 +414,22 @@ export class IndexStateStore {
282
414
  CREATE INDEX IF NOT EXISTS idx_kb_chunks_category ON kb_chunks(category);
283
415
  CREATE INDEX IF NOT EXISTS idx_kb_chunks_collection ON kb_chunks(collection_name);
284
416
 
417
+ CREATE TABLE IF NOT EXISTS kb_parent_chunks (
418
+ id TEXT PRIMARY KEY,
419
+ relative_path TEXT NOT NULL,
420
+ parent_id TEXT NOT NULL,
421
+ content TEXT NOT NULL,
422
+ category TEXT NOT NULL,
423
+ format TEXT NOT NULL,
424
+ collection_name TEXT NOT NULL,
425
+ section_title TEXT,
426
+ chunk_count INTEGER NOT NULL,
427
+ metadata_json TEXT,
428
+ created_at INTEGER NOT NULL
429
+ );
430
+ CREATE INDEX IF NOT EXISTS idx_kb_parent_path ON kb_parent_chunks(relative_path);
431
+ CREATE INDEX IF NOT EXISTS idx_kb_parent_id ON kb_parent_chunks(parent_id);
432
+
285
433
  CREATE TABLE IF NOT EXISTS kb_file_hashes (
286
434
  content_hash TEXT NOT NULL,
287
435
  file_path TEXT PRIMARY KEY,
@@ -301,6 +449,15 @@ export class IndexStateStore {
301
449
  created_at INTEGER NOT NULL
302
450
  );
303
451
 
452
+ CREATE TABLE IF NOT EXISTS kb_lsh_buckets (
453
+ bucket_key TEXT NOT NULL,
454
+ file_path TEXT NOT NULL,
455
+ created_at INTEGER NOT NULL,
456
+ PRIMARY KEY (bucket_key, file_path)
457
+ );
458
+ CREATE INDEX IF NOT EXISTS idx_lsh_bucket ON kb_lsh_buckets(bucket_key);
459
+ CREATE INDEX IF NOT EXISTS idx_lsh_file ON kb_lsh_buckets(file_path);
460
+
304
461
  CREATE TABLE IF NOT EXISTS kb_relationships (
305
462
  id INTEGER PRIMARY KEY AUTOINCREMENT,
306
463
  source_file TEXT NOT NULL,
@@ -336,15 +493,50 @@ export class IndexStateStore {
336
493
  created_at INTEGER NOT NULL
337
494
  );
338
495
  `);
339
- this.setMetadata('schema_version', '1');
496
+ this.initFts();
497
+ this.setMetadata('schema_version', '2');
498
+ }
499
+ initFts() {
500
+ try {
501
+ this.db.exec(`
502
+ CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(
503
+ id UNINDEXED,
504
+ relative_path,
505
+ category,
506
+ format,
507
+ section_title,
508
+ content,
509
+ tokenize = 'unicode61 remove_diacritics 2'
510
+ );
511
+ `);
512
+ this.ftsEnabled = true;
513
+ this.rebuildFtsIfNeeded();
514
+ }
515
+ catch {
516
+ this.ftsEnabled = false;
517
+ }
518
+ }
519
+ rebuildFtsIfNeeded() {
520
+ if (!this.ftsEnabled)
521
+ return;
522
+ const row = this.db.prepare('SELECT COUNT(*) as count FROM kb_chunks_fts').get();
523
+ if (Number(row.count ?? 0) > 0)
524
+ return;
525
+ this.db.prepare(`
526
+ INSERT INTO kb_chunks_fts (id, relative_path, category, format, section_title, content)
527
+ SELECT id, relative_path, category, format, COALESCE(section_title, ''), content FROM kb_chunks
528
+ `).run();
340
529
  }
341
530
  rowToMinHash(row) {
342
531
  const raw = row.signature;
343
532
  const json = Buffer.isBuffer(raw) ? raw.toString('utf8') : String(raw);
533
+ const filePath = String(row.file_path);
534
+ const bucketRows = this.db.prepare('SELECT bucket_key FROM kb_lsh_buckets WHERE file_path = ?').all(filePath);
344
535
  return {
345
- filePath: String(row.file_path),
536
+ filePath,
346
537
  signature: JSON.parse(json),
347
538
  shingleCount: Number(row.shingle_count),
539
+ buckets: bucketRows.map(bucket => bucket.bucket_key),
348
540
  createdAt: Number(row.created_at),
349
541
  };
350
542
  }
@@ -371,7 +563,59 @@ export class IndexStateStore {
371
563
  createdAt: Number(row.created_at),
372
564
  };
373
565
  }
374
- rowToChunk(row, score) {
566
+ rowToParentChunk(row) {
567
+ return {
568
+ id: String(row.id),
569
+ relativePath: String(row.relative_path),
570
+ parentId: String(row.parent_id),
571
+ content: String(row.content),
572
+ category: String(row.category),
573
+ format: String(row.format),
574
+ collectionName: String(row.collection_name),
575
+ sectionTitle: row.section_title == null ? undefined : String(row.section_title),
576
+ chunkCount: Number(row.chunk_count),
577
+ metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
578
+ createdAt: Number(row.created_at),
579
+ };
580
+ }
581
+ splitParentGroups(relativePath, chunks) {
582
+ const maxChildrenPerParent = 12;
583
+ const maxTokensPerParent = 4_000;
584
+ const grouped = new Map();
585
+ for (const chunk of chunks) {
586
+ const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
587
+ const list = grouped.get(parentId) ?? [];
588
+ list.push(chunk);
589
+ grouped.set(parentId, list);
590
+ }
591
+ const result = [];
592
+ for (const [parentId, list] of grouped.entries()) {
593
+ let batch = [];
594
+ let tokenCount = 0;
595
+ let batchIndex = 0;
596
+ const flush = () => {
597
+ if (batch.length === 0)
598
+ return;
599
+ const nextParentId = list.length <= maxChildrenPerParent && tokenCount <= maxTokensPerParent ? parentId : `${parentId}@${batchIndex + 1}`;
600
+ result.push(...batch.map(chunk => ({ ...chunk, metadata: { ...chunk.metadata, parentId: nextParentId, parentGroupIndex: batchIndex } })));
601
+ batch = [];
602
+ tokenCount = 0;
603
+ batchIndex += 1;
604
+ };
605
+ for (const chunk of list) {
606
+ if (batch.length > 0 && (batch.length >= maxChildrenPerParent || tokenCount + chunk.tokenCount > maxTokensPerParent))
607
+ flush();
608
+ batch.push(chunk);
609
+ tokenCount += chunk.tokenCount;
610
+ }
611
+ flush();
612
+ }
613
+ return result.sort((a, b) => a.index - b.index);
614
+ }
615
+ metadataString(value) {
616
+ return typeof value === 'string' ? value : undefined;
617
+ }
618
+ rowToChunk(row, score, scoreDetails) {
375
619
  return {
376
620
  id: String(row.id),
377
621
  relativePath: String(row.relative_path),
@@ -385,13 +629,14 @@ export class IndexStateStore {
385
629
  metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
386
630
  createdAt: Number(row.created_at),
387
631
  score,
632
+ scoreDetails,
388
633
  };
389
634
  }
390
635
  expandSearchTerms(query) {
391
636
  const normalized = query.toLowerCase().trim();
392
- const terms = new Set(normalized.split(/[\s,,。;;::、]+/u).filter(Boolean));
393
- if (normalized)
394
- terms.add(normalized);
637
+ const terms = new Set(normalized ? [normalized] : []);
638
+ for (const term of normalized.split(/[\s,,。;;::、]+/u).filter(Boolean))
639
+ terms.add(term);
395
640
  const synonyms = {
396
641
  招标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
397
642
  投标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
@@ -407,17 +652,44 @@ export class IndexStateStore {
407
652
  }
408
653
  return [...terms];
409
654
  }
410
- scoreChunk(content, terms) {
655
+ toFtsQuery(terms) {
656
+ const normalized = terms.map(term => term.replace(/["*^:(){}\]\\[]/gu, ' ').trim()).filter(term => term.length > 0);
657
+ const exact = normalized[0];
658
+ const weak = normalized.slice(1).filter(term => term.length >= 2).slice(0, 12);
659
+ return [exact ? `"${exact}"` : '', ...weak.map(term => `"${term}"`)].filter(Boolean).join(' OR ');
660
+ }
661
+ bm25ToPositiveScore(score) {
662
+ if (!Number.isFinite(score))
663
+ return 0;
664
+ return 1 / (1 + Math.max(0, score));
665
+ }
666
+ scoreChunkDetailed(content, terms) {
411
667
  const lower = content.toLowerCase();
412
- let score = 0;
668
+ let raw = 0;
669
+ let exactPhraseBoost = 0;
670
+ const exactPhrase = terms[0] ?? '';
671
+ const exactHits = exactPhrase ? this.countOccurrences(lower, exactPhrase) : 0;
672
+ if (exactHits > 0)
673
+ exactPhraseBoost = 1000 + exactHits * 20;
674
+ raw += exactPhraseBoost;
413
675
  for (const term of terms) {
414
- let index = lower.indexOf(term);
415
- while (index !== -1) {
416
- score += 1;
417
- index = lower.indexOf(term, index + term.length);
418
- }
676
+ if (term === exactPhrase)
677
+ continue;
678
+ raw += this.countOccurrences(lower, term) * 0.2;
679
+ }
680
+ return {
681
+ keywordScore: raw / Math.max(1, content.length / 1000),
682
+ exactPhraseBoost,
683
+ };
684
+ }
685
+ countOccurrences(content, term) {
686
+ let count = 0;
687
+ let index = content.indexOf(term);
688
+ while (index !== -1) {
689
+ count += 1;
690
+ index = content.indexOf(term, index + term.length);
419
691
  }
420
- return score / Math.max(1, content.length / 1000);
692
+ return count;
421
693
  }
422
694
  rowToRecord(row) {
423
695
  return {
@@ -1,10 +1,20 @@
1
1
  import { type EmbeddingProvider } from '../embedding/embedding-provider.js';
2
2
  import type { ExternalExtractorRegistry } from '../extraction/external-extractor.js';
3
- import { type FederatedResult, type SearchFilters } from '../search/federation-search.js';
3
+ import type { LLMSearchProvider } from '../llm/llm-search-provider.js';
4
+ import { type FederatedResult, type FederatedSearchItem, type RetrievalWeights, type SearchFilters } from '../search/federation-search.js';
4
5
  import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
5
6
  import type { VectorStoreInterface } from '../vector/types.js';
6
7
  import { type VectorIndexResult } from '../vector/vector-indexer.js';
7
8
  import { IndexStateStore, type ChunkSearchResult, type FileRelationship } from './index-state-store.js';
9
+ export type KnowledgeIndexStage = 'scanning' | 'parsing' | 'chunking' | 'vectorizing' | 'done' | 'error';
10
+ export interface KnowledgeIndexProgress {
11
+ stage: KnowledgeIndexStage;
12
+ percent: number;
13
+ message: string;
14
+ filePath?: string;
15
+ chunkCount?: number;
16
+ vectorStatus?: ReturnType<KnowledgeBaseManager['getVectorStatus']>;
17
+ }
8
18
  export interface KnowledgeBaseManagerOptions {
9
19
  scope: Exclude<KBScope, 'session'>;
10
20
  projectRoot?: string;
@@ -14,6 +24,9 @@ export interface KnowledgeBaseManagerOptions {
14
24
  embeddingProvider?: EmbeddingProvider;
15
25
  vectorStores?: Map<string, VectorStoreInterface>;
16
26
  externalExtractors?: ExternalExtractorRegistry;
27
+ onProgress?: (progress: KnowledgeIndexProgress) => void;
28
+ /** 可选的 LLM Provider,用于查询扩展和语义重排序 */
29
+ llmProvider?: LLMSearchProvider;
17
30
  }
18
31
  export declare class KnowledgeBaseManager {
19
32
  readonly scope: Exclude<KBScope, 'session'>;
@@ -21,6 +34,7 @@ export declare class KnowledgeBaseManager {
21
34
  readonly projectId?: string;
22
35
  readonly kbPath: string;
23
36
  readonly store: IndexStateStore;
37
+ private readonly chromaClient;
24
38
  private readonly classifier;
25
39
  private readonly scanner;
26
40
  private readonly collections;
@@ -33,10 +47,23 @@ export declare class KnowledgeBaseManager {
33
47
  private readonly configManager;
34
48
  private projectConfig?;
35
49
  private lastSkippedFiles;
50
+ private readonly llmProvider?;
51
+ private onProgress?;
36
52
  constructor(options: KnowledgeBaseManagerOptions);
37
53
  initialize(): void;
38
- incrementalIndex(): Promise<DiffResult>;
54
+ incrementalIndex(options?: {
55
+ onProgress?: (progress: KnowledgeIndexProgress) => void;
56
+ vectorMode?: 'sync' | 'defer';
57
+ }): Promise<DiffResult>;
39
58
  search(query: string, limit?: number): ChunkSearchResult[];
59
+ keywordSearchItems(query: string, limit?: number): FederatedSearchItem[];
60
+ expandContext(item: FederatedSearchItem): FederatedSearchItem;
61
+ hybridSearch(query: string, options?: {
62
+ limit?: number;
63
+ filters?: SearchFilters;
64
+ collections?: string[];
65
+ weights?: RetrievalWeights;
66
+ }): Promise<FederatedResult>;
40
67
  semanticSearch(query: string, options?: {
41
68
  limit?: number;
42
69
  filters?: SearchFilters;
@@ -44,8 +71,25 @@ export declare class KnowledgeBaseManager {
44
71
  }): Promise<FederatedResult>;
45
72
  listRelationships(filePath?: string): FileRelationship[];
46
73
  listFiles(): IndexStateRecord[];
74
+ getFileDetail(relativePath: string): {
75
+ file: IndexStateRecord;
76
+ absolutePath: string;
77
+ directory: string;
78
+ chunks: import("./index-state-store.js").StoredChunk[];
79
+ parents: import("./index-state-store.js").StoredParentChunk[];
80
+ relationships: FileRelationship[];
81
+ tags: {
82
+ filePath: string;
83
+ tag: string;
84
+ createdAt: number;
85
+ }[];
86
+ } | undefined;
87
+ reindexFile(relativePath: string): Promise<DiffResult>;
47
88
  addFile(sourcePath: string, targetRelativePath?: string): Promise<DiffResult>;
48
- uploadFile(fileName: string, content: Buffer, targetRelativePath?: string): Promise<DiffResult>;
89
+ getUploadRelativePath(fileName: string, targetRelativePath?: string): string;
90
+ uploadFile(fileName: string, content: Buffer, targetRelativePath?: string, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
91
+ vectorMode?: 'sync' | 'defer';
92
+ }): Promise<DiffResult>;
49
93
  listFailedFiles(): DiffResult['skippedFiles'];
50
94
  removeFile(relativePath: string): Promise<void>;
51
95
  tagFile(relativePath: string, tags: string[]): void;
@@ -68,7 +112,30 @@ export declare class KnowledgeBaseManager {
68
112
  }): Promise<VectorIndexResult[]>;
69
113
  getProjectConfig(): ProjectConfig | undefined;
70
114
  getStats(): KnowledgeBaseStats;
115
+ getVectorStatus(): {
116
+ status: string;
117
+ error?: string;
118
+ indexedChunks: number;
119
+ lastIndexedAt: number;
120
+ backend: string;
121
+ };
122
+ private rewriteQueries;
123
+ private llmExpandQueries;
124
+ private retrievalWeights;
125
+ private heuristicRerank;
126
+ private llmRerank;
127
+ private toFederatedItem;
128
+ private mergeHybridItems;
129
+ private parseChunkIndex;
130
+ private parseMetadataString;
131
+ private parseMetadata;
132
+ private metadataString;
133
+ private metadataFacets;
71
134
  close(): void;
135
+ private reportProgress;
136
+ private ensureVectorStore;
137
+ private deleteVectorFile;
138
+ private ensureVectorIndexFresh;
72
139
  private hasUsableContent;
73
140
  private defaultUploadRelativePath;
74
141
  private resolveKbRelativePath;