@customize-agent/knowledge 4.1.0 → 4.1.2

package/dist/core/index-state-store.js

@@ -180,7 +180,7 @@ export class IndexStateStore {
                 const rowRange = this.metadataString(chunk.metadata.rowRange) ?? null;
                 const searchContent = this.buildChunkSearchContent(relativePath, file, chunk, titlePath);
                 insert.run(chunkId, relativePath, chunk.index, chunk.text, searchContent, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, titlePath || null, parentId, chunkKind, rowRange, Number(chunk.metadata.startChar ?? chunk.startChar), Number(chunk.metadata.endChar ?? chunk.endChar), JSON.stringify(chunk.metadata), now);
-                insertFts?.run(chunkId, relativePath, file.category, file.format, chunk.sectionTitle ?? '', titlePath, chunkKind ?? '', searchContent);
+                insertFts?.run(chunkId, relativePath, file.category, file.format, chunk.sectionTitle ?? '', titlePath, chunkKind ?? '', `${searchContent} ${chunk.text}`);
             }
         });
         transaction();
@@ -357,12 +357,20 @@ export class IndexStateStore {
         }
     }
     searchChunksLike(terms, limit, filePaths) {
+        // 兜底拆两组：小列（路径/分类/标题等，行均几十字节）对所有词全表 LIKE，成本毫秒级；
+        // 大列（search_content/content）仅对 <3 字符短词扫描——trigram FTS 无法索引短词
+        // （中文 2 字词如“验收”），3+ 字符词已由 FTS 覆盖，不再全表扫大列
+        // （历史缺陷：40 term × 7 列全表扫描阻塞 Node 事件循环数分钟）
+        const shortTerms = [...new Set(terms.filter(term => term.length < 3))].slice(0, 6);
+        const smallColumns = terms.map(() => '(LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ? OR LOWER(COALESCE(title_path, \'\')) LIKE ? OR LOWER(COALESCE(chunk_kind, \'\')) LIKE ? OR LOWER(COALESCE(section_title, \'\')) LIKE ?)').join(' OR ');
+        const largeColumns = shortTerms.map(() => '(LOWER(search_content) LIKE ? OR LOWER(content) LIKE ?)').join(' OR ');
+        const condition = [smallColumns, largeColumns].filter(Boolean).join(' OR ');
         const rows = this.db.prepare(`
       SELECT rowid, * FROM kb_chunks
-      WHERE (${terms.map(() => '(LOWER(search_content) LIKE ? OR LOWER(content) LIKE ? OR LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ? OR LOWER(COALESCE(title_path, \'\')) LIKE ? OR LOWER(COALESCE(chunk_kind, \'\')) LIKE ?)').join(' OR ')})${this.filePathFilterClause(filePaths)}
+      WHERE (${condition})${this.filePathFilterClause(filePaths)}
       ORDER BY created_at DESC
       LIMIT ?
-    `).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), ...filePaths, limit * 6);
+    `).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), ...shortTerms.flatMap(term => [`%${term}%`, `%${term}%`]), ...filePaths, limit * 6);
         return rows
             .map(row => {
             const keyword = this.scoreChunkDetailed(this.searchableRowText(row), terms);
@@ -733,21 +741,48 @@ export class IndexStateStore {
     }
     initFts() {
         try {
-            this.db.exec(`
-        CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(
-          id UNINDEXED,
-          relative_path,
-          category,
-          format,
-          section_title,
-          title_path,
-          chunk_kind,
-          content,
-          tokenize = 'unicode61 remove_diacritics 2'
-        );
-      `);
-            this.ftsEnabled = true;
-            this.rebuildFtsIfNeeded();
+            const existing = this.db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'kb_chunks_fts'").get();
+            // unicode61 分词下中文子串（如“混凝土”）几乎无法命中 FTS，检索频繁落入
+            // LOWER(x) LIKE '%term%' 全表兜底（最多 40 term × 7 列），在大库上同步扫描会
+            // 阻塞 Node 事件循环数分钟（页面表现为转圈）。trigram 可对任意 >=3 字符子串建索引。
+            if (existing?.sql && !/tokenize\s*=\s*'trigram/u.test(existing.sql)) {
+                this.db.exec('DROP TABLE IF EXISTS kb_chunks_fts');
+            }
+            try {
+                this.db.exec(`
+          CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(
+            id UNINDEXED,
+            relative_path,
+            category,
+            format,
+            section_title,
+            title_path,
+            chunk_kind,
+            content,
+            tokenize = 'trigram case_sensitive 0'
+          );
+        `);
+                this.ftsEnabled = true;
+                this.rebuildFtsIfNeeded();
+            }
+            catch {
+                // 旧版 SQLite 无 trigram tokenizer 时回退 unicode61，保持原有行为
+                this.db.exec(`
+          CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(
+            id UNINDEXED,
+            relative_path,
+            category,
+            format,
+            section_title,
+            title_path,
+            chunk_kind,
+            content,
+            tokenize = 'unicode61 remove_diacritics 2'
+          );
+        `);
+                this.ftsEnabled = true;
+                this.rebuildFtsIfNeeded();
+            }
         }
         catch {
             this.ftsEnabled = false;
@@ -759,9 +794,11 @@ export class IndexStateStore {
         const row = this.db.prepare('SELECT COUNT(*) as count FROM kb_chunks_fts').get();
         if (Number(row.count ?? 0) > 0)
             return;
+        // content 列同时灌入 search_content 与正文：trigram 需要索引正文才能对正文内
+        // 中文子串召回，避免检索再次落入 LIKE 全表兜底
         this.db.prepare(`
       INSERT INTO kb_chunks_fts (id, relative_path, category, format, section_title, title_path, chunk_kind, content)
-      SELECT id, relative_path, category, format, COALESCE(section_title, ''), COALESCE(title_path, ''), COALESCE(chunk_kind, ''), COALESCE(search_content, content) FROM kb_chunks
+      SELECT id, relative_path, category, format, COALESCE(section_title, ''), COALESCE(title_path, ''), COALESCE(chunk_kind, ''), TRIM(COALESCE(search_content, '') || ' ' || COALESCE(content, '')) FROM kb_chunks
     `).run();
     }
     rowToMinHash(row) {
@@ -924,9 +961,10 @@ export class IndexStateStore {
         return [...terms].filter(term => term.length > 0).slice(0, 40);
     }
     toFtsQuery(terms) {
-        const normalized = terms.map(term => term.replace(/["*^:(){}\]\\[]/gu, ' ').trim()).filter(term => term.length > 0);
+        // trigram tokenizer 只支持 >=3 字符的查询 token，短词由 LIKE 小列兜底补齐
+        const normalized = terms.map(term => term.replace(/["*^:(){}\]\\[]/gu, ' ').trim()).filter(term => term.length >= 3);
         const exact = normalized[0];
-        const weak = normalized.slice(1).filter(term => term.length >= 2).slice(0, 12);
+        const weak = normalized.slice(1).filter(term => term.length >= 3).slice(0, 12);
         return [exact ? `"${exact}"` : '', ...weak.map(term => `"${term}"`)].filter(Boolean).join(' OR ');
     }
     mergeKeywordResults(results, limit) {

package/dist/vector/hnsw-vector-store.js

@@ -26,10 +26,24 @@ export class HNSWVectorStore {
         this.loadDocuments();
         const mod = require('hnswlib-node');
         this.index = new mod.HierarchicalNSW('cosine', this.dimensions);
-        if (fs.existsSync(this.indexPath))
+        const indexExists = fs.existsSync(this.indexPath);
+        if (indexExists) {
             this.index.readIndexSync(this.indexPath, true);
-        else
+            // 空壳索引防护：hnswlib v3.0.0 的 loadIndex 对空索引（cur_element_count=0）会把
+            // max_elements_ 覆盖为 0，此后任何 addPoint 都抛 "The number of elements exceeds
+            // the specified limit"（真实生成中向量化中断于空壳 kb_other 索引，状态永久 error）
+            if (this.index.getMaxElements() < 1 || this.index.getMaxElements() < this.index.getCurrentCount()) {
+                this.index.initIndex(this.maxElements, 16, 200, 100, true);
+            }
+            // 旧索引防护：documents sidecar 丢失/为空但索引文件仍有数据时，旧 label 与当前 sqlite
+            // rowid 脱节，语义检索按 label 回查 documents 全部落空（召回静默为零）→ 重建索引
+            if (this.documents.size === 0 && this.index.getCurrentCount() > 0) {
+                this.index.initIndex(this.maxElements, 16, 200, 100, true);
+            }
+        }
+        else {
             this.index.initIndex(this.maxElements, 16, 200, 100, true);
+        }
     }
     async upsert(documents, options = {}) {
         await this.ensureCollection();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.1.0",
+  "version": "4.1.2",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",