@customize-agent/knowledge 2.0.0 → 2.1.0

package/dist/core/index-state-store.js

@@ -3,6 +3,7 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 export class IndexStateStore {
     db;
+    ftsEnabled = false;
     constructor(dbPath) {
         fs.mkdirSync(path.dirname(dbPath), { recursive: true });
         this.db = new Database(dbPath);
@@ -59,14 +60,40 @@ export class IndexStateStore {
         const now = Date.now();
         const transaction = this.db.transaction(() => {
             this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
+            this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
+            if (this.ftsEnabled)
+                this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
             const insert = this.db.prepare(`
         INSERT INTO kb_chunks (
           id, relative_path, chunk_index, content, category, format,
           collection_name, token_count, section_title, metadata_json, created_at
         ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       `);
-            for (const chunk of chunks) {
-                insert.run(`${relativePath}#${chunk.index}`, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
+            const insertFts = this.ftsEnabled ? this.db.prepare(`
+        INSERT INTO kb_chunks_fts (id, relative_path, category, format, section_title, content)
+        VALUES (?, ?, ?, ?, ?, ?)
+      `) : undefined;
+            const insertParent = this.db.prepare(`
+        INSERT INTO kb_parent_chunks (
+          id, relative_path, parent_id, content, category, format,
+          collection_name, section_title, chunk_count, metadata_json, created_at
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+            const parentGroups = new Map();
+            const groupedChunks = this.splitParentGroups(relativePath, chunks);
+            for (const chunk of groupedChunks) {
+                const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
+                const group = parentGroups.get(parentId) ?? [];
+                group.push(chunk);
+                parentGroups.set(parentId, group);
+            }
+            for (const [parentId, group] of parentGroups.entries()) {
+                insertParent.run(parentId, relativePath, parentId, group.map(chunk => chunk.text).join('\n\n---\n\n'), file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind) }), now);
+            }
+            for (const chunk of groupedChunks) {
+                const chunkId = `${relativePath}#${chunk.index}`;
+                insert.run(chunkId, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
+                insertFts?.run(chunkId, relativePath, file.category, file.format, chunk.sectionTitle ?? '', chunk.text);
             }
         });
         transaction();
@@ -94,10 +121,78 @@ export class IndexStateStore {
     `).all(...params);
         return rows.map(row => this.rowToChunk(row, 0));
     }
+    getContextChunks(relativePath, chunkIndex, window = 1) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_chunks
+      WHERE relative_path = ? AND chunk_index BETWEEN ? AND ?
+      ORDER BY chunk_index
+    `).all(relativePath, Math.max(0, chunkIndex - window), chunkIndex + window);
+        return rows.map(row => this.rowToChunk(row, 0));
+    }
+    listParentChunks(relativePath) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_parent_chunks
+      WHERE relative_path = ?
+      ORDER BY parent_id
+    `).all(relativePath);
+        return rows.map(row => this.rowToParentChunk(row));
+    }
+    getParentChunk(relativePath, parentId) {
+        const row = this.db.prepare(`
+      SELECT * FROM kb_parent_chunks
+      WHERE relative_path = ? AND parent_id = ?
+      LIMIT 1
+    `).get(relativePath, parentId);
+        return row ? this.rowToParentChunk(row) : undefined;
+    }
+    getChunksByParent(relativePath, parentId, limit = 6) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_chunks
+      WHERE relative_path = ? AND metadata_json LIKE ?
+      ORDER BY chunk_index
+      LIMIT ?
+    `).all(relativePath, `%"parentId":"${parentId.replace(/[%_]/gu, '')}"%`, limit);
+        return rows.map(row => this.rowToChunk(row, 0));
+    }
     searchChunks(query, limit = 10) {
         const terms = this.expandSearchTerms(query);
         if (terms.length === 0)
             return [];
+        if (this.ftsEnabled) {
+            const ftsResults = this.searchChunksFts(terms, limit);
+            if (ftsResults.length > 0)
+                return ftsResults;
+        }
+        return this.searchChunksLike(terms, limit);
+    }
+    searchChunksFts(terms, limit) {
+        try {
+            const matchQuery = this.toFtsQuery(terms);
+            if (!matchQuery)
+                return [];
+            const rows = this.db.prepare(`
+        SELECT c.*, bm25(kb_chunks_fts, 1.2, 0.8, 0.6, 1.0, 2.0) as bm25_score
+        FROM kb_chunks_fts
+        INNER JOIN kb_chunks c ON c.id = kb_chunks_fts.id
+        WHERE kb_chunks_fts MATCH ?
+        ORDER BY bm25_score ASC
+        LIMIT ?
+      `).all(matchQuery, limit * 8);
+            return rows
+                .map(row => {
+                const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
+                const bm25Score = this.bm25ToPositiveScore(Number(row.bm25_score));
+                return this.rowToChunk(row, keyword.keywordScore + bm25Score, { ...keyword, bm25Score });
+            })
+                .filter(row => row.score > 0)
+                .sort((a, b) => b.score - a.score)
+                .slice(0, limit);
+        }
+        catch {
+            return [];
+        }
+    }
+    searchChunksLike(terms, limit) {
         const rows = this.db.prepare(`
       SELECT * FROM kb_chunks
       WHERE ${terms.map(() => '(LOWER(content) LIKE ? OR LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ?)').join(' OR ')}
@@ -105,7 +200,10 @@ export class IndexStateStore {
       LIMIT ?
     `).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), limit * 6);
         return rows
-            .map(row => this.rowToChunk(row, this.scoreChunk(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms)))
+            .map(row => {
+            const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
+            return this.rowToChunk(row, keyword.keywordScore, keyword);
+        })
             .filter(row => row.score > 0)
             .sort((a, b) => b.score - a.score)
             .slice(0, limit);
@@ -232,6 +330,9 @@ export class IndexStateStore {
     }
     deleteRecord(relativePath) {
         this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
+        if (this.ftsEnabled)
+            this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
         this.db.prepare('DELETE FROM kb_index_state WHERE relative_path = ?').run(relativePath);
         this.db.prepare('DELETE FROM kb_file_hashes WHERE file_path = ?').run(relativePath);
         this.db.prepare('DELETE FROM kb_minhash WHERE file_path = ?').run(relativePath);
@@ -245,6 +346,10 @@ export class IndexStateStore {
       ON CONFLICT(key) DO UPDATE SET value = excluded.value
     `).run(key, value);
     }
+    getMetadata(key) {
+        const row = this.db.prepare('SELECT value FROM kb_metadata WHERE key = ?').get(key);
+        return row?.value;
+    }
     getStats() {
         const stats = this.db.prepare(`
       SELECT
@@ -309,6 +414,22 @@ export class IndexStateStore {
       CREATE INDEX IF NOT EXISTS idx_kb_chunks_category ON kb_chunks(category);
       CREATE INDEX IF NOT EXISTS idx_kb_chunks_collection ON kb_chunks(collection_name);
 
+      CREATE TABLE IF NOT EXISTS kb_parent_chunks (
+        id              TEXT PRIMARY KEY,
+        relative_path   TEXT NOT NULL,
+        parent_id       TEXT NOT NULL,
+        content         TEXT NOT NULL,
+        category        TEXT NOT NULL,
+        format          TEXT NOT NULL,
+        collection_name TEXT NOT NULL,
+        section_title   TEXT,
+        chunk_count     INTEGER NOT NULL,
+        metadata_json   TEXT,
+        created_at      INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_parent_path ON kb_parent_chunks(relative_path);
+      CREATE INDEX IF NOT EXISTS idx_kb_parent_id ON kb_parent_chunks(parent_id);
+
       CREATE TABLE IF NOT EXISTS kb_file_hashes (
         content_hash     TEXT NOT NULL,
         file_path        TEXT PRIMARY KEY,
@@ -372,7 +493,39 @@ export class IndexStateStore {
         created_at  INTEGER NOT NULL
       );
     `);
-        this.setMetadata('schema_version', '1');
+        this.initFts();
+        this.setMetadata('schema_version', '2');
+    }
+    initFts() {
+        try {
+            this.db.exec(`
+        CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(
+          id UNINDEXED,
+          relative_path,
+          category,
+          format,
+          section_title,
+          content,
+          tokenize = 'unicode61 remove_diacritics 2'
+        );
+      `);
+            this.ftsEnabled = true;
+            this.rebuildFtsIfNeeded();
+        }
+        catch {
+            this.ftsEnabled = false;
+        }
+    }
+    rebuildFtsIfNeeded() {
+        if (!this.ftsEnabled)
+            return;
+        const row = this.db.prepare('SELECT COUNT(*) as count FROM kb_chunks_fts').get();
+        if (Number(row.count ?? 0) > 0)
+            return;
+        this.db.prepare(`
+      INSERT INTO kb_chunks_fts (id, relative_path, category, format, section_title, content)
+      SELECT id, relative_path, category, format, COALESCE(section_title, ''), content FROM kb_chunks
+    `).run();
     }
     rowToMinHash(row) {
         const raw = row.signature;
@@ -410,7 +563,59 @@ export class IndexStateStore {
             createdAt: Number(row.created_at),
         };
     }
-    rowToChunk(row, score) {
+    rowToParentChunk(row) {
+        return {
+            id: String(row.id),
+            relativePath: String(row.relative_path),
+            parentId: String(row.parent_id),
+            content: String(row.content),
+            category: String(row.category),
+            format: String(row.format),
+            collectionName: String(row.collection_name),
+            sectionTitle: row.section_title == null ? undefined : String(row.section_title),
+            chunkCount: Number(row.chunk_count),
+            metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
+            createdAt: Number(row.created_at),
+        };
+    }
+    splitParentGroups(relativePath, chunks) {
+        const maxChildrenPerParent = 12;
+        const maxTokensPerParent = 4_000;
+        const grouped = new Map();
+        for (const chunk of chunks) {
+            const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
+            const list = grouped.get(parentId) ?? [];
+            list.push(chunk);
+            grouped.set(parentId, list);
+        }
+        const result = [];
+        for (const [parentId, list] of grouped.entries()) {
+            let batch = [];
+            let tokenCount = 0;
+            let batchIndex = 0;
+            const flush = () => {
+                if (batch.length === 0)
+                    return;
+                const nextParentId = list.length <= maxChildrenPerParent && tokenCount <= maxTokensPerParent ? parentId : `${parentId}@${batchIndex + 1}`;
+                result.push(...batch.map(chunk => ({ ...chunk, metadata: { ...chunk.metadata, parentId: nextParentId, parentGroupIndex: batchIndex } })));
+                batch = [];
+                tokenCount = 0;
+                batchIndex += 1;
+            };
+            for (const chunk of list) {
+                if (batch.length > 0 && (batch.length >= maxChildrenPerParent || tokenCount + chunk.tokenCount > maxTokensPerParent))
+                    flush();
+                batch.push(chunk);
+                tokenCount += chunk.tokenCount;
+            }
+            flush();
+        }
+        return result.sort((a, b) => a.index - b.index);
+    }
+    metadataString(value) {
+        return typeof value === 'string' ? value : undefined;
+    }
+    rowToChunk(row, score, scoreDetails) {
         return {
             id: String(row.id),
             relativePath: String(row.relative_path),
@@ -424,13 +629,14 @@ export class IndexStateStore {
             metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
             createdAt: Number(row.created_at),
             score,
+            scoreDetails,
         };
     }
     expandSearchTerms(query) {
         const normalized = query.toLowerCase().trim();
-        const terms = new Set(normalized.split(/[\s,，。；;：:、]+/u).filter(Boolean));
-        if (normalized)
-            terms.add(normalized);
+        const terms = new Set(normalized ? [normalized] : []);
+        for (const term of normalized.split(/[\s,，。；;：:、]+/u).filter(Boolean))
+            terms.add(term);
         const synonyms = {
             招标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
             投标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
@@ -446,17 +652,44 @@ export class IndexStateStore {
         }
         return [...terms];
     }
-    scoreChunk(content, terms) {
+    toFtsQuery(terms) {
+        const normalized = terms.map(term => term.replace(/["*^:(){}\]\\[]/gu, ' ').trim()).filter(term => term.length > 0);
+        const exact = normalized[0];
+        const weak = normalized.slice(1).filter(term => term.length >= 2).slice(0, 12);
+        return [exact ? `"${exact}"` : '', ...weak.map(term => `"${term}"`)].filter(Boolean).join(' OR ');
+    }
+    bm25ToPositiveScore(score) {
+        if (!Number.isFinite(score))
+            return 0;
+        return 1 / (1 + Math.max(0, score));
+    }
+    scoreChunkDetailed(content, terms) {
         const lower = content.toLowerCase();
-        let score = 0;
+        let raw = 0;
+        let exactPhraseBoost = 0;
+        const exactPhrase = terms[0] ?? '';
+        const exactHits = exactPhrase ? this.countOccurrences(lower, exactPhrase) : 0;
+        if (exactHits > 0)
+            exactPhraseBoost = 1000 + exactHits * 20;
+        raw += exactPhraseBoost;
         for (const term of terms) {
-            let index = lower.indexOf(term);
-            while (index !== -1) {
-                score += 1;
-                index = lower.indexOf(term, index + term.length);
-            }
+            if (term === exactPhrase)
+                continue;
+            raw += this.countOccurrences(lower, term) * 0.2;
+        }
+        return {
+            keywordScore: raw / Math.max(1, content.length / 1000),
+            exactPhraseBoost,
+        };
+    }
+    countOccurrences(content, term) {
+        let count = 0;
+        let index = content.indexOf(term);
+        while (index !== -1) {
+            count += 1;
+            index = content.indexOf(term, index + term.length);
         }
-        return score / Math.max(1, content.length / 1000);
+        return count;
     }
     rowToRecord(row) {
         return {

package/dist/core/knowledge-base-manager.d.ts

@@ -1,10 +1,20 @@
 import { type EmbeddingProvider } from '../embedding/embedding-provider.js';
 import type { ExternalExtractorRegistry } from '../extraction/external-extractor.js';
-import { type FederatedResult, type SearchFilters } from '../search/federation-search.js';
+import type { LLMSearchProvider } from '../llm/llm-search-provider.js';
+import { type FederatedResult, type FederatedSearchItem, type RetrievalWeights, type SearchFilters } from '../search/federation-search.js';
 import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
 import type { VectorStoreInterface } from '../vector/types.js';
 import { type VectorIndexResult } from '../vector/vector-indexer.js';
 import { IndexStateStore, type ChunkSearchResult, type FileRelationship } from './index-state-store.js';
+export type KnowledgeIndexStage = 'scanning' | 'parsing' | 'chunking' | 'vectorizing' | 'done' | 'error';
+export interface KnowledgeIndexProgress {
+    stage: KnowledgeIndexStage;
+    percent: number;
+    message: string;
+    filePath?: string;
+    chunkCount?: number;
+    vectorStatus?: ReturnType<KnowledgeBaseManager['getVectorStatus']>;
+}
 export interface KnowledgeBaseManagerOptions {
     scope: Exclude<KBScope, 'session'>;
     projectRoot?: string;
@@ -14,6 +24,9 @@ export interface KnowledgeBaseManagerOptions {
     embeddingProvider?: EmbeddingProvider;
     vectorStores?: Map<string, VectorStoreInterface>;
     externalExtractors?: ExternalExtractorRegistry;
+    onProgress?: (progress: KnowledgeIndexProgress) => void;
+    /** 可选的 LLM Provider，用于查询扩展和语义重排序 */
+    llmProvider?: LLMSearchProvider;
 }
 export declare class KnowledgeBaseManager {
     readonly scope: Exclude<KBScope, 'session'>;
@@ -21,6 +34,7 @@ export declare class KnowledgeBaseManager {
     readonly projectId?: string;
     readonly kbPath: string;
     readonly store: IndexStateStore;
+    private readonly chromaClient;
     private readonly classifier;
     private readonly scanner;
     private readonly collections;
@@ -33,10 +47,23 @@ export declare class KnowledgeBaseManager {
     private readonly configManager;
     private projectConfig?;
     private lastSkippedFiles;
+    private readonly llmProvider?;
+    private onProgress?;
     constructor(options: KnowledgeBaseManagerOptions);
     initialize(): void;
-    incrementalIndex(): Promise<DiffResult>;
+    incrementalIndex(options?: {
+        onProgress?: (progress: KnowledgeIndexProgress) => void;
+        vectorMode?: 'sync' | 'defer';
+    }): Promise<DiffResult>;
     search(query: string, limit?: number): ChunkSearchResult[];
+    keywordSearchItems(query: string, limit?: number): FederatedSearchItem[];
+    expandContext(item: FederatedSearchItem): FederatedSearchItem;
+    hybridSearch(query: string, options?: {
+        limit?: number;
+        filters?: SearchFilters;
+        collections?: string[];
+        weights?: RetrievalWeights;
+    }): Promise<FederatedResult>;
     semanticSearch(query: string, options?: {
         limit?: number;
         filters?: SearchFilters;
@@ -44,9 +71,25 @@ export declare class KnowledgeBaseManager {
     }): Promise<FederatedResult>;
     listRelationships(filePath?: string): FileRelationship[];
     listFiles(): IndexStateRecord[];
+    getFileDetail(relativePath: string): {
+        file: IndexStateRecord;
+        absolutePath: string;
+        directory: string;
+        chunks: import("./index-state-store.js").StoredChunk[];
+        parents: import("./index-state-store.js").StoredParentChunk[];
+        relationships: FileRelationship[];
+        tags: {
+            filePath: string;
+            tag: string;
+            createdAt: number;
+        }[];
+    } | undefined;
+    reindexFile(relativePath: string): Promise<DiffResult>;
     addFile(sourcePath: string, targetRelativePath?: string): Promise<DiffResult>;
     getUploadRelativePath(fileName: string, targetRelativePath?: string): string;
-    uploadFile(fileName: string, content: Buffer, targetRelativePath?: string): Promise<DiffResult>;
+    uploadFile(fileName: string, content: Buffer, targetRelativePath?: string, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
+        vectorMode?: 'sync' | 'defer';
+    }): Promise<DiffResult>;
     listFailedFiles(): DiffResult['skippedFiles'];
     removeFile(relativePath: string): Promise<void>;
     tagFile(relativePath: string, tags: string[]): void;
@@ -69,7 +112,30 @@ export declare class KnowledgeBaseManager {
     }): Promise<VectorIndexResult[]>;
     getProjectConfig(): ProjectConfig | undefined;
     getStats(): KnowledgeBaseStats;
+    getVectorStatus(): {
+        status: string;
+        error?: string;
+        indexedChunks: number;
+        lastIndexedAt: number;
+        backend: string;
+    };
+    private rewriteQueries;
+    private llmExpandQueries;
+    private retrievalWeights;
+    private heuristicRerank;
+    private llmRerank;
+    private toFederatedItem;
+    private mergeHybridItems;
+    private parseChunkIndex;
+    private parseMetadataString;
+    private parseMetadata;
+    private metadataString;
+    private metadataFacets;
     close(): void;
+    private reportProgress;
+    private ensureVectorStore;
+    private deleteVectorFile;
+    private ensureVectorIndexFresh;
     private hasUsableContent;
     private defaultUploadRelativePath;
     private resolveKbRelativePath;