@customize-agent/knowledge 4.0.1 → 4.0.2

package/dist/core/index-state-store.js

@@ -56,11 +56,64 @@ export class IndexStateStore {
     `).all();
         return rows.map(row => this.rowToRecord(row));
     }
+    enqueueIndexJob(job) {
+        const now = Date.now();
+        this.db.prepare(`
+      INSERT INTO kb_index_jobs (id, relative_path, status, percent, message, created_at, updated_at)
+      VALUES (?, ?, 'PENDING', 0, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET status = 'PENDING', percent = 0, message = excluded.message, error_message = NULL, updated_at = excluded.updated_at
+    `).run(job.id, job.relativePath, job.message ?? '等待后台索引', now, now);
+        return this.getIndexJob(job.id);
+    }
+    updateIndexJob(id, patch) {
+        const current = this.getIndexJob(id);
+        if (!current)
+            return;
+        this.db.prepare(`
+      UPDATE kb_index_jobs
+      SET status = ?, percent = ?, message = ?, error_message = ?, updated_at = ?
+      WHERE id = ?
+    `).run(patch.status ?? current.status, patch.percent ?? current.percent, patch.message ?? current.message, patch.errorMessage ?? null, Date.now(), id);
+    }
+    getIndexJob(id) {
+        const row = this.db.prepare('SELECT * FROM kb_index_jobs WHERE id = ?').get(id);
+        return row ? this.rowToJob(row) : undefined;
+    }
+    listPendingIndexJobs(limit = 20) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_jobs
+      WHERE status = 'PENDING'
+      ORDER BY created_at ASC
+      LIMIT ?
+    `).all(limit);
+        return rows.map(row => this.rowToJob(row));
+    }
+    countPendingIndexJobs() {
+        const row = this.db.prepare("SELECT COUNT(*) as count FROM kb_index_jobs WHERE status = 'PENDING'").get();
+        return Number(row?.count ?? 0);
+    }
+    listActiveIndexJobsByPath(relativePath) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_jobs
+      WHERE relative_path = ? AND status IN ('PENDING', 'PARSING', 'CHUNKING', 'INDEXING')
+      ORDER BY created_at ASC
+    `).all(relativePath);
+        return rows.map(row => this.rowToJob(row));
+    }
+    listIndexJobsByPrefix(prefix) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_jobs
+      WHERE id LIKE ?
+      ORDER BY created_at ASC
+    `).all(`${prefix}%`);
+        return rows.map(row => this.rowToJob(row));
+    }
     replaceChunks(relativePath, chunks, file) {
         const now = Date.now();
         const transaction = this.db.transaction(() => {
             this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
             this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
+            this.db.prepare('DELETE FROM kb_document_chunks WHERE relative_path = ?').run(relativePath);
             if (this.ftsEnabled)
                 this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
             const insert = this.db.prepare(`
@@ -78,6 +131,12 @@ export class IndexStateStore {
           id, relative_path, parent_id, content, category, format,
           collection_name, section_title, chunk_count, metadata_json, created_at
         ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+            const insertDocument = this.db.prepare(`
+        INSERT INTO kb_document_chunks (
+          id, relative_path, content, category, format,
+          collection_name, parent_count, chunk_count, metadata_json, created_at
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       `);
             const parentGroups = new Map();
             const groupedChunks = this.splitParentGroups(relativePath, chunks);
@@ -87,9 +146,13 @@ export class IndexStateStore {
                 group.push(chunk);
                 parentGroups.set(parentId, group);
             }
+            const parentContents = [];
             for (const [parentId, group] of parentGroups.entries()) {
-                insertParent.run(parentId, relativePath, parentId, group.map(chunk => chunk.text).join('\n\n---\n\n'), file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind) }), now);
+                const parentContent = group.map(chunk => chunk.text).join('\n\n---\n\n');
+                parentContents.push(parentContent);
+                insertParent.run(parentId, relativePath, parentId, parentContent, file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind) }), now);
             }
+            insertDocument.run(`${relativePath}#document`, relativePath, parentContents.join('\n\n=== SECTION ===\n\n'), file.category, file.format, file.collectionName, parentGroups.size, groupedChunks.length, JSON.stringify({ parentType: 'document', splitStrategy: 'document_section_child_v1' }), now);
             for (const chunk of groupedChunks) {
                 const chunkId = `${relativePath}#${chunk.index}`;
                 insert.run(chunkId, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
@@ -114,16 +177,28 @@ export class IndexStateStore {
         if (options.limit)
             params.push(options.limit);
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       ${where}
       ORDER BY relative_path, chunk_index
       ${limit}
     `).all(...params);
         return rows.map(row => this.rowToChunk(row, 0));
     }
+    getChunkByRowid(rowid) {
+        const row = this.db.prepare('SELECT rowid, * FROM kb_chunks WHERE rowid = ?').get(rowid);
+        return row ? this.rowToChunk(row, 0) : undefined;
+    }
+    getChunksByRowids(rowids) {
+        if (rowids.length === 0)
+            return [];
+        const placeholders = rowids.map(() => '?').join(',');
+        const rows = this.db.prepare(`SELECT rowid, * FROM kb_chunks WHERE rowid IN (${placeholders})`).all(...rowids);
+        const byRowid = new Map(rows.map(row => [Number(row.rowid), this.rowToChunk(row, 0)]));
+        return rowids.flatMap(rowid => byRowid.get(rowid) ?? []);
+    }
     getContextChunks(relativePath, chunkIndex, window = 1) {
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       WHERE relative_path = ? AND chunk_index BETWEEN ? AND ?
       ORDER BY chunk_index
     `).all(relativePath, Math.max(0, chunkIndex - window), chunkIndex + window);
@@ -145,9 +220,17 @@ export class IndexStateStore {
     `).get(relativePath, parentId);
         return row ? this.rowToParentChunk(row) : undefined;
     }
+    getDocumentChunk(relativePath) {
+        const row = this.db.prepare(`
+      SELECT * FROM kb_document_chunks
+      WHERE relative_path = ?
+      LIMIT 1
+    `).get(relativePath);
+        return row ? this.rowToDocumentChunk(row) : undefined;
+    }
     getChunksByParent(relativePath, parentId, limit = 6) {
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       WHERE relative_path = ? AND metadata_json LIKE ?
       ORDER BY chunk_index
       LIMIT ?
@@ -158,12 +241,11 @@ export class IndexStateStore {
         const terms = this.expandSearchTerms(query);
         if (terms.length === 0)
             return [];
-        if (this.ftsEnabled) {
-            const ftsResults = this.searchChunksFts(terms, limit);
-            if (ftsResults.length > 0)
-                return ftsResults;
-        }
-        return this.searchChunksLike(terms, limit);
+        const results = [
+            ...(this.ftsEnabled ? this.searchChunksFts(terms, limit) : []),
+            ...this.searchChunksLike(terms, limit),
+        ];
+        return this.mergeKeywordResults(results, limit);
     }
     searchChunksFts(terms, limit) {
         try {
@@ -171,7 +253,7 @@ export class IndexStateStore {
             if (!matchQuery)
                 return [];
             const rows = this.db.prepare(`
-        SELECT c.*, bm25(kb_chunks_fts, 1.2, 0.8, 0.6, 1.0, 2.0) as bm25_score
+        SELECT c.rowid, c.*, bm25(kb_chunks_fts, 1.2, 0.8, 0.6, 1.0, 2.0) as bm25_score
         FROM kb_chunks_fts
         INNER JOIN kb_chunks c ON c.id = kb_chunks_fts.id
         WHERE kb_chunks_fts MATCH ?
@@ -194,7 +276,7 @@ export class IndexStateStore {
     }
     searchChunksLike(terms, limit) {
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       WHERE ${terms.map(() => '(LOWER(content) LIKE ? OR LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ?)').join(' OR ')}
       ORDER BY created_at DESC
       LIMIT ?
@@ -331,6 +413,7 @@ export class IndexStateStore {
     deleteRecord(relativePath) {
         this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
         this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_document_chunks WHERE relative_path = ?').run(relativePath);
         if (this.ftsEnabled)
             this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
         this.db.prepare('DELETE FROM kb_index_state WHERE relative_path = ?').run(relativePath);
@@ -398,7 +481,8 @@ export class IndexStateStore {
       CREATE INDEX IF NOT EXISTS idx_kb_state_collection ON kb_index_state(collection_name);
 
       CREATE TABLE IF NOT EXISTS kb_chunks (
-        id              TEXT PRIMARY KEY,
+        rowid           INTEGER PRIMARY KEY AUTOINCREMENT,
+        id              TEXT NOT NULL UNIQUE,
         relative_path   TEXT NOT NULL,
         chunk_index     INTEGER NOT NULL,
         content         TEXT NOT NULL,
@@ -430,6 +514,20 @@ export class IndexStateStore {
       CREATE INDEX IF NOT EXISTS idx_kb_parent_path ON kb_parent_chunks(relative_path);
       CREATE INDEX IF NOT EXISTS idx_kb_parent_id ON kb_parent_chunks(parent_id);
 
+      CREATE TABLE IF NOT EXISTS kb_document_chunks (
+        id              TEXT PRIMARY KEY,
+        relative_path   TEXT NOT NULL,
+        content         TEXT NOT NULL,
+        category        TEXT NOT NULL,
+        format          TEXT NOT NULL,
+        collection_name TEXT NOT NULL,
+        parent_count    INTEGER NOT NULL,
+        chunk_count     INTEGER NOT NULL,
+        metadata_json   TEXT,
+        created_at      INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_document_path ON kb_document_chunks(relative_path);
+
       CREATE TABLE IF NOT EXISTS kb_file_hashes (
         content_hash     TEXT NOT NULL,
         file_path        TEXT PRIMARY KEY,
@@ -481,6 +579,19 @@ export class IndexStateStore {
       );
       CREATE INDEX IF NOT EXISTS idx_tags_tag ON kb_tags(tag);
 
+      CREATE TABLE IF NOT EXISTS kb_index_jobs (
+        id             TEXT PRIMARY KEY,
+        relative_path  TEXT NOT NULL,
+        status         TEXT NOT NULL,
+        percent        INTEGER NOT NULL DEFAULT 0,
+        message        TEXT NOT NULL DEFAULT '',
+        error_message  TEXT,
+        created_at     INTEGER NOT NULL,
+        updated_at     INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_jobs_status ON kb_index_jobs(status);
+      CREATE INDEX IF NOT EXISTS idx_kb_jobs_path ON kb_index_jobs(relative_path);
+
       CREATE TABLE IF NOT EXISTS kb_metadata (
         key   TEXT PRIMARY KEY,
         value TEXT NOT NULL
@@ -575,7 +686,7 @@ export class IndexStateStore {
             relativePath: String(row.relative_path),
             parentId: String(row.parent_id),
             content: String(row.content),
-            category: String(row.category),
+            category: row.category,
             format: String(row.format),
             collectionName: String(row.collection_name),
             sectionTitle: row.section_title == null ? undefined : String(row.section_title),
@@ -584,45 +695,41 @@ export class IndexStateStore {
             createdAt: Number(row.created_at),
         };
     }
-    splitParentGroups(relativePath, chunks) {
-        const maxChildrenPerParent = 12;
-        const maxTokensPerParent = 4_000;
-        const grouped = new Map();
-        for (const chunk of chunks) {
-            const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
-            const list = grouped.get(parentId) ?? [];
-            list.push(chunk);
-            grouped.set(parentId, list);
-        }
-        const result = [];
-        for (const [parentId, list] of grouped.entries()) {
-            let batch = [];
-            let tokenCount = 0;
-            let batchIndex = 0;
-            const flush = () => {
-                if (batch.length === 0)
-                    return;
-                const nextParentId = list.length <= maxChildrenPerParent && tokenCount <= maxTokensPerParent ? parentId : `${parentId}@${batchIndex + 1}`;
-                result.push(...batch.map(chunk => ({ ...chunk, metadata: { ...chunk.metadata, parentId: nextParentId, parentGroupIndex: batchIndex } })));
-                batch = [];
-                tokenCount = 0;
-                batchIndex += 1;
-            };
-            for (const chunk of list) {
-                if (batch.length > 0 && (batch.length >= maxChildrenPerParent || tokenCount + chunk.tokenCount > maxTokensPerParent))
-                    flush();
-                batch.push(chunk);
-                tokenCount += chunk.tokenCount;
-            }
-            flush();
-        }
-        return result.sort((a, b) => a.index - b.index);
+    rowToDocumentChunk(row) {
+        return {
+            id: String(row.id),
+            relativePath: String(row.relative_path),
+            content: String(row.content),
+            category: row.category,
+            format: String(row.format),
+            collectionName: String(row.collection_name),
+            parentCount: Number(row.parent_count),
+            chunkCount: Number(row.chunk_count),
+            metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
+            createdAt: Number(row.created_at),
+        };
+    }
+    splitParentGroups(_relativePath, chunks) {
+        return chunks;
+    }
+    rowToJob(row) {
+        return {
+            id: String(row.id),
+            relativePath: String(row.relative_path),
+            status: String(row.status),
+            percent: Number(row.percent),
+            message: String(row.message ?? ''),
+            errorMessage: row.error_message == null ? undefined : String(row.error_message),
+            createdAt: Number(row.created_at),
+            updatedAt: Number(row.updated_at),
+        };
     }
     metadataString(value) {
         return typeof value === 'string' ? value : undefined;
     }
     rowToChunk(row, score, scoreDetails) {
         return {
+            rowid: Number(row.rowid ?? 0),
             id: String(row.id),
             relativePath: String(row.relative_path),
             chunkIndex: Number(row.chunk_index),
@@ -641,8 +748,13 @@ export class IndexStateStore {
     expandSearchTerms(query) {
         const normalized = query.toLowerCase().trim();
         const terms = new Set(normalized ? [normalized] : []);
-        for (const term of normalized.split(/[\s,，。；;：:、]+/u).filter(Boolean))
+        for (const term of normalized.split(/[\s,，。；;：:、]+/u).filter(Boolean)) {
             terms.add(term);
+            for (const gram of this.chineseNgrams(term))
+                terms.add(gram);
+        }
+        for (const gram of this.chineseNgrams(normalized))
+            terms.add(gram);
         const synonyms = {
             招标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
             投标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
@@ -656,7 +768,7 @@ export class IndexStateStore {
                 for (const value of values)
                     terms.add(value.toLowerCase());
         }
-        return [...terms];
+        return [...terms].filter(term => term.length > 0).slice(0, 40);
     }
     toFtsQuery(terms) {
         const normalized = terms.map(term => term.replace(/["*^:(){}\]\\[]/gu, ' ').trim()).filter(term => term.length > 0);
@@ -664,10 +776,32 @@ export class IndexStateStore {
         const weak = normalized.slice(1).filter(term => term.length >= 2).slice(0, 12);
         return [exact ? `"${exact}"` : '', ...weak.map(term => `"${term}"`)].filter(Boolean).join(' OR ');
     }
+    mergeKeywordResults(results, limit) {
+        const byId = new Map();
+        for (const result of results) {
+            const existing = byId.get(result.id);
+            if (!existing || result.score > existing.score)
+                byId.set(result.id, result);
+        }
+        return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
+    }
+    chineseNgrams(text) {
+        const han = text.match(/[\p{Script=Han}]+/gu) ?? [];
+        const grams = [];
+        for (const token of han) {
+            for (let size = 2; size <= 3; size++) {
+                if (token.length <= size)
+                    continue;
+                for (let index = 0; index <= token.length - size; index++)
+                    grams.push(token.slice(index, index + size));
+            }
+        }
+        return grams;
+    }
     bm25ToPositiveScore(score) {
         if (!Number.isFinite(score))
             return 0;
-        return 1 / (1 + Math.max(0, score));
+        return score < 0 ? -score : 1 / (1 + score);
     }
     scoreChunkDetailed(content, terms) {
         const lower = content.toLowerCase();

package/dist/core/knowledge-base-manager.d.ts

@@ -34,7 +34,7 @@ export declare class KnowledgeBaseManager {
     readonly projectId?: string;
     readonly kbPath: string;
     readonly store: IndexStateStore;
-    private readonly sqliteVecClient;
+    private readonly vectorRoot;
     private readonly classifier;
     private readonly scanner;
     private readonly collections;
@@ -51,9 +51,19 @@ export declare class KnowledgeBaseManager {
     private onProgress?;
     constructor(options: KnowledgeBaseManagerOptions);
     initialize(): void;
+    forceReindexAll(options?: {
+        onProgress?: (progress: KnowledgeIndexProgress) => void;
+        vectorMode?: 'sync' | 'defer';
+    }): Promise<DiffResult>;
+    consumePendingIndexJobs(options?: {
+        onProgress?: (progress: KnowledgeIndexProgress) => void;
+        vectorMode?: 'sync' | 'defer';
+        limit?: number;
+    }): Promise<DiffResult>;
     incrementalIndex(options?: {
         onProgress?: (progress: KnowledgeIndexProgress) => void;
         vectorMode?: 'sync' | 'defer';
+        onlyRelativePaths?: string[];
     }): Promise<DiffResult>;
     search(query: string, limit?: number): ChunkSearchResult[];
     keywordSearchItems(query: string, limit?: number): FederatedSearchItem[];
@@ -90,6 +100,11 @@ export declare class KnowledgeBaseManager {
     uploadFile(fileName: string, content: Buffer, targetRelativePath?: string, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
         vectorMode?: 'sync' | 'defer';
     }): Promise<DiffResult>;
+    stageUploadedFiles(files: Array<{
+        fileName: string;
+        content: Buffer;
+        targetRelativePath?: string;
+    }>, operationId?: string): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
     uploadFiles(files: Array<{
         fileName: string;
         content: Buffer;
@@ -119,6 +134,9 @@ export declare class KnowledgeBaseManager {
     }): Promise<VectorIndexResult[]>;
     getProjectConfig(): ProjectConfig | undefined;
     getStats(): KnowledgeBaseStats;
+    listIndexJobsByPrefix(prefix: string): import("./index-state-store.js").KnowledgeIndexJob[];
+    countPendingIndexJobs(): number;
+    failPendingIndexJobs(message: string): void;
     getVectorStatus(): {
         status: string;
         error?: string;
@@ -131,8 +149,11 @@ export declare class KnowledgeBaseManager {
     private retrievalWeights;
     private heuristicRerank;
     private llmRerank;
+    private hydrateVectorResultsFromSqlite;
     private toFederatedItem;
-    private mergeHybridItems;
+    private mergeHybridRankedLists;
+    private mergeContexts;
+    private contextKey;
     private parseChunkIndex;
     private parseMetadataString;
     private parseMetadata;
@@ -140,6 +161,7 @@ export declare class KnowledgeBaseManager {
     private metadataFacets;
     close(): void;
     private reportProgress;
+    private updateJobsForFile;
     private ensureVectorStore;
     private deleteVectorFile;
     private ensureVectorIndexFresh;