@customize-agent/knowledge 4.0.1 → 4.0.3

package/dist/core/index-state-store.d.ts

@@ -1,6 +1,18 @@
 import type { TextChunk } from '../chunking/text-chunker.js';
 import type { FileCategory, IndexStateRecord } from '../types.js';
+export type KnowledgeJobStatus = 'PENDING' | 'PARSING' | 'CHUNKING' | 'INDEXING' | 'SUCCESS' | 'ERROR';
+export interface KnowledgeIndexJob {
+    id: string;
+    relativePath: string;
+    status: KnowledgeJobStatus;
+    percent: number;
+    message: string;
+    errorMessage?: string;
+    createdAt: number;
+    updatedAt: number;
+}
 export interface StoredChunk {
+    rowid: number;
     id: string;
     relativePath: string;
     chunkIndex: number;
@@ -34,6 +46,18 @@ export interface StoredParentChunk {
     metadataJson?: string;
     createdAt: number;
 }
+export interface StoredDocumentChunk {
+    id: string;
+    relativePath: string;
+    content: string;
+    category: FileCategory;
+    format: string;
+    collectionName: string;
+    parentCount: number;
+    chunkCount: number;
+    metadataJson?: string;
+    createdAt: number;
+}
 export interface FileHashRecord {
     contentHash: string;
     filePath: string;
@@ -60,14 +84,40 @@ export interface FileRelationship {
     userConfirmed: number;
     createdAt: number;
 }
+/** 索引状态存储器，使用 SQLite 管理知识库索引的持久化状态 */
 export declare class IndexStateStore {
     private readonly db;
     private ftsEnabled;
     constructor(dbPath: string);
+    /** 加载所有活跃的索引记录 */
     loadActiveRecords(): Map<string, IndexStateRecord>;
+    /** 插入或更新索引记录 */
     upsertRecord(record: IndexStateRecord): void;
+    /** 更新已验证文件的时间戳和状态 */
     updateVerified(relativePath: string, mtime: number): void;
     listRecords(): IndexStateRecord[];
+    enqueueIndexJob(job: {
+        id: string;
+        relativePath: string;
+        message?: string;
+    }): KnowledgeIndexJob;
+    updateIndexJob(id: string, patch: {
+        status?: KnowledgeJobStatus;
+        percent?: number;
+        message?: string;
+        errorMessage?: string;
+    }): void;
+    getIndexJob(id: string): KnowledgeIndexJob | undefined;
+    listPendingIndexJobs(limit?: number): KnowledgeIndexJob[];
+    countPendingIndexJobs(): number;
+    listActiveIndexJobsByPath(relativePath: string): KnowledgeIndexJob[];
+    listIndexJobsByPrefix(prefix: string): KnowledgeIndexJob[];
+    /**
+     * 替换指定文件的切片数据（使用事务批量更新）
+     * @param relativePath 文件相对路径
+     * @param chunks 文本切片列表
+     * @param file 文件分类信息
+     */
     replaceChunks(relativePath: string, chunks: TextChunk[], file: {
         category: FileCategory;
         format: string;
@@ -78,10 +128,19 @@ export declare class IndexStateStore {
         relativePath?: string;
         limit?: number;
     }): StoredChunk[];
+    getChunkByRowid(rowid: number): StoredChunk | undefined;
+    getChunksByRowids(rowids: number[]): StoredChunk[];
     getContextChunks(relativePath: string, chunkIndex: number, window?: number): StoredChunk[];
     listParentChunks(relativePath: string): StoredParentChunk[];
     getParentChunk(relativePath: string, parentId: string): StoredParentChunk | undefined;
+    getDocumentChunk(relativePath: string): StoredDocumentChunk | undefined;
     getChunksByParent(relativePath: string, parentId: string, limit?: number): StoredChunk[];
+    /**
+     * 使用关键词搜索切片（支持 FTS5 全文搜索和 LIKE 模糊匹配）
+     * @param query 搜索查询
+     * @param limit 返回结果数量上限
+     * @returns 搜索结果列表（按相关性得分排序）
+     */
     searchChunks(query: string, limit?: number): ChunkSearchResult[];
     private searchChunksFts;
     private searchChunksLike;
@@ -106,6 +165,10 @@ export declare class IndexStateStore {
         enabled: boolean;
         createdAt: number;
     }>;
+    /**
+     * 删除指定文件的全部索引数据（含切片、哈希、MinHash、标签、关系等）
+     * @param relativePath 文件相对路径
+     */
     deleteRecord(relativePath: string): void;
     setMetadata(key: string, value: string): void;
     getMetadata(key: string): string | undefined;
@@ -127,11 +190,15 @@ export declare class IndexStateStore {
     private rowToFileHash;
     private rowToRelationship;
     private rowToParentChunk;
+    private rowToDocumentChunk;
     private splitParentGroups;
+    private rowToJob;
     private metadataString;
     private rowToChunk;
     private expandSearchTerms;
     private toFtsQuery;
+    private mergeKeywordResults;
+    private chineseNgrams;
     private bm25ToPositiveScore;
     private scoreChunkDetailed;
     private countOccurrences;

package/dist/core/index-state-store.js

@@ -1,6 +1,7 @@
 import Database from 'better-sqlite3';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+/** 索引状态存储器，使用 SQLite 管理知识库索引的持久化状态 */
 export class IndexStateStore {
     db;
     ftsEnabled = false;
@@ -10,6 +11,7 @@ export class IndexStateStore {
         this.db.pragma('journal_mode = WAL');
         this.initTables();
     }
+    /** 加载所有活跃的索引记录 */
     loadActiveRecords() {
         const rows = this.db.prepare(`
       SELECT * FROM kb_index_state
@@ -20,6 +22,7 @@ export class IndexStateStore {
             return [record.relativePath, record];
         }));
     }
+    /** 插入或更新索引记录 */
     upsertRecord(record) {
         this.db.prepare(`
       INSERT INTO kb_index_state (
@@ -41,6 +44,7 @@ export class IndexStateStore {
         metadata_json = excluded.metadata_json
     `).run(record.relativePath, record.category, record.format, record.contentHash, record.fileSize, Math.round(record.mtime), record.chunkCount, record.collectionName, record.indexedAt, record.lastVerifiedAt, record.status, record.errorMessage ?? null, record.metadataJson ?? null);
     }
+    /** 更新已验证文件的时间戳和状态 */
     updateVerified(relativePath, mtime) {
         this.db.prepare(`
       UPDATE kb_index_state
@@ -56,11 +60,70 @@ export class IndexStateStore {
     `).all();
         return rows.map(row => this.rowToRecord(row));
     }
+    enqueueIndexJob(job) {
+        const now = Date.now();
+        this.db.prepare(`
+      INSERT INTO kb_index_jobs (id, relative_path, status, percent, message, created_at, updated_at)
+      VALUES (?, ?, 'PENDING', 0, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET status = 'PENDING', percent = 0, message = excluded.message, error_message = NULL, updated_at = excluded.updated_at
+    `).run(job.id, job.relativePath, job.message ?? '等待后台索引', now, now);
+        return this.getIndexJob(job.id);
+    }
+    updateIndexJob(id, patch) {
+        const current = this.getIndexJob(id);
+        if (!current)
+            return;
+        this.db.prepare(`
+      UPDATE kb_index_jobs
+      SET status = ?, percent = ?, message = ?, error_message = ?, updated_at = ?
+      WHERE id = ?
+    `).run(patch.status ?? current.status, patch.percent ?? current.percent, patch.message ?? current.message, patch.errorMessage ?? null, Date.now(), id);
+    }
+    getIndexJob(id) {
+        const row = this.db.prepare('SELECT * FROM kb_index_jobs WHERE id = ?').get(id);
+        return row ? this.rowToJob(row) : undefined;
+    }
+    listPendingIndexJobs(limit = 20) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_jobs
+      WHERE status = 'PENDING'
+      ORDER BY created_at ASC
+      LIMIT ?
+    `).all(limit);
+        return rows.map(row => this.rowToJob(row));
+    }
+    countPendingIndexJobs() {
+        const row = this.db.prepare("SELECT COUNT(*) as count FROM kb_index_jobs WHERE status = 'PENDING'").get();
+        return Number(row?.count ?? 0);
+    }
+    listActiveIndexJobsByPath(relativePath) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_jobs
+      WHERE relative_path = ? AND status IN ('PENDING', 'PARSING', 'CHUNKING', 'INDEXING')
+      ORDER BY created_at ASC
+    `).all(relativePath);
+        return rows.map(row => this.rowToJob(row));
+    }
+    listIndexJobsByPrefix(prefix) {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_jobs
+      WHERE id LIKE ?
+      ORDER BY created_at ASC
+    `).all(`${prefix}%`);
+        return rows.map(row => this.rowToJob(row));
+    }
+    /**
+     * 替换指定文件的切片数据（使用事务批量更新）
+     * @param relativePath 文件相对路径
+     * @param chunks 文本切片列表
+     * @param file 文件分类信息
+     */
     replaceChunks(relativePath, chunks, file) {
         const now = Date.now();
         const transaction = this.db.transaction(() => {
             this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
             this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
+            this.db.prepare('DELETE FROM kb_document_chunks WHERE relative_path = ?').run(relativePath);
             if (this.ftsEnabled)
                 this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
             const insert = this.db.prepare(`
@@ -78,6 +141,12 @@ export class IndexStateStore {
           id, relative_path, parent_id, content, category, format,
           collection_name, section_title, chunk_count, metadata_json, created_at
         ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+            const insertDocument = this.db.prepare(`
+        INSERT INTO kb_document_chunks (
+          id, relative_path, content, category, format,
+          collection_name, parent_count, chunk_count, metadata_json, created_at
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       `);
             const parentGroups = new Map();
             const groupedChunks = this.splitParentGroups(relativePath, chunks);
@@ -87,9 +156,13 @@ export class IndexStateStore {
                 group.push(chunk);
                 parentGroups.set(parentId, group);
             }
+            const parentContents = [];
             for (const [parentId, group] of parentGroups.entries()) {
-                insertParent.run(parentId, relativePath, parentId, group.map(chunk => chunk.text).join('\n\n---\n\n'), file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind) }), now);
+                const parentContent = group.map(chunk => chunk.text).join('\n\n---\n\n');
+                parentContents.push(parentContent);
+                insertParent.run(parentId, relativePath, parentId, parentContent, file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind) }), now);
             }
+            insertDocument.run(`${relativePath}#document`, relativePath, parentContents.join('\n\n=== SECTION ===\n\n'), file.category, file.format, file.collectionName, parentGroups.size, groupedChunks.length, JSON.stringify({ parentType: 'document', splitStrategy: 'document_section_child_v1' }), now);
             for (const chunk of groupedChunks) {
                 const chunkId = `${relativePath}#${chunk.index}`;
                 insert.run(chunkId, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
@@ -114,16 +187,28 @@ export class IndexStateStore {
         if (options.limit)
             params.push(options.limit);
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       ${where}
       ORDER BY relative_path, chunk_index
       ${limit}
     `).all(...params);
         return rows.map(row => this.rowToChunk(row, 0));
     }
+    getChunkByRowid(rowid) {
+        const row = this.db.prepare('SELECT rowid, * FROM kb_chunks WHERE rowid = ?').get(rowid);
+        return row ? this.rowToChunk(row, 0) : undefined;
+    }
+    getChunksByRowids(rowids) {
+        if (rowids.length === 0)
+            return [];
+        const placeholders = rowids.map(() => '?').join(',');
+        const rows = this.db.prepare(`SELECT rowid, * FROM kb_chunks WHERE rowid IN (${placeholders})`).all(...rowids);
+        const byRowid = new Map(rows.map(row => [Number(row.rowid), this.rowToChunk(row, 0)]));
+        return rowids.flatMap(rowid => byRowid.get(rowid) ?? []);
+    }
     getContextChunks(relativePath, chunkIndex, window = 1) {
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       WHERE relative_path = ? AND chunk_index BETWEEN ? AND ?
       ORDER BY chunk_index
     `).all(relativePath, Math.max(0, chunkIndex - window), chunkIndex + window);
@@ -145,25 +230,38 @@ export class IndexStateStore {
     `).get(relativePath, parentId);
         return row ? this.rowToParentChunk(row) : undefined;
     }
+    getDocumentChunk(relativePath) {
+        const row = this.db.prepare(`
+      SELECT * FROM kb_document_chunks
+      WHERE relative_path = ?
+      LIMIT 1
+    `).get(relativePath);
+        return row ? this.rowToDocumentChunk(row) : undefined;
+    }
     getChunksByParent(relativePath, parentId, limit = 6) {
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       WHERE relative_path = ? AND metadata_json LIKE ?
       ORDER BY chunk_index
       LIMIT ?
     `).all(relativePath, `%"parentId":"${parentId.replace(/[%_]/gu, '')}"%`, limit);
         return rows.map(row => this.rowToChunk(row, 0));
     }
+    /**
+     * 使用关键词搜索切片（支持 FTS5 全文搜索和 LIKE 模糊匹配）
+     * @param query 搜索查询
+     * @param limit 返回结果数量上限
+     * @returns 搜索结果列表（按相关性得分排序）
+     */
     searchChunks(query, limit = 10) {
         const terms = this.expandSearchTerms(query);
         if (terms.length === 0)
             return [];
-        if (this.ftsEnabled) {
-            const ftsResults = this.searchChunksFts(terms, limit);
-            if (ftsResults.length > 0)
-                return ftsResults;
-        }
-        return this.searchChunksLike(terms, limit);
+        const results = [
+            ...(this.ftsEnabled ? this.searchChunksFts(terms, limit) : []),
+            ...this.searchChunksLike(terms, limit),
+        ];
+        return this.mergeKeywordResults(results, limit);
     }
     searchChunksFts(terms, limit) {
         try {
@@ -171,7 +269,7 @@ export class IndexStateStore {
             if (!matchQuery)
                 return [];
             const rows = this.db.prepare(`
-        SELECT c.*, bm25(kb_chunks_fts, 1.2, 0.8, 0.6, 1.0, 2.0) as bm25_score
+        SELECT c.rowid, c.*, bm25(kb_chunks_fts, 1.2, 0.8, 0.6, 1.0, 2.0) as bm25_score
         FROM kb_chunks_fts
         INNER JOIN kb_chunks c ON c.id = kb_chunks_fts.id
         WHERE kb_chunks_fts MATCH ?
@@ -194,7 +292,7 @@ export class IndexStateStore {
     }
     searchChunksLike(terms, limit) {
         const rows = this.db.prepare(`
-      SELECT * FROM kb_chunks
+      SELECT rowid, * FROM kb_chunks
       WHERE ${terms.map(() => '(LOWER(content) LIKE ? OR LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ?)').join(' OR ')}
       ORDER BY created_at DESC
       LIMIT ?
@@ -328,9 +426,14 @@ export class IndexStateStore {
             createdAt: Number(row.created_at),
         }));
     }
+    /**
+     * 删除指定文件的全部索引数据（含切片、哈希、MinHash、标签、关系等）
+     * @param relativePath 文件相对路径
+     */
     deleteRecord(relativePath) {
         this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
         this.db.prepare('DELETE FROM kb_parent_chunks WHERE relative_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_document_chunks WHERE relative_path = ?').run(relativePath);
         if (this.ftsEnabled)
             this.db.prepare('DELETE FROM kb_chunks_fts WHERE relative_path = ?').run(relativePath);
         this.db.prepare('DELETE FROM kb_index_state WHERE relative_path = ?').run(relativePath);
@@ -398,7 +501,8 @@ export class IndexStateStore {
       CREATE INDEX IF NOT EXISTS idx_kb_state_collection ON kb_index_state(collection_name);
 
       CREATE TABLE IF NOT EXISTS kb_chunks (
-        id              TEXT PRIMARY KEY,
+        rowid           INTEGER PRIMARY KEY AUTOINCREMENT,
+        id              TEXT NOT NULL UNIQUE,
         relative_path   TEXT NOT NULL,
         chunk_index     INTEGER NOT NULL,
         content         TEXT NOT NULL,
@@ -430,6 +534,20 @@ export class IndexStateStore {
       CREATE INDEX IF NOT EXISTS idx_kb_parent_path ON kb_parent_chunks(relative_path);
       CREATE INDEX IF NOT EXISTS idx_kb_parent_id ON kb_parent_chunks(parent_id);
 
+      CREATE TABLE IF NOT EXISTS kb_document_chunks (
+        id              TEXT PRIMARY KEY,
+        relative_path   TEXT NOT NULL,
+        content         TEXT NOT NULL,
+        category        TEXT NOT NULL,
+        format          TEXT NOT NULL,
+        collection_name TEXT NOT NULL,
+        parent_count    INTEGER NOT NULL,
+        chunk_count     INTEGER NOT NULL,
+        metadata_json   TEXT,
+        created_at      INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_document_path ON kb_document_chunks(relative_path);
+
       CREATE TABLE IF NOT EXISTS kb_file_hashes (
         content_hash     TEXT NOT NULL,
         file_path        TEXT PRIMARY KEY,
@@ -481,6 +599,19 @@ export class IndexStateStore {
       );
       CREATE INDEX IF NOT EXISTS idx_tags_tag ON kb_tags(tag);
 
+      CREATE TABLE IF NOT EXISTS kb_index_jobs (
+        id             TEXT PRIMARY KEY,
+        relative_path  TEXT NOT NULL,
+        status         TEXT NOT NULL,
+        percent        INTEGER NOT NULL DEFAULT 0,
+        message        TEXT NOT NULL DEFAULT '',
+        error_message  TEXT,
+        created_at     INTEGER NOT NULL,
+        updated_at     INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_jobs_status ON kb_index_jobs(status);
+      CREATE INDEX IF NOT EXISTS idx_kb_jobs_path ON kb_index_jobs(relative_path);
+
       CREATE TABLE IF NOT EXISTS kb_metadata (
         key   TEXT PRIMARY KEY,
         value TEXT NOT NULL
@@ -575,7 +706,7 @@ export class IndexStateStore {
             relativePath: String(row.relative_path),
             parentId: String(row.parent_id),
             content: String(row.content),
-            category: String(row.category),
+            category: row.category,
             format: String(row.format),
             collectionName: String(row.collection_name),
             sectionTitle: row.section_title == null ? undefined : String(row.section_title),
@@ -584,45 +715,41 @@ export class IndexStateStore {
             createdAt: Number(row.created_at),
         };
     }
-    splitParentGroups(relativePath, chunks) {
-        const maxChildrenPerParent = 12;
-        const maxTokensPerParent = 4_000;
-        const grouped = new Map();
-        for (const chunk of chunks) {
-            const parentId = this.metadataString(chunk.metadata.parentId) ?? `${relativePath}#parent-${chunk.index}`;
-            const list = grouped.get(parentId) ?? [];
-            list.push(chunk);
-            grouped.set(parentId, list);
-        }
-        const result = [];
-        for (const [parentId, list] of grouped.entries()) {
-            let batch = [];
-            let tokenCount = 0;
-            let batchIndex = 0;
-            const flush = () => {
-                if (batch.length === 0)
-                    return;
-                const nextParentId = list.length <= maxChildrenPerParent && tokenCount <= maxTokensPerParent ? parentId : `${parentId}@${batchIndex + 1}`;
-                result.push(...batch.map(chunk => ({ ...chunk, metadata: { ...chunk.metadata, parentId: nextParentId, parentGroupIndex: batchIndex } })));
-                batch = [];
-                tokenCount = 0;
-                batchIndex += 1;
-            };
-            for (const chunk of list) {
-                if (batch.length > 0 && (batch.length >= maxChildrenPerParent || tokenCount + chunk.tokenCount > maxTokensPerParent))
-                    flush();
-                batch.push(chunk);
-                tokenCount += chunk.tokenCount;
-            }
-            flush();
-        }
-        return result.sort((a, b) => a.index - b.index);
+    rowToDocumentChunk(row) {
+        return {
+            id: String(row.id),
+            relativePath: String(row.relative_path),
+            content: String(row.content),
+            category: row.category,
+            format: String(row.format),
+            collectionName: String(row.collection_name),
+            parentCount: Number(row.parent_count),
+            chunkCount: Number(row.chunk_count),
+            metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
+            createdAt: Number(row.created_at),
+        };
+    }
+    splitParentGroups(_relativePath, chunks) {
+        return chunks;
+    }
+    rowToJob(row) {
+        return {
+            id: String(row.id),
+            relativePath: String(row.relative_path),
+            status: String(row.status),
+            percent: Number(row.percent),
+            message: String(row.message ?? ''),
+            errorMessage: row.error_message == null ? undefined : String(row.error_message),
+            createdAt: Number(row.created_at),
+            updatedAt: Number(row.updated_at),
+        };
     }
     metadataString(value) {
         return typeof value === 'string' ? value : undefined;
     }
     rowToChunk(row, score, scoreDetails) {
         return {
+            rowid: Number(row.rowid ?? 0),
             id: String(row.id),
             relativePath: String(row.relative_path),
             chunkIndex: Number(row.chunk_index),
@@ -641,8 +768,13 @@ export class IndexStateStore {
     expandSearchTerms(query) {
         const normalized = query.toLowerCase().trim();
         const terms = new Set(normalized ? [normalized] : []);
-        for (const term of normalized.split(/[\s,，。；;：:、]+/u).filter(Boolean))
+        for (const term of normalized.split(/[\s,，。；;：:、]+/u).filter(Boolean)) {
             terms.add(term);
+            for (const gram of this.chineseNgrams(term))
+                terms.add(gram);
+        }
+        for (const gram of this.chineseNgrams(normalized))
+            terms.add(gram);
         const synonyms = {
             招标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
             投标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
@@ -656,7 +788,7 @@ export class IndexStateStore {
                 for (const value of values)
                     terms.add(value.toLowerCase());
         }
-        return [...terms];
+        return [...terms].filter(term => term.length > 0).slice(0, 40);
     }
     toFtsQuery(terms) {
         const normalized = terms.map(term => term.replace(/["*^:(){}\]\\[]/gu, ' ').trim()).filter(term => term.length > 0);
@@ -664,10 +796,32 @@ export class IndexStateStore {
         const weak = normalized.slice(1).filter(term => term.length >= 2).slice(0, 12);
         return [exact ? `"${exact}"` : '', ...weak.map(term => `"${term}"`)].filter(Boolean).join(' OR ');
     }
+    mergeKeywordResults(results, limit) {
+        const byId = new Map();
+        for (const result of results) {
+            const existing = byId.get(result.id);
+            if (!existing || result.score > existing.score)
+                byId.set(result.id, result);
+        }
+        return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
+    }
+    chineseNgrams(text) {
+        const han = text.match(/[\p{Script=Han}]+/gu) ?? [];
+        const grams = [];
+        for (const token of han) {
+            for (let size = 2; size <= 3; size++) {
+                if (token.length <= size)
+                    continue;
+                for (let index = 0; index <= token.length - size; index++)
+                    grams.push(token.slice(index, index + size));
+            }
+        }
+        return grams;
+    }
     bm25ToPositiveScore(score) {
         if (!Number.isFinite(score))
             return 0;
-        return 1 / (1 + Math.max(0, score));
+        return score < 0 ? -score : 1 / (1 + score);
     }
     scoreChunkDetailed(content, terms) {
         const lower = content.toLowerCase();

package/dist/core/knowledge-base-manager.d.ts

@@ -34,7 +34,7 @@ export declare class KnowledgeBaseManager {
     readonly projectId?: string;
     readonly kbPath: string;
     readonly store: IndexStateStore;
-    private readonly sqliteVecClient;
+    private readonly vectorRoot;
     private readonly classifier;
     private readonly scanner;
     private readonly collections;
@@ -51,9 +51,19 @@ export declare class KnowledgeBaseManager {
     private onProgress?;
     constructor(options: KnowledgeBaseManagerOptions);
     initialize(): void;
+    forceReindexAll(options?: {
+        onProgress?: (progress: KnowledgeIndexProgress) => void;
+        vectorMode?: 'sync' | 'defer';
+    }): Promise<DiffResult>;
+    consumePendingIndexJobs(options?: {
+        onProgress?: (progress: KnowledgeIndexProgress) => void;
+        vectorMode?: 'sync' | 'defer';
+        limit?: number;
+    }): Promise<DiffResult>;
     incrementalIndex(options?: {
         onProgress?: (progress: KnowledgeIndexProgress) => void;
         vectorMode?: 'sync' | 'defer';
+        onlyRelativePaths?: string[];
     }): Promise<DiffResult>;
     search(query: string, limit?: number): ChunkSearchResult[];
     keywordSearchItems(query: string, limit?: number): FederatedSearchItem[];
@@ -90,6 +100,11 @@ export declare class KnowledgeBaseManager {
     uploadFile(fileName: string, content: Buffer, targetRelativePath?: string, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
         vectorMode?: 'sync' | 'defer';
     }): Promise<DiffResult>;
+    stageUploadedFiles(files: Array<{
+        fileName: string;
+        content: Buffer;
+        targetRelativePath?: string;
+    }>, operationId?: string): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
     uploadFiles(files: Array<{
         fileName: string;
         content: Buffer;
@@ -119,6 +134,9 @@ export declare class KnowledgeBaseManager {
     }): Promise<VectorIndexResult[]>;
     getProjectConfig(): ProjectConfig | undefined;
     getStats(): KnowledgeBaseStats;
+    listIndexJobsByPrefix(prefix: string): import("./index-state-store.js").KnowledgeIndexJob[];
+    countPendingIndexJobs(): number;
+    failPendingIndexJobs(message: string): void;
     getVectorStatus(): {
         status: string;
         error?: string;
@@ -131,8 +149,11 @@ export declare class KnowledgeBaseManager {
     private retrievalWeights;
     private heuristicRerank;
     private llmRerank;
+    private hydrateVectorResultsFromSqlite;
     private toFederatedItem;
-    private mergeHybridItems;
+    private mergeHybridRankedLists;
+    private mergeContexts;
+    private contextKey;
     private parseChunkIndex;
     private parseMetadataString;
     private parseMetadata;
@@ -140,6 +161,7 @@ export declare class KnowledgeBaseManager {
     private metadataFacets;
     close(): void;
     private reportProgress;
+    private updateJobsForFile;
     private ensureVectorStore;
     private deleteVectorFile;
     private ensureVectorIndexFresh;