@customize-agent/knowledge 4.0.5 → 4.0.7

package/dist/core/knowledge-base-manager.d.ts

@@ -108,6 +108,11 @@ export declare class KnowledgeBaseManager {
         content: Buffer;
         targetRelativePath?: string;
     }>, operationId?: string): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
+    stageUploadedFilePaths(files: Array<{
+        fileName: string;
+        sourcePath: string;
+        targetRelativePath?: string;
+    }>, operationId?: string, offset?: number): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
     uploadFiles(files: Array<{
         fileName: string;
         content: Buffer;
@@ -133,7 +138,10 @@ export declare class KnowledgeBaseManager {
     indexVectors(options?: {
         collectionName?: string;
         relativePath?: string;
+        relativePaths?: string[];
+        cleanupCollectionNames?: Iterable<string>;
         limit?: number;
+        rebuild?: boolean;
     }): Promise<VectorIndexResult[]>;
     getProjectConfig(): ProjectConfig | undefined;
     getStats(): KnowledgeBaseStats;
@@ -170,6 +178,7 @@ export declare class KnowledgeBaseManager {
     private ensureVectorIndexFresh;
     private hasUsableContent;
     private defaultUploadRelativePath;
+    private validateUploadRelativePath;
     private resolveKbRelativePath;
     private normalizeRelativePath;
 }

package/dist/core/knowledge-base-manager.js

@@ -105,6 +105,7 @@ export class KnowledgeBaseManager {
             const tracker = new ChangeTracker(this.store);
             const diff = await tracker.computeDiff(diskFiles, this.classifier, this.kbPath);
             const onlyRelativePaths = options.onlyRelativePaths ? new Set(options.onlyRelativePaths) : undefined;
+            let vectorDeletesApplied = 0;
             if (onlyRelativePaths) {
                 diff.newFiles = diff.newFiles.filter(file => onlyRelativePaths.has(file.relativePath));
                 diff.modifiedFiles = diff.modifiedFiles.filter(file => onlyRelativePaths.has(file.relativePath));
@@ -118,18 +119,24 @@ export class KnowledgeBaseManager {
             }
             for (const deleted of diff.deletedFiles) {
                 await this.deleteVectorFile(deleted.collectionName, deleted.relativePath);
+                vectorDeletesApplied += 1;
                 this.store.deleteRecord(deleted.relativePath);
                 this.updateJobsForFile(deleted.relativePath, 'SUCCESS', 100, '文件已删除，索引记录和向量已清理');
             }
             const now = Date.now();
             const indexedBefore = [...this.store.loadActiveRecords().values()];
             const filesToIndex = [...diff.newFiles, ...diff.modifiedFiles];
+            const vectorRelativePaths = [];
+            const changedCollectionNames = new Set();
             for (const [index, file] of filesToIndex.entries()) {
                 const hash = tracker.hashFile(file.absolutePath);
                 const duplicate = this.store.findExactDuplicate(hash, file.relativePath);
                 const collectionName = this.scope === 'global'
                     ? this.collections.getCollectionName('global', file.category)
                     : this.collections.getCollectionName('project', file.category, this.projectId);
+                const previousRecord = indexedBefore.find(record => record.relativePath === file.relativePath);
+                if (previousRecord?.collectionName)
+                    changedCollectionNames.add(previousRecord.collectionName);
                 const basePercent = filesToIndex.length === 0 ? 40 : 20 + Math.round((index / filesToIndex.length) * 45);
                 this.updateJobsForFile(file.relativePath, 'PARSING', basePercent, `正在解析 ${file.relativePath}`);
                 this.reportProgress({ stage: 'parsing', percent: basePercent, message: `正在解析 ${file.relativePath}`, filePath: file.relativePath });
@@ -244,6 +251,8 @@ export class KnowledgeBaseManager {
                     format: file.format,
                     collectionName,
                 });
+                vectorRelativePaths.push(file.relativePath);
+                changedCollectionNames.add(collectionName);
                 this.updateJobsForFile(file.relativePath, options.vectorMode === 'defer' ? 'SUCCESS' : 'INDEXING', options.vectorMode === 'defer' ? 100 : 85, options.vectorMode === 'defer' ? '解析和切片已完成' : '等待向量入库');
             }
             const stats = this.getStats();
@@ -257,7 +266,7 @@ export class KnowledgeBaseManager {
                 this.reportProgress({ stage: 'vectorizing', percent: 85, message: '解析和切片已完成，向量入库转入后台/稍后执行', chunkCount: stats.chunkCount, vectorStatus: this.getVectorStatus() });
             }
             else {
-                await this.ensureVectorIndexFresh(stats.chunkCount, hasIndexChanges);
+                await this.ensureVectorIndexFresh(stats.chunkCount, { changedRelativePaths: vectorRelativePaths, changedCollectionNames, deletesApplied: vectorDeletesApplied });
             }
             this.lastSkippedFiles = diff.skippedFiles;
             const vectorStatus = this.getVectorStatus();
@@ -417,7 +426,7 @@ export class KnowledgeBaseManager {
         return this.incrementalIndex();
     }
     getUploadRelativePath(fileName, targetRelativePath) {
-        return targetRelativePath ? this.normalizeRelativePath(targetRelativePath) : this.defaultUploadRelativePath(fileName);
+        return this.validateUploadRelativePath(targetRelativePath ? this.normalizeRelativePath(targetRelativePath) : this.defaultUploadRelativePath(fileName));
     }
     async uploadFile(fileName, content, targetRelativePath, onProgress, options = {}) {
         return this.uploadFiles([{ fileName, content, targetRelativePath }], onProgress, options);
@@ -439,6 +448,23 @@ export class KnowledgeBaseManager {
         }
         return jobs;
     }
+    async stageUploadedFilePaths(files, operationId = `upload-${Date.now()}`, offset = 0) {
+        this.initialize();
+        const jobs = [];
+        for (let index = 0; index < files.length; index++) {
+            const file = files[index];
+            const relativePath = this.getUploadRelativePath(file.fileName, file.targetRelativePath);
+            const targetPath = this.resolveKbRelativePath(relativePath);
+            fs.mkdirSync(path.dirname(targetPath), { recursive: true });
+            fs.copyFileSync(file.sourcePath, targetPath);
+            const record = this.store.listRecords().find(item => item.relativePath === relativePath);
+            if (record)
+                await this.deleteVectorFile(record.collectionName, relativePath);
+            this.store.deleteRecord(relativePath);
+            jobs.push(this.store.enqueueIndexJob({ id: `${operationId}-${offset + index}`, relativePath, message: '文件已落盘，等待后台解析' }));
+        }
+        return jobs;
+    }
     async uploadFiles(files, onProgress, options = {}) {
         await this.stageUploadedFiles(files);
         return this.incrementalIndex({ onProgress, vectorMode: options.vectorMode });
@@ -482,19 +508,36 @@ export class KnowledgeBaseManager {
         return this.store.listIgnoreRules();
     }
     async indexVectors(options = {}) {
-        const chunks = this.store.listChunks(options);
+        const chunks = options.relativePaths?.length
+            ? options.relativePaths.flatMap(relativePath => this.store.listChunks({ collectionName: options.collectionName, relativePath }))
+            : this.store.listChunks(options);
         const collectionNames = new Set(chunks.map(chunk => chunk.collectionName));
-        for (const collectionName of collectionNames)
+        const cleanupCollectionNames = new Set([...collectionNames, ...(options.cleanupCollectionNames ?? [])]);
+        for (const collectionName of cleanupCollectionNames)
             this.ensureVectorStore(collectionName);
-        if (!options.relativePath) {
+        if (options.rebuild || (!options.relativePath && !options.relativePaths?.length)) {
             for (const collectionName of collectionNames)
                 await this.vectorStores.get(collectionName)?.clearCollection?.();
         }
         else {
-            for (const collectionName of collectionNames)
-                await this.vectorStores.get(collectionName)?.deleteByFilePath(options.relativePath);
+            for (const relativePath of options.relativePaths ?? [options.relativePath].filter(Boolean)) {
+                for (const collectionName of cleanupCollectionNames)
+                    await this.vectorStores.get(collectionName)?.deleteByFilePath(relativePath, { persist: false });
+            }
+            if (chunks.length === 0) {
+                for (const collectionName of cleanupCollectionNames)
+                    await this.vectorStores.get(collectionName)?.flush?.();
+            }
         }
         this.reportProgress({ stage: 'vectorizing', percent: 85, message: `正在写入 HNSWLib 向量库，共 ${chunks.length} 个切片`, chunkCount: chunks.length });
+        if (chunks.length === 0) {
+            const totalChunks = this.getStats().chunkCount;
+            this.store.setMetadata('vector_indexed_chunks', String(totalChunks));
+            this.store.setMetadata('vector_index_status', 'ready');
+            this.store.setMetadata('vector_index_error', '');
+            this.store.setMetadata('last_vector_index_at', String(Date.now()));
+            return [];
+        }
         const indexer = new VectorIndexer(this.embeddingProvider, this.vectorStores);
         let lastVectorPercent = -1;
         try {
@@ -512,9 +555,10 @@ export class KnowledgeBaseManager {
             });
             const actualModel = results[0]?.embeddingModel ?? this.embeddingProvider.model;
             const actualDimension = results[0]?.embeddingDimension ?? this.embeddingProvider.dimensions;
+            const totalChunks = this.getStats().chunkCount;
             this.store.setMetadata('embedding_model', actualModel);
             this.store.setMetadata('embedding_dimension', String(actualDimension));
-            this.store.setMetadata('vector_indexed_chunks', String(chunks.length));
+            this.store.setMetadata('vector_indexed_chunks', String(totalChunks));
             this.store.setMetadata('vector_index_status', 'ready');
             this.store.setMetadata('vector_index_error', '');
             this.store.setMetadata('last_vector_index_at', String(Date.now()));
@@ -854,20 +898,34 @@ ${resultsText}
             this.store.setMetadata('vector_index_error', error instanceof Error ? error.message : String(error));
         }
     }
-    async ensureVectorIndexFresh(chunkCount, force = false) {
+    async ensureVectorIndexFresh(chunkCount, options = {}) {
         if (chunkCount === 0)
             return;
         for (const record of this.store.listRecords())
             this.ensureVectorStore(record.collectionName);
-        if ([...this.vectorStores.values()].some(store => store.needsRebuild?.())) {
-            await this.indexVectors();
+        if (options.rebuild || [...this.vectorStores.values()].some(store => store.needsRebuild?.())) {
+            await this.indexVectors({ rebuild: true });
             return;
         }
         const indexedChunks = Number(this.store.getMetadata('vector_indexed_chunks') ?? 0);
         const status = this.store.getMetadata('vector_index_status');
-        if (!force && indexedChunks === chunkCount && status === 'ready')
+        const changedRelativePaths = [...new Set(options.changedRelativePaths ?? [])];
+        if (changedRelativePaths.length === 0 && options.deletesApplied && status === 'ready') {
+            this.store.setMetadata('vector_indexed_chunks', String(chunkCount));
+            this.store.setMetadata('vector_index_status', 'ready');
+            this.store.setMetadata('vector_index_error', '');
+            this.store.setMetadata('last_vector_index_at', String(Date.now()));
+            return;
+        }
+        if (changedRelativePaths.length > 0 && status === 'ready') {
+            for (const collectionName of options.changedCollectionNames ?? [])
+                this.ensureVectorStore(collectionName);
+            await this.indexVectors({ relativePaths: changedRelativePaths, cleanupCollectionNames: options.changedCollectionNames });
             return;
-        await this.indexVectors();
+        }
+        if (indexedChunks === chunkCount && status === 'ready')
+            return;
+        await this.indexVectors({ rebuild: true });
     }
     hasUsableContent(text, metadata) {
         const coverage = String(metadata.contentCoverage ?? '');
@@ -881,6 +939,21 @@ ${resultsText}
         const dir = configDirs[classification.category] ?? DEFAULT_CATEGORY_DIRS[classification.category];
         return `${dir}/${path.basename(fileName)}`;
     }
+    validateUploadRelativePath(relativePath) {
+        const normalized = this.normalizeRelativePath(relativePath);
+        if (!normalized || normalized === '.')
+            throw new Error('上传文件路径无效');
+        if (normalized.length > 1000)
+            throw new Error('上传文件路径过长，请缩短文件夹层级或文件名');
+        if (normalized.includes('\0'))
+            throw new Error('上传文件路径包含非法字符');
+        const parts = normalized.split('/');
+        if (parts.some(part => !part || part === '..'))
+            throw new Error('上传文件路径无效');
+        if (parts.some(part => part.length > 255))
+            throw new Error('上传文件名过长，请缩短文件名后重试');
+        return normalized;
+    }
     resolveKbRelativePath(relativePath) {
         const normalized = this.normalizeRelativePath(relativePath);
         const targetPath = path.resolve(this.kbPath, normalized);

package/dist/index.d.ts

@@ -15,7 +15,7 @@ export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, Pro
 export { ProjectRegistry } from './core/project-registry.js';
 export { HNSWVectorStore } from './vector/hnsw-vector-store.js';
 export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
-export type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './vector/types.js';
+export type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface, VectorWriteOptions } from './vector/types.js';
 export { VectorIndexer, type VectorIndexResult } from './vector/vector-indexer.js';
 export { FederationSearch, type FederatedQuery, type FederatedResult, type FederatedSearchItem, type SearchFilters, type SearchScope } from './search/federation-search.js';
 export type { LLMChatMessage, LLMChatOptions, LLMChatResponse, LLMSearchProvider } from './llm/llm-search-provider.js';

package/dist/vector/hnsw-vector-store.d.ts

@@ -1,4 +1,4 @@
-import type { VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './types.js';
+import type { VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface, VectorWriteOptions } from './types.js';
 /** HNSW（分层可导航小世界图）向量存储，基于 hnswlib-node 实现的高效近似最近邻搜索 */
 export declare class HNSWVectorStore implements VectorStoreInterface {
     readonly collectionName: string;
@@ -7,15 +7,18 @@ export declare class HNSWVectorStore implements VectorStoreInterface {
     private readonly maxElements;
     private index?;
     private deletedSinceRebuild;
+    private dirty;
     private readonly documents;
     constructor(collectionName: string, indexPath: string, dimensions?: number, maxElements?: number);
     ensureCollection(): Promise<void>;
-    upsert(documents: VectorDocument[]): Promise<void>;
+    upsert(documents: VectorDocument[], options?: VectorWriteOptions): Promise<void>;
     clearCollection(): Promise<void>;
-    deleteByFilePath(filePath: string): Promise<void>;
+    deleteByFilePath(filePath: string, options?: VectorWriteOptions): Promise<void>;
+    flush(): Promise<void>;
     needsRebuild(): boolean;
     search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
     private persist;
+    private toStoredDocument;
     private loadDocuments;
     private metadataPath;
 }

package/dist/vector/hnsw-vector-store.js

@@ -10,6 +10,7 @@ export class HNSWVectorStore {
     maxElements;
     index;
     deletedSinceRebuild = 0;
+    dirty = false;
     documents = new Map();
     constructor(collectionName, indexPath, dimensions = 512, maxElements = 500_000) {
         this.collectionName = collectionName;
@@ -29,16 +30,18 @@ export class HNSWVectorStore {
         else
             this.index.initIndex(this.maxElements, 16, 200, 100, true);
     }
-    async upsert(documents) {
+    async upsert(documents, options = {}) {
         await this.ensureCollection();
         for (const document of documents) {
             const rowid = Number(document.metadata.sqlite_rowid);
             if (!Number.isFinite(rowid) || rowid <= 0)
                 throw new Error(`HNSW 向量写入缺少有效 sqlite_rowid: ${document.id}`);
             this.index.addPoint(document.embedding, rowid, true);
-            this.documents.set(rowid, document);
+            this.documents.set(rowid, this.toStoredDocument(document));
+            this.dirty = true;
         }
-        this.persist();
+        if (options.persist !== false)
+            this.persist();
     }
     async clearCollection() {
         if (fs.existsSync(this.indexPath))
@@ -47,10 +50,11 @@ export class HNSWVectorStore {
             fs.rmSync(this.metadataPath(), { force: true });
         this.documents.clear();
         this.deletedSinceRebuild = 0;
+        this.dirty = false;
         this.index = undefined;
         await this.ensureCollection();
     }
-    async deleteByFilePath(filePath) {
+    async deleteByFilePath(filePath, options = {}) {
         await this.ensureCollection();
         for (const [rowid, document] of this.documents.entries()) {
             if (document.metadata.file_path === filePath) {
@@ -60,9 +64,16 @@ export class HNSWVectorStore {
                 }
                 catch { /* 忽略缺失的标签 */ }
                 this.documents.delete(rowid);
+                this.dirty = true;
             }
         }
-        this.persist();
+        if (options.persist !== false)
+            this.persist();
+    }
+    async flush() {
+        await this.ensureCollection();
+        if (this.dirty)
+            this.persist();
     }
     needsRebuild() {
         const total = this.documents.size + this.deletedSinceRebuild;
@@ -85,6 +96,14 @@ export class HNSWVectorStore {
     persist() {
         this.index.writeIndexSync(this.indexPath);
         fs.writeFileSync(this.metadataPath(), JSON.stringify({ deletedSinceRebuild: this.deletedSinceRebuild, documents: [...this.documents.entries()] }), 'utf8');
+        this.dirty = false;
+    }
+    toStoredDocument(document) {
+        return {
+            id: document.id,
+            content: '',
+            metadata: document.metadata,
+        };
     }
     loadDocuments() {
         const file = this.metadataPath();
@@ -96,7 +115,7 @@ export class HNSWVectorStore {
             this.deletedSinceRebuild = Array.isArray(parsed) ? 0 : Number(parsed.deletedSinceRebuild ?? 0);
             this.documents.clear();
             for (const [rowid, document] of entries)
-                this.documents.set(Number(rowid), document);
+                this.documents.set(Number(rowid), { id: document.id, content: document.content ?? '', metadata: document.metadata });
         }
         catch {
             this.documents.clear();

package/dist/vector/types.d.ts

@@ -24,11 +24,15 @@ export interface VectorCollectionInfo {
     metadata?: Record<string, unknown>;
 }
 /** 向量存储接口，定义所有向量存储实现必须支持的方法 */
+export interface VectorWriteOptions {
+    persist?: boolean;
+}
 export interface VectorStoreInterface {
     readonly collectionName: string;
     ensureCollection(metadata?: Record<string, unknown>): Promise<void>;
-    upsert(documents: VectorDocument[]): Promise<void>;
-    deleteByFilePath(filePath: string): Promise<void>;
+    upsert(documents: VectorDocument[], options?: VectorWriteOptions): Promise<void>;
+    deleteByFilePath(filePath: string, options?: VectorWriteOptions): Promise<void>;
+    flush?(): Promise<void>;
     clearCollection?(): Promise<void>;
     needsRebuild?(): boolean;
     search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;

package/dist/vector/vector-indexer.d.ts

@@ -16,6 +16,7 @@ export interface VectorIndexProgress {
 }
 export interface VectorIndexOptions {
     batchSize?: number;
+    persistEachBatch?: boolean;
     onProgress?: (progress: VectorIndexProgress) => void;
 }
 /** 向量索引器，负责将文本切片生成 Embedding 并写入向量存储 */

package/dist/vector/vector-indexer.js

@@ -32,11 +32,12 @@ export class VectorIndexer {
                 const texts = batchChunks.map(chunk => chunk.content);
                 const embeddings = await this.embedDocuments(texts);
                 const documents = batchChunks.map((chunk, index) => this.toVectorDocument(chunk, embeddings[index] ?? []));
-                await store.upsert(documents);
+                await store.upsert(documents, { persist: options.persistEachBatch === true });
                 processedChunks += documents.length;
                 processedTotalChunks += documents.length;
                 options.onProgress?.({ collectionName, processedChunks: processedTotalChunks, totalChunks, batchSize: documents.length });
             }
+            await store.flush?.();
             results.push({
                 collectionName,
                 chunkCount: processedChunks,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.5",
+  "version": "4.0.7",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",