@customize-agent/knowledge 4.0.6 → 4.0.8

package/dist/core/change-tracker.d.ts

@@ -19,6 +19,6 @@ export declare class ChangeTracker {
      * @param filePath 文件路径
      * @returns SHA-256 哈希字符串
      */
-    hashFile(filePath: string): string;
+    hashFile(filePath: string): Promise<string>;
     private parseMetadata;
 }

package/dist/core/change-tracker.js

@@ -1,5 +1,6 @@
 import * as crypto from 'node:crypto';
 import * as fs from 'node:fs';
+import { createReadStream } from 'node:fs';
 import * as path from 'node:path';
 /** 文件变更追踪器，用于比对磁盘文件与索引状态之间的差异 */
 export class ChangeTracker {
@@ -51,7 +52,7 @@ export class ChangeTracker {
                 continue;
             }
             if (Math.round(diskStat.mtime) !== Math.round(indexed.mtime) || diskStat.size !== indexed.fileSize) {
-                const contentHash = this.hashFile(absolutePath);
+                const contentHash = await this.hashFile(absolutePath);
                 if (contentHash !== indexed.contentHash) {
                     modifiedFiles.push(classified);
                 }
@@ -86,8 +87,15 @@ export class ChangeTracker {
      * @param filePath 文件路径
      * @returns SHA-256 哈希字符串
      */
-    hashFile(filePath) {
-        return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
+    async hashFile(filePath) {
+        const hash = crypto.createHash('sha256');
+        await new Promise((resolve, reject) => {
+            const stream = createReadStream(filePath);
+            stream.on('data', chunk => hash.update(chunk));
+            stream.on('error', reject);
+            stream.on('end', resolve);
+        });
+        return hash.digest('hex');
     }
     parseMetadata(metadataJson) {
         if (!metadataJson)

package/dist/core/knowledge-base-manager.d.ts

@@ -59,6 +59,7 @@ export declare class KnowledgeBaseManager {
         onProgress?: (progress: KnowledgeIndexProgress) => void;
         vectorMode?: 'sync' | 'defer';
         limit?: number;
+        waitForUploadId?: string;
     }): Promise<DiffResult>;
     incrementalIndex(options?: {
         onProgress?: (progress: KnowledgeIndexProgress) => void;
@@ -108,6 +109,11 @@ export declare class KnowledgeBaseManager {
         content: Buffer;
         targetRelativePath?: string;
     }>, operationId?: string): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
+    stageUploadedFilePaths(files: Array<{
+        fileName: string;
+        sourcePath: string;
+        targetRelativePath?: string;
+    }>, operationId?: string, offset?: number, uploadComplete?: boolean): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
     uploadFiles(files: Array<{
         fileName: string;
         content: Buffer;
@@ -171,8 +177,13 @@ export declare class KnowledgeBaseManager {
     private ensureVectorStore;
     private deleteVectorFile;
     private ensureVectorIndexFresh;
+    uploadSessionIsOpen(operationId: string): boolean;
+    private emptyDiff;
+    private statRelativePaths;
+    private moveUploadedFile;
     private hasUsableContent;
     private defaultUploadRelativePath;
+    private validateUploadRelativePath;
     private resolveKbRelativePath;
     private normalizeRelativePath;
 }

package/dist/core/knowledge-base-manager.js

@@ -87,9 +87,12 @@ export class KnowledgeBaseManager {
     }
     async consumePendingIndexJobs(options = {}) {
         this.initialize();
-        const jobs = this.store.listPendingIndexJobs(options.limit ?? 50);
-        if (jobs.length === 0)
+        const jobs = this.store.listPendingIndexJobs(options.limit ?? 500);
+        if (jobs.length === 0) {
+            if (options.waitForUploadId && this.uploadSessionIsOpen(options.waitForUploadId))
+                return this.emptyDiff();
             return this.incrementalIndex(options);
+        }
         return this.incrementalIndex({ ...options, onlyRelativePaths: jobs.map(job => job.relativePath) });
     }
     async incrementalIndex(options = {}) {
@@ -101,10 +104,10 @@ export class KnowledgeBaseManager {
             this.reportProgress({ stage: 'scanning', percent: 10, message: '正在扫描知识库文件' });
             const kbIgnore = this.scanner.loadKbIgnore(this.kbPath);
             const configIgnore = this.projectConfig?.kbignore ?? [];
-            const diskFiles = await this.scanner.scan(this.kbPath, [...kbIgnore, ...configIgnore]);
+            const onlyRelativePaths = options.onlyRelativePaths ? new Set(options.onlyRelativePaths) : undefined;
+            const diskFiles = onlyRelativePaths ? this.statRelativePaths([...onlyRelativePaths]) : await this.scanner.scan(this.kbPath, [...kbIgnore, ...configIgnore]);
             const tracker = new ChangeTracker(this.store);
             const diff = await tracker.computeDiff(diskFiles, this.classifier, this.kbPath);
-            const onlyRelativePaths = options.onlyRelativePaths ? new Set(options.onlyRelativePaths) : undefined;
             let vectorDeletesApplied = 0;
             if (onlyRelativePaths) {
                 diff.newFiles = diff.newFiles.filter(file => onlyRelativePaths.has(file.relativePath));
@@ -129,7 +132,7 @@ export class KnowledgeBaseManager {
             const vectorRelativePaths = [];
             const changedCollectionNames = new Set();
             for (const [index, file] of filesToIndex.entries()) {
-                const hash = tracker.hashFile(file.absolutePath);
+                const hash = await tracker.hashFile(file.absolutePath);
                 const duplicate = this.store.findExactDuplicate(hash, file.relativePath);
                 const collectionName = this.scope === 'global'
                     ? this.collections.getCollectionName('global', file.category)
@@ -426,7 +429,7 @@ export class KnowledgeBaseManager {
         return this.incrementalIndex();
     }
     getUploadRelativePath(fileName, targetRelativePath) {
-        return targetRelativePath ? this.normalizeRelativePath(targetRelativePath) : this.defaultUploadRelativePath(fileName);
+        return this.validateUploadRelativePath(targetRelativePath ? this.normalizeRelativePath(targetRelativePath) : this.defaultUploadRelativePath(fileName));
     }
     async uploadFile(fileName, content, targetRelativePath, onProgress, options = {}) {
         return this.uploadFiles([{ fileName, content, targetRelativePath }], onProgress, options);
@@ -448,6 +451,24 @@ export class KnowledgeBaseManager {
         }
         return jobs;
     }
+    async stageUploadedFilePaths(files, operationId = `upload-${Date.now()}`, offset = 0, uploadComplete = true) {
+        this.initialize();
+        this.store.setMetadata(`upload_session:${operationId}`, uploadComplete ? 'complete' : 'open');
+        const jobs = [];
+        for (let index = 0; index < files.length; index++) {
+            const file = files[index];
+            const relativePath = this.getUploadRelativePath(file.fileName, file.targetRelativePath);
+            const targetPath = this.resolveKbRelativePath(relativePath);
+            fs.mkdirSync(path.dirname(targetPath), { recursive: true });
+            this.moveUploadedFile(file.sourcePath, targetPath);
+            const record = this.store.listRecords().find(item => item.relativePath === relativePath);
+            if (record)
+                await this.deleteVectorFile(record.collectionName, relativePath);
+            this.store.deleteRecord(relativePath);
+            jobs.push(this.store.enqueueIndexJob({ id: `${operationId}-${offset + index}`, relativePath, message: '文件已落盘，等待后台解析' }));
+        }
+        return jobs;
+    }
     async uploadFiles(files, onProgress, options = {}) {
         await this.stageUploadedFiles(files);
         return this.incrementalIndex({ onProgress, vectorMode: options.vectorMode });
@@ -910,6 +931,39 @@ ${resultsText}
             return;
         await this.indexVectors({ rebuild: true });
     }
+    uploadSessionIsOpen(operationId) {
+        return this.store.getMetadata(`upload_session:${operationId}`) === 'open';
+    }
+    emptyDiff() {
+        return { newFiles: [], modifiedFiles: [], deletedFiles: [], unchangedCount: 0, mtimeOnlyCount: 0, skippedFiles: [], hasChanges: false, diffTimeMs: 0 };
+    }
+    statRelativePaths(relativePaths) {
+        const files = new Map();
+        for (const relativePath of relativePaths) {
+            const normalized = this.normalizeRelativePath(relativePath);
+            const absolutePath = this.resolveKbRelativePath(normalized);
+            if (!fs.existsSync(absolutePath))
+                continue;
+            const stat = fs.statSync(absolutePath);
+            if (stat.isFile())
+                files.set(normalized, { size: stat.size, mtime: stat.mtimeMs });
+        }
+        return files;
+    }
+    moveUploadedFile(sourcePath, targetPath) {
+        if (fs.existsSync(targetPath))
+            fs.rmSync(targetPath, { force: true });
+        try {
+            fs.renameSync(sourcePath, targetPath);
+        }
+        catch (error) {
+            const code = error && typeof error === 'object' && 'code' in error ? String(error.code) : '';
+            if (code !== 'EXDEV')
+                throw error;
+            fs.copyFileSync(sourcePath, targetPath);
+            fs.rmSync(sourcePath, { force: true });
+        }
+    }
     hasUsableContent(text, metadata) {
         const coverage = String(metadata.contentCoverage ?? '');
         if (['metadata', 'metadata_filename', 'pdf_metadata_only', 'office_zip_empty_text', 'office_zip_failed'].includes(coverage))
@@ -922,6 +976,21 @@ ${resultsText}
         const dir = configDirs[classification.category] ?? DEFAULT_CATEGORY_DIRS[classification.category];
         return `${dir}/${path.basename(fileName)}`;
     }
+    validateUploadRelativePath(relativePath) {
+        const normalized = this.normalizeRelativePath(relativePath);
+        if (!normalized || normalized === '.')
+            throw new Error('上传文件路径无效');
+        if (normalized.length > 1000)
+            throw new Error('上传文件路径过长，请缩短文件夹层级或文件名');
+        if (normalized.includes('\0'))
+            throw new Error('上传文件路径包含非法字符');
+        const parts = normalized.split('/');
+        if (parts.some(part => !part || part === '..'))
+            throw new Error('上传文件路径无效');
+        if (parts.some(part => part.length > 255))
+            throw new Error('上传文件名过长，请缩短文件名后重试');
+        return normalized;
+    }
     resolveKbRelativePath(relativePath) {
         const normalized = this.normalizeRelativePath(relativePath);
         const targetPath = path.resolve(this.kbPath, normalized);
@@ -932,6 +1001,6 @@ ${resultsText}
         return targetPath;
     }
     normalizeRelativePath(relativePath) {
-        return relativePath.replace(/\\/gu, '/').split(path.sep).join('/').replace(/^\/+/, '');
+        return relativePath.replace(/\\/gu, '/').split(path.sep).join('/').replace(/^\/+/, '').replace(/\/+/gu, '/');
     }
 }

package/dist/vector/vector-indexer.js

@@ -1,8 +1,8 @@
 function resolveVectorIndexBatchSize(configured) {
     const raw = configured ?? Number(process.env.CUSTOMIZE_VECTOR_INDEX_BATCH_SIZE ?? process.env.KB_VECTOR_INDEX_BATCH_SIZE);
     if (!Number.isFinite(raw) || raw <= 0)
-        return 128;
-    return Math.max(1, Math.min(512, Math.floor(raw)));
+        return 256;
+    return Math.max(1, Math.min(1024, Math.floor(raw)));
 }
 /** 向量索引器，负责将文本切片生成 Embedding 并写入向量存储 */
 export class VectorIndexer {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.6",
+  "version": "4.0.8",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",