@customize-agent/knowledge 4.0.7 → 4.0.9

package/dist/chunking/text-chunker.js

@@ -324,7 +324,7 @@ export class TextChunker {
     withHeader(text, file, config) {
         if (!config.headerInjection)
             return text;
-        return `文件: ${file.relativePath}\n类型: ${file.category}/${file.format}\n\n${text}`;
+        return `资料类型: ${file.category}/${file.format}\n\n${text}`;
     }
     createChunk(index, candidate, file, metadata) {
         const text = candidate.text.trim();

package/dist/core/change-tracker.d.ts

@@ -19,6 +19,6 @@ export declare class ChangeTracker {
      * @param filePath 文件路径
      * @returns SHA-256 哈希字符串
      */
-    hashFile(filePath: string): string;
+    hashFile(filePath: string): Promise<string>;
     private parseMetadata;
 }

package/dist/core/change-tracker.js

@@ -1,5 +1,6 @@
 import * as crypto from 'node:crypto';
 import * as fs from 'node:fs';
+import { createReadStream } from 'node:fs';
 import * as path from 'node:path';
 /** 文件变更追踪器，用于比对磁盘文件与索引状态之间的差异 */
 export class ChangeTracker {
@@ -51,7 +52,7 @@ export class ChangeTracker {
                 continue;
             }
             if (Math.round(diskStat.mtime) !== Math.round(indexed.mtime) || diskStat.size !== indexed.fileSize) {
-                const contentHash = this.hashFile(absolutePath);
+                const contentHash = await this.hashFile(absolutePath);
                 if (contentHash !== indexed.contentHash) {
                     modifiedFiles.push(classified);
                 }
@@ -86,8 +87,15 @@ export class ChangeTracker {
      * @param filePath 文件路径
      * @returns SHA-256 哈希字符串
      */
-    hashFile(filePath) {
-        return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
+    async hashFile(filePath) {
+        const hash = crypto.createHash('sha256');
+        await new Promise((resolve, reject) => {
+            const stream = createReadStream(filePath);
+            stream.on('data', chunk => hash.update(chunk));
+            stream.on('error', reject);
+            stream.on('end', resolve);
+        });
+        return hash.digest('hex');
     }
     parseMetadata(metadataJson) {
         if (!metadataJson)

package/dist/core/index-state-store.js

@@ -278,7 +278,7 @@ export class IndexStateStore {
       `).all(matchQuery, limit * 8);
             return rows
                 .map(row => {
-                const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
+                const keyword = this.scoreChunkDetailed(`${String(row.section_title ?? '')}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
                 const bm25Score = this.bm25ToPositiveScore(Number(row.bm25_score));
                 return this.rowToChunk(row, keyword.keywordScore + bm25Score, { ...keyword, bm25Score });
             })
@@ -299,7 +299,7 @@ export class IndexStateStore {
     `).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), limit * 6);
         return rows
             .map(row => {
-            const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
+            const keyword = this.scoreChunkDetailed(`${String(row.section_title ?? '')}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
             return this.rowToChunk(row, keyword.keywordScore, keyword);
         })
             .filter(row => row.score > 0)

package/dist/core/knowledge-base-manager.d.ts

@@ -59,6 +59,7 @@ export declare class KnowledgeBaseManager {
         onProgress?: (progress: KnowledgeIndexProgress) => void;
         vectorMode?: 'sync' | 'defer';
         limit?: number;
+        waitForUploadId?: string;
     }): Promise<DiffResult>;
     incrementalIndex(options?: {
         onProgress?: (progress: KnowledgeIndexProgress) => void;
@@ -112,7 +113,7 @@ export declare class KnowledgeBaseManager {
         fileName: string;
         sourcePath: string;
         targetRelativePath?: string;
-    }>, operationId?: string, offset?: number): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
+    }>, operationId?: string, offset?: number, uploadComplete?: boolean): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
     uploadFiles(files: Array<{
         fileName: string;
         content: Buffer;
@@ -176,6 +177,10 @@ export declare class KnowledgeBaseManager {
     private ensureVectorStore;
     private deleteVectorFile;
     private ensureVectorIndexFresh;
+    uploadSessionIsOpen(operationId: string): boolean;
+    private emptyDiff;
+    private statRelativePaths;
+    private moveUploadedFile;
     private hasUsableContent;
     private defaultUploadRelativePath;
     private validateUploadRelativePath;

package/dist/core/knowledge-base-manager.js

@@ -87,9 +87,12 @@ export class KnowledgeBaseManager {
     }
     async consumePendingIndexJobs(options = {}) {
         this.initialize();
-        const jobs = this.store.listPendingIndexJobs(options.limit ?? 50);
-        if (jobs.length === 0)
+        const jobs = this.store.listPendingIndexJobs(options.limit ?? 500);
+        if (jobs.length === 0) {
+            if (options.waitForUploadId && this.uploadSessionIsOpen(options.waitForUploadId))
+                return this.emptyDiff();
             return this.incrementalIndex(options);
+        }
         return this.incrementalIndex({ ...options, onlyRelativePaths: jobs.map(job => job.relativePath) });
     }
     async incrementalIndex(options = {}) {
@@ -101,10 +104,10 @@ export class KnowledgeBaseManager {
             this.reportProgress({ stage: 'scanning', percent: 10, message: '正在扫描知识库文件' });
             const kbIgnore = this.scanner.loadKbIgnore(this.kbPath);
             const configIgnore = this.projectConfig?.kbignore ?? [];
-            const diskFiles = await this.scanner.scan(this.kbPath, [...kbIgnore, ...configIgnore]);
+            const onlyRelativePaths = options.onlyRelativePaths ? new Set(options.onlyRelativePaths) : undefined;
+            const diskFiles = onlyRelativePaths ? this.statRelativePaths([...onlyRelativePaths]) : await this.scanner.scan(this.kbPath, [...kbIgnore, ...configIgnore]);
             const tracker = new ChangeTracker(this.store);
             const diff = await tracker.computeDiff(diskFiles, this.classifier, this.kbPath);
-            const onlyRelativePaths = options.onlyRelativePaths ? new Set(options.onlyRelativePaths) : undefined;
             let vectorDeletesApplied = 0;
             if (onlyRelativePaths) {
                 diff.newFiles = diff.newFiles.filter(file => onlyRelativePaths.has(file.relativePath));
@@ -129,7 +132,7 @@ export class KnowledgeBaseManager {
             const vectorRelativePaths = [];
             const changedCollectionNames = new Set();
             for (const [index, file] of filesToIndex.entries()) {
-                const hash = tracker.hashFile(file.absolutePath);
+                const hash = await tracker.hashFile(file.absolutePath);
                 const duplicate = this.store.findExactDuplicate(hash, file.relativePath);
                 const collectionName = this.scope === 'global'
                     ? this.collections.getCollectionName('global', file.category)
@@ -448,15 +451,16 @@ export class KnowledgeBaseManager {
         }
         return jobs;
     }
-    async stageUploadedFilePaths(files, operationId = `upload-${Date.now()}`, offset = 0) {
+    async stageUploadedFilePaths(files, operationId = `upload-${Date.now()}`, offset = 0, uploadComplete = true) {
         this.initialize();
+        this.store.setMetadata(`upload_session:${operationId}`, uploadComplete ? 'complete' : 'open');
         const jobs = [];
         for (let index = 0; index < files.length; index++) {
             const file = files[index];
             const relativePath = this.getUploadRelativePath(file.fileName, file.targetRelativePath);
             const targetPath = this.resolveKbRelativePath(relativePath);
             fs.mkdirSync(path.dirname(targetPath), { recursive: true });
-            fs.copyFileSync(file.sourcePath, targetPath);
+            this.moveUploadedFile(file.sourcePath, targetPath);
             const record = this.store.listRecords().find(item => item.relativePath === relativePath);
             if (record)
                 await this.deleteVectorFile(record.collectionName, relativePath);
@@ -927,6 +931,39 @@ ${resultsText}
             return;
         await this.indexVectors({ rebuild: true });
     }
+    uploadSessionIsOpen(operationId) {
+        return this.store.getMetadata(`upload_session:${operationId}`) === 'open';
+    }
+    emptyDiff() {
+        return { newFiles: [], modifiedFiles: [], deletedFiles: [], unchangedCount: 0, mtimeOnlyCount: 0, skippedFiles: [], hasChanges: false, diffTimeMs: 0 };
+    }
+    statRelativePaths(relativePaths) {
+        const files = new Map();
+        for (const relativePath of relativePaths) {
+            const normalized = this.normalizeRelativePath(relativePath);
+            const absolutePath = this.resolveKbRelativePath(normalized);
+            if (!fs.existsSync(absolutePath))
+                continue;
+            const stat = fs.statSync(absolutePath);
+            if (stat.isFile())
+                files.set(normalized, { size: stat.size, mtime: stat.mtimeMs });
+        }
+        return files;
+    }
+    moveUploadedFile(sourcePath, targetPath) {
+        if (fs.existsSync(targetPath))
+            fs.rmSync(targetPath, { force: true });
+        try {
+            fs.renameSync(sourcePath, targetPath);
+        }
+        catch (error) {
+            const code = error && typeof error === 'object' && 'code' in error ? String(error.code) : '';
+            if (code !== 'EXDEV')
+                throw error;
+            fs.copyFileSync(sourcePath, targetPath);
+            fs.rmSync(sourcePath, { force: true });
+        }
+    }
     hasUsableContent(text, metadata) {
         const coverage = String(metadata.contentCoverage ?? '');
         if (['metadata', 'metadata_filename', 'pdf_metadata_only', 'office_zip_empty_text', 'office_zip_failed'].includes(coverage))
@@ -964,6 +1001,6 @@ ${resultsText}
         return targetPath;
     }
     normalizeRelativePath(relativePath) {
-        return relativePath.replace(/\\/gu, '/').split(path.sep).join('/').replace(/^\/+/, '');
+        return relativePath.replace(/\\/gu, '/').split(path.sep).join('/').replace(/^\/+/, '').replace(/\/+/gu, '/');
     }
 }

package/dist/extraction/content-extractor.d.ts

@@ -16,6 +16,9 @@ export declare class ContentExtractor {
     private extractTextClipping;
     private swapUtf16Bytes;
     private extractReadableFragments;
+    private cleanCadReadableText;
+    private isReadableCadValue;
+    private cleanExtractedText;
     private textScore;
     private extractCad;
     private extractDxf;

package/dist/extraction/content-extractor.js

@@ -5,6 +5,8 @@ import { tmpdir } from 'node:os';
 import { pathToFileURL } from 'node:url';
 import { ExternalExtractorRegistry } from './external-extractor.js';
 import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
+const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb\w+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*)\b/giu;
+const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous)$/iu;
 /** 文件内容提取器，支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
 export class ContentExtractor {
     externalExtractors;
@@ -102,7 +104,7 @@ export class ContentExtractor {
             }
         }
         return {
-            text: text.trim(),
+            text: this.cleanExtractedText(text, file).trim(),
             metadata,
             warnings,
             extractionTimeMs: Date.now() - start,
@@ -177,9 +179,36 @@ export class ContentExtractor {
         return value
             .replace(/[^\p{L}\p{N}\p{P}\p{S}\s]/gu, '\n')
             .split(/[\r\n]+/u)
-            .map(line => line.replace(/\s+/gu, ' ').trim())
+            .map(line => this.cleanCadReadableText(line))
             .filter(line => line.length >= 2 && /[\p{L}\p{N}]/u.test(line));
     }
+    cleanCadReadableText(value) {
+        return value
+            .replace(CAD_INTERNAL_TOKEN_RE, '')
+            .replace(/\b(?:LINE|LWPOLYLINE|POLYLINE|INSERT|HATCH|CIRCLE|ARC|DIMENSION|TEXT|MTEXT)\b/giu, '')
+            .replace(/\s+/gu, ' ')
+            .trim();
+    }
+    isReadableCadValue(value) {
+        const cleaned = this.cleanCadReadableText(value);
+        return cleaned.length >= 2 && !CAD_INTERNAL_LINE_RE.test(cleaned) && /[\p{Script=Han}\p{Letter}\d]/u.test(cleaned);
+    }
+    cleanExtractedText(value, file) {
+        const normalized = [...value]
+            .filter(char => {
+            const code = char.charCodeAt(0);
+            return code === 9 || code === 10 || code === 13 || code >= 32;
+        })
+            .join('');
+        if (file.category !== 'cad')
+            return normalized.replace(/\n{3,}/gu, '\n\n');
+        return normalized
+            .split(/\r?\n/u)
+            .map(line => this.cleanCadReadableText(line))
+            .filter(line => line && !CAD_INTERNAL_LINE_RE.test(line))
+            .join('\n')
+            .replace(/\n{3,}/gu, '\n\n');
+    }
     textScore(value) {
         const cjk = (value.match(/[\p{Script=Han}]/gu) ?? []).length;
         const alnum = (value.match(/[\p{L}\p{N}]/gu) ?? []).length;
@@ -202,9 +231,9 @@ export class ContentExtractor {
         }
         if (file.format === 'autocad' && ext === '.dxf') {
             const raw = fs.readFileSync(file.absolutePath, 'utf8');
-            const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
+            const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
             const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 500);
-            const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
+            const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
             const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
             const uniqueLayers = Array.from(new Set(layers));
             const uniqueBlocks = Array.from(new Set(blocks));
@@ -278,7 +307,7 @@ export class ContentExtractor {
             if (result.text.trim())
                 return result;
         }
-        const readable = this.extractBinaryReadableFragments(file.absolutePath).slice(0, 800);
+        const readable = this.extractBinaryReadableFragments(file.absolutePath).filter(value => this.isReadableCadValue(value)).slice(0, 5000);
         metadata.extractionMode = 'builtin_cad_readable_fragments';
         metadata.contentCoverage = readable.length > 0 ? 'cad_readable_text_fragments' : 'metadata';
         metadata.stringCount = readable.length;
@@ -303,10 +332,10 @@ export class ContentExtractor {
         catch {
             warnings.push('dxf-parser 解析失败，已使用 DXF 文本结构抽取回退');
         }
-        const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
-        const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 800);
-        const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
-        const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
+        const layers = this.matchAll(raw, /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+        const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 5000);
+        const blocks = this.matchAll(raw, /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+        const entityTypes = this.matchAll(raw, /(?:^|\r?\n)\s*0\s*\r?\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
         const uniqueLayers = Array.from(new Set(layers));
         const uniqueBlocks = Array.from(new Set(blocks));
         const uniqueEntityTypes = Array.from(new Set(entityTypes));
@@ -350,18 +379,18 @@ export class ContentExtractor {
         });
     }
     extractDxfTextAnnotations(raw) {
-        const entities = raw.split(/\n\s*0\s*\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
+        const entities = raw.split(/(?:^|\r?\n)\s*0\s*\r?\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
         return entities.flatMap(section => {
-            const text = /\n\s*(?:1|3)\s*\n([^\n]+)/u.exec(section)?.[1]?.trim();
-            if (!text)
+            const text = this.cleanCadReadableText(/(?:^|\r?\n)\s*(?:1|3)\s*\r?\n([^\r\n]+)/u.exec(section)?.[1] ?? '');
+            if (!text || !this.isReadableCadValue(text))
                 return [];
             return [{
                     text,
-                    layer: /\n\s*8\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
-                    block: /\n\s*2\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
+                    layer: /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim(),
+                    block: /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim(),
                     entityType: section.trim().split(/\s+/u)[0],
-                    x: Number(/\n\s*10\s*\n([^\n]+)/u.exec(section)?.[1]),
-                    y: Number(/\n\s*20\s*\n([^\n]+)/u.exec(section)?.[1]),
+                    x: Number(/(?:^|\r?\n)\s*10\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
+                    y: Number(/(?:^|\r?\n)\s*20\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
                 }].map(item => ({ ...item, x: Number.isFinite(item.x) ? item.x : undefined, y: Number.isFinite(item.y) ? item.y : undefined }));
         });
     }
@@ -1165,7 +1194,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
             if (pages.length > 0) {
                 const combined = pages.join('\n\n');
                 if (combined.trim())
-                    return combined.slice(0, 250_000);
+                    return combined;
             }
         }
         catch (e) {
@@ -1179,7 +1208,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
             if (pdfParse) {
                 const result = await pdfParse(buffer);
                 if (result.text.trim())
-                    return result.text.slice(0, 250_000);
+                    return result.text;
             }
         }
         catch {
@@ -1450,15 +1479,8 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
         return lines.filter(line => !line.endsWith(':'));
     }
     metadataOnlyText(file) {
-        const fileName = path.basename(file.relativePath);
-        const directory = path.dirname(file.relativePath);
-        const searchableName = fileName.replace(/[_\-.]+/gu, ' ');
         return [
-            `文件名: ${fileName}`,
-            `文件路径: ${file.relativePath}`,
-            `所在目录: ${directory === '.' ? 'knowledgeBase' : directory}`,
-            `可搜索名称: ${searchableName}`,
-            `文件类型: ${file.category}/${file.format}`,
+            `资料类型: ${file.category}/${file.format}`,
             `MIME: ${file.mimeType}`,
             `文件大小: ${file.fileSize} bytes`,
         ].join('\n');

package/dist/vector/vector-indexer.js

@@ -1,8 +1,8 @@
 function resolveVectorIndexBatchSize(configured) {
     const raw = configured ?? Number(process.env.CUSTOMIZE_VECTOR_INDEX_BATCH_SIZE ?? process.env.KB_VECTOR_INDEX_BATCH_SIZE);
     if (!Number.isFinite(raw) || raw <= 0)
-        return 128;
-    return Math.max(1, Math.min(512, Math.floor(raw)));
+        return 256;
+    return Math.max(1, Math.min(1024, Math.floor(raw)));
 }
 /** 向量索引器，负责将文本切片生成 Embedding 并写入向量存储 */
 export class VectorIndexer {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.7",
+  "version": "4.0.9",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",