@customize-agent/knowledge 4.3.5 → 4.3.6

package/dist/cleaning/text-cleaner.js

@@ -396,14 +396,15 @@ export function cleanExtractedText(input) {
             index += run;
             continue;
         }
-        // 3. 页眉/页脚/图框标题栏高重复行
-        if (isHeaderFooterLine(trimmed, lineCounts.get(trimmed) ?? 1)) {
+        // 3. 页眉/页脚/图框标题栏高重复行（CAD 豁免：图纸标注重复出现是数据本身——
+        // 门窗编号「FM1524」、材料规格等会随楼层重复标注，非页眉页脚；CAD 图框噪声已由 K3 规则覆盖）
+        if (!isCad && isHeaderFooterLine(trimmed, lineCounts.get(trimmed) ?? 1)) {
             drop(line, s => { s.headerFooterLines += 1; });
             index += 1;
             continue;
         }
-        // 4. 纯页码行
-        if (isPageNumberLine(trimmed, totalLines)) {
+        // 4. 纯页码行（CAD 豁免：图纸纯数字行是尺寸/标高/门窗表数值，非页码）
+        if (!isCad && isPageNumberLine(trimmed, totalLines)) {
             drop(line, s => { s.pageNumberLines += 1; });
             index += 1;
             continue;

package/dist/core/index-state-store.d.ts

@@ -103,6 +103,8 @@ export declare class IndexStateStore {
     /** 更新已验证文件的时间戳和状态 */
     updateVerified(relativePath: string, mtime: number): void;
     listRecords(): IndexStateRecord[];
+    /** 仅取去重后的集合名：走 idx_kb_state_collection 索引，比 listRecords 全行扫描+对象映射廉价得多 */
+    listCollectionNames(): string[];
     enqueueIndexJob(job: {
         id: string;
         relativePath: string;

package/dist/core/index-state-store.js

@@ -63,6 +63,14 @@ export class IndexStateStore {
     `).all();
         return rows.map(row => this.rowToRecord(row));
     }
+    /** 仅取去重后的集合名：走 idx_kb_state_collection 索引，比 listRecords 全行扫描+对象映射廉价得多 */
+    listCollectionNames() {
+        const rows = this.db.prepare(`
+      SELECT DISTINCT collection_name FROM kb_index_state
+      WHERE status != 'deleted'
+    `).all();
+        return rows.map(row => String(row.collection_name));
+    }
     enqueueIndexJob(job) {
         const now = Date.now();
         this.db.prepare(`

package/dist/core/knowledge-base-manager.d.ts

@@ -51,6 +51,9 @@ export declare class KnowledgeBaseManager {
     private queryExpansionActive;
     private readonly queryExpansionWaiters;
     private onProgress?;
+    /** 集合名缓存：避免每次向量搜索都全表扫描 kb_index_state；TTL 兜底 worker 子进程跨进程写入的新集合 */
+    private collectionNamesCache;
+    private static readonly COLLECTION_NAMES_TTL_MS;
     constructor(options: KnowledgeBaseManagerOptions);
     initialize(): void;
     forceReindexAll(options?: {
@@ -118,9 +121,16 @@ export declare class KnowledgeBaseManager {
     }): Promise<DiffResult>;
     addFile(sourcePath: string, targetRelativePath?: string): Promise<DiffResult>;
     getUploadRelativePath(fileName: string, targetRelativePath?: string): string;
+    /**
+     * @deprecated 生产上传链路已切换为 stageUploadedFilePaths + 后台索引 worker（不落内存 buffer、支持上传会话等待）。
+     * 本同步路径仅保留给测试与 CLI 使用；新代码请走 stageUploadedFilePaths。
+     */
     uploadFile(fileName: string, content: Buffer, targetRelativePath?: string, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
         vectorMode?: 'sync' | 'defer';
     }): Promise<DiffResult>;
+    /**
+     * @deprecated 生产上传链路已切换为 stageUploadedFilePaths（路径落盘，不占内存）。仅测试/CLI 使用。
+     */
     stageUploadedFiles(files: Array<{
         fileName: string;
         content: Buffer;
@@ -131,6 +141,9 @@ export declare class KnowledgeBaseManager {
         sourcePath: string;
         targetRelativePath?: string;
     }>, operationId?: string, offset?: number, uploadComplete?: boolean): Promise<import("./index-state-store.js").KnowledgeIndexJob[]>;
+    /**
+     * @deprecated 同步上传+索引一体化旧路径，仅测试/CLI 使用；生产请用 stageUploadedFilePaths + 后台 worker。
+     */
     uploadFiles(files: Array<{
         fileName: string;
         content: Buffer;
@@ -199,7 +212,16 @@ export declare class KnowledgeBaseManager {
     private reportProgress;
     private updateJobsForFile;
     private ensureVectorStore;
+    /** 本进程写入新记录时即时补充集合名缓存（不等 TTL），保证紧随其后的搜索立即可见 */
+    private noteCollectionName;
+    /** 为全部已知集合确保向量存储已加载；集合名集合带 2s TTL 缓存，避免每次搜索全表扫描 */
+    private ensureAllVectorStores;
     private deleteVectorFile;
+    /** 登记删除失败的孤儿向量（去重、限量 500 条防止 metadata 膨胀） */
+    private noteOrphanVector;
+    private listOrphanVectors;
+    /** 清扫孤儿向量：重试删除此前失败的条目，成功的出队，仍失败的留待下次（best-effort，不阻断索引主流程） */
+    private sweepOrphanVectors;
     private consumePendingVectorRelativePaths;
     private ensureVectorIndexFresh;
     uploadSessionIsOpen(operationId: string): boolean;

package/dist/core/knowledge-base-manager.js

@@ -48,6 +48,9 @@ export class KnowledgeBaseManager {
     queryExpansionActive = 0;
     queryExpansionWaiters = [];
     onProgress;
+    /** 集合名缓存：避免每次向量搜索都全表扫描 kb_index_state；TTL 兜底 worker 子进程跨进程写入的新集合 */
+    collectionNamesCache = null;
+    static COLLECTION_NAMES_TTL_MS = 2000;
     constructor(options) {
         this.scope = options.scope;
         this.projectRoot = options.projectRoot;
@@ -196,6 +199,7 @@ export class KnowledgeBaseManager {
                         errorMessage: metadataOnly ? undefined : reason,
                         metadataJson: JSON.stringify({ mimeType: file.mimeType, ...extraction.metadata, warnings: extraction.warnings, metadataOnly }),
                     });
+                    this.noteCollectionName(collectionName);
                     continue;
                 }
                 // K1 入库前清洗（源头治理）：解析完成后、分块入库前移除确定性噪声（页眉页脚/目录/页码/
@@ -310,6 +314,7 @@ export class KnowledgeBaseManager {
                     format: file.format,
                     collectionName,
                 });
+                this.noteCollectionName(collectionName);
                 vectorRelativePaths.push(file.relativePath);
                 changedCollectionNames.add(collectionName);
                 this.updateJobsForFile(file.relativePath, options.vectorMode === 'defer' ? 'SUCCESS' : 'INDEXING', options.vectorMode === 'defer' ? 100 : 85, options.vectorMode === 'defer' ? '解析和切片已完成' : '等待向量入库');
@@ -490,8 +495,7 @@ export class KnowledgeBaseManager {
         };
     }
     async semanticSearch(query, options = {}) {
-        for (const record of this.store.listRecords())
-            this.ensureVectorStore(record.collectionName);
+        this.ensureAllVectorStores();
         const queryEmbedding = await this.embeddingProvider.embedQuery(query);
         const search = new FederationSearch(this.vectorStores);
         try {
@@ -565,9 +569,16 @@ export class KnowledgeBaseManager {
     getUploadRelativePath(fileName, targetRelativePath) {
         return this.validateUploadRelativePath(targetRelativePath ? this.normalizeRelativePath(targetRelativePath) : this.defaultUploadRelativePath(fileName));
     }
+    /**
+     * @deprecated 生产上传链路已切换为 stageUploadedFilePaths + 后台索引 worker（不落内存 buffer、支持上传会话等待）。
+     * 本同步路径仅保留给测试与 CLI 使用；新代码请走 stageUploadedFilePaths。
+     */
     async uploadFile(fileName, content, targetRelativePath, onProgress, options = {}) {
         return this.uploadFiles([{ fileName, content, targetRelativePath }], onProgress, options);
     }
+    /**
+     * @deprecated 生产上传链路已切换为 stageUploadedFilePaths（路径落盘，不占内存）。仅测试/CLI 使用。
+     */
     async stageUploadedFiles(files, operationId = `upload-${Date.now()}`) {
         this.initialize();
         const jobs = [];
@@ -595,6 +606,9 @@ export class KnowledgeBaseManager {
         }
         return jobs;
     }
+    /**
+     * @deprecated 同步上传+索引一体化旧路径，仅测试/CLI 使用；生产请用 stageUploadedFilePaths + 后台 worker。
+     */
     async uploadFiles(files, onProgress, options = {}) {
         const jobs = await this.stageUploadedFiles(files);
         return this.incrementalIndex({ onProgress, vectorMode: options.vectorMode, onlyRelativePaths: jobs.map(job => job.relativePath) });
@@ -638,6 +652,8 @@ export class KnowledgeBaseManager {
         return this.store.listIgnoreRules();
     }
     async indexVectors(options = {}) {
+        // 先清扫此前删除失败的孤儿向量（重试幂等，失败留队下次再试）
+        await this.sweepOrphanVectors();
         const pendingRelativePaths = !options.rebuild && !options.relativePath && !options.relativePaths?.length
             ? this.consumePendingVectorRelativePaths()
             : [];
@@ -1144,6 +1160,19 @@ export class KnowledgeBaseManager {
         const safeName = collectionName.replace(/[^a-zA-Z0-9_.-]/gu, '_');
         this.vectorStores.set(collectionName, new HNSWVectorStore(collectionName, path.join(this.vectorRoot, `${safeName}.hnsw`), this.embeddingProvider.dimensions));
     }
+    /** 本进程写入新记录时即时补充集合名缓存（不等 TTL），保证紧随其后的搜索立即可见 */
+    noteCollectionName(collectionName) {
+        this.collectionNamesCache?.names.add(collectionName);
+    }
+    /** 为全部已知集合确保向量存储已加载；集合名集合带 2s TTL 缓存，避免每次搜索全表扫描 */
+    ensureAllVectorStores() {
+        const now = Date.now();
+        if (!this.collectionNamesCache || now - this.collectionNamesCache.fetchedAt > KnowledgeBaseManager.COLLECTION_NAMES_TTL_MS) {
+            this.collectionNamesCache = { names: new Set(this.store.listCollectionNames()), fetchedAt: now };
+        }
+        for (const name of this.collectionNamesCache.names)
+            this.ensureVectorStore(name);
+    }
     async deleteVectorFile(collectionName, relativePath) {
         this.ensureVectorStore(collectionName);
         try {
@@ -1152,7 +1181,50 @@ export class KnowledgeBaseManager {
         catch (error) {
             this.store.setMetadata('vector_index_status', 'error');
             this.store.setMetadata('vector_index_error', error instanceof Error ? error.message : String(error));
+            // 删除失败会留下孤儿向量（已删文件的内容仍可被召回），登记待清扫队列由 indexVectors 重试
+            this.noteOrphanVector(collectionName, relativePath);
+        }
+    }
+    /** 登记删除失败的孤儿向量（去重、限量 500 条防止 metadata 膨胀） */
+    noteOrphanVector(collectionName, relativePath) {
+        const pending = this.listOrphanVectors();
+        if (pending.some(item => item.collectionName === collectionName && item.relativePath === relativePath))
+            return;
+        pending.push({ collectionName, relativePath });
+        this.store.setMetadata('vector_orphan_pending', JSON.stringify(pending.slice(-500)));
+    }
+    listOrphanVectors() {
+        const raw = this.store.getMetadata('vector_orphan_pending');
+        if (!raw)
+            return [];
+        try {
+            const parsed = JSON.parse(raw);
+            if (!Array.isArray(parsed))
+                return [];
+            return parsed.filter((item) => typeof item === 'object' && item !== null
+                && typeof item.collectionName === 'string'
+                && typeof item.relativePath === 'string');
+        }
+        catch {
+            return [];
+        }
+    }
+    /** 清扫孤儿向量：重试删除此前失败的条目，成功的出队，仍失败的留待下次（best-effort，不阻断索引主流程） */
+    async sweepOrphanVectors() {
+        const pending = this.listOrphanVectors();
+        if (pending.length === 0)
+            return;
+        const remaining = [];
+        for (const item of pending) {
+            this.ensureVectorStore(item.collectionName);
+            try {
+                await this.vectorStores.get(item.collectionName)?.deleteByFilePath(item.relativePath);
+            }
+            catch {
+                remaining.push(item);
+            }
         }
+        this.store.setMetadata('vector_orphan_pending', remaining.length > 0 ? JSON.stringify(remaining) : '');
     }
     consumePendingVectorRelativePaths() {
         const raw = this.store.getMetadata('vector_pending_relative_paths');
@@ -1173,8 +1245,7 @@ export class KnowledgeBaseManager {
     async ensureVectorIndexFresh(chunkCount, options = {}) {
         if (chunkCount === 0)
             return;
-        for (const record of this.store.listRecords())
-            this.ensureVectorStore(record.collectionName);
+        this.ensureAllVectorStores();
         if (options.rebuild || [...this.vectorStores.values()].some(store => store.needsRebuild?.())) {
             await this.indexVectors({ rebuild: true });
             return;

package/dist/embedding/local-reranker.d.ts

@@ -3,7 +3,21 @@ export declare class LocalReranker {
     private static loadingPromise;
     private static disabledUntil;
     private static modelName;
+    /** rerank 推理 worker 线程（懒加载常驻）；unavailable 后不再重建，本进程内回退主线程推理 */
+    private static worker;
+    private static workerUnavailable;
+    private static workerNextId;
+    private static workerPending;
+    /** 单批（≤30 候选）推理超时：超时按失败处理，由调用方 catch 回落启发式重排 */
+    private static readonly WORKER_TIMEOUT_MS;
     static getInstance(): Promise<any>;
+    /**
+     * worker 线程路径开关：默认开启（cross-encoder ONNX 推理在 worker 线程执行，主线程零阻塞）；
+     * KB_RERANKER_WORKER=0/false 显式回退主线程推理（排查 worker 问题时的对照通道）
+     */
+    private static workerEnabled;
+    private static getWorker;
+    private static rerankInWorker;
     /**
      * 对多条文本和查询进行相关性重排
      */

package/dist/embedding/local-reranker.js

@@ -2,11 +2,20 @@ import { pipeline } from '@huggingface/transformers';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { Worker } from 'node:worker_threads';
 export class LocalReranker {
     static instance = null;
     static loadingPromise = null;
     static disabledUntil = 0;
     static modelName = process.env.KB_RERANKER_MODEL || 'Xenova/bge-reranker-base';
+    /** rerank 推理 worker 线程（懒加载常驻）；unavailable 后不再重建，本进程内回退主线程推理 */
+    static worker = null;
+    static workerUnavailable = false;
+    static workerNextId = 0;
+    static workerPending = new Map();
+    /** 单批（≤30 候选）推理超时：超时按失败处理，由调用方 catch 回落启发式重排 */
+    static WORKER_TIMEOUT_MS = 180_000;
     static async getInstance() {
         if (this.instance)
             return this.instance;
@@ -35,12 +44,90 @@ export class LocalReranker {
         });
         return this.loadingPromise;
     }
+    /**
+     * worker 线程路径开关：默认开启（cross-encoder ONNX 推理在 worker 线程执行，主线程零阻塞）；
+     * KB_RERANKER_WORKER=0/false 显式回退主线程推理（排查 worker 问题时的对照通道）
+     */
+    static workerEnabled() {
+        const raw = process.env.KB_RERANKER_WORKER;
+        return raw !== '0' && raw !== 'false';
+    }
+    static getWorker() {
+        if (this.worker)
+            return this.worker;
+        if (this.workerUnavailable)
+            return null;
+        try {
+            // dist 编译产物同目录下存在 rerank-worker-thread.js；源码直跑（无编译产物）时回落主线程推理
+            const workerPath = fileURLToPath(new URL('./rerank-worker-thread.js', import.meta.url));
+            if (!fs.existsSync(workerPath)) {
+                this.workerUnavailable = true;
+                return null;
+            }
+            const worker = new Worker(workerPath);
+            // unref：worker 不阻止主进程退出（常驻模型加载不应拖住进程生命周期）
+            worker.unref();
+            worker.on('message', (response) => {
+                const pending = this.workerPending.get(response.id);
+                if (!pending)
+                    return;
+                this.workerPending.delete(response.id);
+                clearTimeout(pending.timer);
+                if (response.error)
+                    pending.reject(new Error(response.error));
+                else
+                    pending.resolve(response.scores || []);
+            });
+            worker.on('error', (error) => {
+                // worker 崩溃：拒绝全部在途请求（调用方回退主线程推理），本进程内不再重建 worker
+                this.workerUnavailable = true;
+                this.worker = null;
+                for (const pending of this.workerPending.values()) {
+                    clearTimeout(pending.timer);
+                    pending.reject(error);
+                }
+                this.workerPending.clear();
+                worker.terminate().catch(() => { });
+            });
+            this.worker = worker;
+            return worker;
+        }
+        catch {
+            this.workerUnavailable = true;
+            return null;
+        }
+    }
+    static rerankInWorker(query, texts) {
+        const worker = this.getWorker();
+        if (!worker)
+            return Promise.reject(new Error('rerank worker unavailable'));
+        const id = this.workerNextId++;
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.workerPending.delete(id);
+                reject(new Error('rerank worker timeout'));
+            }, this.WORKER_TIMEOUT_MS);
+            timer.unref?.();
+            this.workerPending.set(id, { resolve, reject, timer });
+            worker.postMessage({ id, query, texts });
+        });
+    }
     /**
      * 对多条文本和查询进行相关性重排
      */
     static async rerank(query, texts) {
         if (!texts.length)
             return [];
+        // 优先走 worker 线程（默认）：ONNX 推理不阻塞主线程事件循环；
+        // worker 不可用/失败时回退主线程推理（与历史行为一致，KB_ENABLE_LOCAL_RERANKER=false 总开关仍生效）
+        if (this.workerEnabled() && process.env.KB_ENABLE_LOCAL_RERANKER !== 'false') {
+            try {
+                return await this.rerankInWorker(query, texts);
+            }
+            catch {
+                // worker 路径失败，回落主线程推理
+            }
+        }
         const ranker = await this.getInstance();
         const scores = [];
         const safeQuery = this.takeHeadTail(query, 240);

package/dist/embedding/rerank-worker-thread.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/embedding/rerank-worker-thread.js

@@ -0,0 +1,74 @@
+/**
+ * LocalReranker 的 worker 线程宿主：cross-encoder（bge-reranker-base）ONNX 推理是 CPU 密集计算，
+ * 在主线程执行会阻塞事件循环（生成场景数百组查询 × 30 候选的推理曾阻塞主线程 20-60 分钟，
+ * HTTP 无响应、前端误判卡死）。推理整体下沉到本 worker，主线程仅做消息收发；
+ * 请求串行处理（单次推理占满 worker 线程，并发只叠加内存无吞吐收益），模型懒加载一次常驻。
+ */
+import { parentPort } from 'node:worker_threads';
+import { pipeline } from '@huggingface/transformers';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+let rankerPromise = null;
+/** 模型加载失败熔断：60 秒内快速失败，避免每个请求都重试一次分钟级加载 */
+let loadDisabledUntil = 0;
+function loadRanker() {
+    if (Date.now() < loadDisabledUntil)
+        return Promise.reject(new Error('rerank worker model load is temporarily disabled after failure'));
+    if (!rankerPromise) {
+        rankerPromise = (async () => {
+            const cacheDir = process.env.TRANSFORMERS_CACHE || path.join(os.homedir(), '.customize-agent', 'models');
+            if (!fs.existsSync(cacheDir)) {
+                fs.mkdirSync(cacheDir, { recursive: true });
+            }
+            process.env.TRANSFORMERS_CACHE = cacheDir;
+            return pipeline('text-classification', process.env.KB_RERANKER_MODEL || 'Xenova/bge-reranker-base', { dtype: 'q8' });
+        })().catch(error => {
+            rankerPromise = null;
+            loadDisabledUntil = Date.now() + 60_000;
+            throw error;
+        });
+    }
+    return rankerPromise;
+}
+function takeHeadTail(text, maxLength) {
+    if (text.length <= maxLength)
+        return text;
+    const headLength = Math.ceil(maxLength * 0.65);
+    const tailLength = maxLength - headLength;
+    return `${text.slice(0, headLength)}\n...\n${text.slice(-tailLength)}`;
+}
+function extractScore(output) {
+    const first = Array.isArray(output) ? output[0] : output;
+    if (Array.isArray(first))
+        return extractScore(first);
+    const score = Number(first?.score ?? 0);
+    return Number.isFinite(score) ? score : 0;
+}
+async function handle(request) {
+    const ranker = await loadRanker();
+    const scores = [];
+    const safeQuery = takeHeadTail(request.query, 240);
+    for (const text of request.texts) {
+        try {
+            const out = await ranker({ text: safeQuery, text_pair: takeHeadTail(text, 1400) });
+            scores.push(extractScore(out));
+        }
+        catch {
+            scores.push(0);
+        }
+    }
+    return scores;
+}
+let queue = Promise.resolve();
+parentPort?.on('message', (request) => {
+    queue = queue.then(async () => {
+        try {
+            const scores = await handle(request);
+            parentPort?.postMessage({ id: request.id, scores });
+        }
+        catch (error) {
+            parentPort?.postMessage({ id: request.id, error: error instanceof Error ? error.message : String(error) });
+        }
+    });
+});

package/dist/extraction/content-extractor.d.ts

@@ -15,6 +15,8 @@ export declare class ContentExtractor {
     private cleanCadReadableText;
     private isLikelyGarbledCadText;
     private isReadableCadValue;
+    /** 图层/块名可读性：在通用可读性过滤外，排除纯数字名称（默认图层「0」等 CAD 内部标识） */
+    private isUsableCadName;
     private cleanExtractedText;
     private textScore;
     /** 统计图纸提取片段中的实际字符数据量（汉字/字母/数字），用于判断"无字符数据不入库" */

package/dist/extraction/content-extractor.js

@@ -5,8 +5,12 @@ import { tmpdir } from 'node:os';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import { resolveAndImport, resolvePackage } from './module-resolver.js';
 import { createOcrProvider } from './ocr-providers.js';
+import { decodeTextBuffer, normalizeSymbolicPua } from './text-encoding.js';
 const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb[\w:]+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*|MLEADERSTYLE|AppInfoHistory|AppInfoDataList|ObjectDBX|Classes|DICTIONARYVARP|ObjFreeSpaceP|AuxHeaderT|\$AUDIT_BAD_\w+)\b/giu;
-const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb[\w:]+|\$AUDIT_BAD_\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous|MLEADERSTYLE|ObjectDBX)$/iu;
+// 注意：不再整行排除纯数字（\d+）——真实图纸的尺寸标注/标高/门窗表数值常为纯数字行
+// （如「100」「2.900」），整行排除会误杀图纸数据；图层/块名等内部纯数字标识在提取处
+// 单独排除（extractDxf 的 layers/blocks 过滤），不受此正则影响
+const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb[\w:]+|\$AUDIT_BAD_\w+|[A-F0-9]{8,}|Model|Layout\d*|ByLayer|Continuous|MLEADERSTYLE|ObjectDBX)$/iu;
 /** DWG 二进制误读产生的 C1 控制字符（U+0080-U+009F，如 GBK 半字节），正常图纸标注不会出现 */
 const CAD_C1_CONTROL_RE = /[\u0080-\u009F]/gu;
 /** AutoCAD 控制码：%%c=Φ、%%d=°、%%p=±、%%132 等数字码=Φ，统一解码为可读符号 */
@@ -87,8 +91,10 @@ export class ContentExtractor {
             warnings.push(...result.warnings);
         }
         else if (this.isTextReadable(file)) {
-            text = fs.readFileSync(file.absolutePath, 'utf8');
+            const decoded = decodeTextBuffer(fs.readFileSync(file.absolutePath));
+            text = decoded.text;
             metadata.extractionMode = 'plain_text';
+            metadata.encoding = decoded.encoding;
             metadata.vectorizable = true;
         }
         else {
@@ -281,8 +287,13 @@ export class ContentExtractor {
         const cleaned = this.cleanCadReadableText(value);
         return cleaned.length >= 2 && !CAD_INTERNAL_LINE_RE.test(cleaned) && /[\p{Script=Han}\p{Letter}\d]/u.test(cleaned) && !this.isLikelyGarbledCadText(cleaned);
     }
+    /** 图层/块名可读性：在通用可读性过滤外，排除纯数字名称（默认图层「0」等 CAD 内部标识） */
+    isUsableCadName(value) {
+        return this.isReadableCadValue(value) && !/^-?\d+(?:\.\d+)?$/u.test(value.trim());
+    }
     cleanExtractedText(value, file) {
-        const normalized = [...value]
+        // 符号字体私用区字符（CAD SHX 直径符号、PDF Wingdings 选项框等）统一映射回标准符号
+        const normalized = [...normalizeSymbolicPua(value)]
             .filter(char => {
             const code = char.charCodeAt(0);
             // U+FFFF 非字符：PDF 表单空白下划线/占位符的提取产物，无检索意义，统一剔除
@@ -312,7 +323,7 @@ export class ContentExtractor {
         const warnings = [];
         const ext = path.extname(file.absolutePath).toLowerCase();
         if (ext === '.dxf')
-            return await this.extractDxf(file, fs.readFileSync(file.absolutePath, 'utf8'), metadata);
+            return await this.extractDxf(file, decodeTextBuffer(fs.readFileSync(file.absolutePath)).text, metadata);
         if (ext === '.dwg') {
             const converted = await this.tryConvertDwgToDxf(file.absolutePath);
             if (converted?.dxfText) {
@@ -323,10 +334,10 @@ export class ContentExtractor {
             warnings.push(...(converted?.warnings ?? ['未检测到可用 DWG→DXF 转换器，使用内置图纸可读文本抽取']));
         }
         if (file.format === 'autocad' && ext === '.dxf') {
-            const raw = fs.readFileSync(file.absolutePath, 'utf8');
-            const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+            const raw = decodeTextBuffer(fs.readFileSync(file.absolutePath)).text;
+            const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).filter(value => this.isUsableCadName(value)).slice(0, 300);
             const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 500);
-            const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+            const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).filter(value => this.isUsableCadName(value)).slice(0, 300);
             const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
             const uniqueLayers = Array.from(new Set(layers));
             const uniqueBlocks = Array.from(new Set(blocks));
@@ -338,9 +349,11 @@ export class ContentExtractor {
             metadata.blockNames = uniqueBlocks.slice(0, 80);
             metadata.entityTypeCount = uniqueEntityTypes.length;
             metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
-            const characterDataCount = this.countCadCharacterData([...textEntities.map(annotation => annotation.text), ...uniqueLayers, ...uniqueBlocks]);
+            // 判空口径只统计标注文本：图层/块名是 CAD 内部结构信息，图纸「空数据」= 无文字标注。
+            // 把图层/块名计入字符数会让空图纸（仅图层结构、无任何标注）错误入库
+            const characterDataCount = this.countCadCharacterData(textEntities.map(annotation => annotation.text));
             if (characterDataCount < MIN_CAD_CHARACTER_DATA) {
-                // 图纸无字符数据（无文字标注、图层/块名均为内部默认值），不入库
+                // 图纸无字符数据（无文字标注），不入库——空数据图纸直接过滤，仅元数据可查
                 metadata.contentCoverage = 'cad_no_extractable_text';
                 metadata.characterDataCount = characterDataCount;
                 warnings.push(`${file.format} DXF 未提取到字符数据（仅 ${characterDataCount} 个可读字符），图纸内容未入库`);
@@ -362,7 +375,7 @@ export class ContentExtractor {
             };
         }
         if (file.format === 'step') {
-            const raw = fs.readFileSync(file.absolutePath, 'utf8');
+            const raw = decodeTextBuffer(fs.readFileSync(file.absolutePath)).text;
             const products = this.matchAll(raw, /PRODUCT\s*\(\s*'([^']*)'\s*,\s*'([^']*)'/giu).slice(0, 300);
             const materials = this.matchAll(raw, /MATERIAL[^']*'([^']+)'/giu).slice(0, 120);
             const entities = this.matchAll(raw, /#\d+\s*=\s*([A-Z0-9_]+)/gu).slice(0, 1000);
@@ -387,7 +400,7 @@ export class ContentExtractor {
             };
         }
         if (file.format === 'iges') {
-            const raw = fs.readFileSync(file.absolutePath, 'utf8');
+            const raw = decodeTextBuffer(fs.readFileSync(file.absolutePath)).text;
             const names = this.matchAll(raw, /'([^']{2,120})'/gu).slice(0, 500);
             const entityTypes = this.matchAll(raw, /^\s*(\d{3,4})\s*,/gmu).slice(0, 1000);
             const uniqueIgesTypes = Array.from(new Set(entityTypes));
@@ -411,7 +424,7 @@ export class ContentExtractor {
         // OLE 属性集（<prop_set ...>）是 DWG 文件内部的二进制属性结构，不含图纸标注字符，
         // 属于解析噪声而非字符数据，直接排除（不锚定行首：误读字符可能混在片段开头）
         const withoutPropSets = binaryFragments.filter(value => !/<prop_set\b/u.test(value));
-        let readable = withoutPropSets.filter(value => this.isReadableCadValue(value)).slice(0, 5000);
+        let readable = withoutPropSets.filter(value => this.isUsableCadName(value)).slice(0, 5000);
         // 文档级字符词频过滤：真实标注文字在图纸中会重复出现（标题/图层/图名等），
         // 随机二进制噪声片段中每个字符几乎只出现一次（孤立字符）；短片段若全部由
         // 孤立字符构成即为随机噪声，丢弃
@@ -468,17 +481,23 @@ export class ContentExtractor {
     async extractDxf(file, raw, metadata) {
         const warnings = [];
         let parsed;
-        try {
-            const mod = await resolveAndImport('dxf-parser');
-            const Parser = mod.default ?? mod;
-            parsed = new Parser().parseSync(raw);
-        }
-        catch {
-            warnings.push('dxf-parser 解析失败，已使用 DXF 文本结构抽取回退');
+        // dxf-parser 对缺少坐标组码的残缺实体（无 10/20 的 LINE/CIRCLE/POLYLINE）存在解析
+        // 死循环缺陷，无限循环不抛异常、try-catch 无法兜住；其解析结果仅用于 metadata 标记，
+        // 不参与文本提取。无任何标注实体的图纸（空图纸）必然判空不入库，跳过 parseSync
+        // 避免触发库死循环；含标注实体的图纸按正常路径解析
+        if (/(?:^|\r?\n)\s*0\s*\r?\n(?:TEXT|MTEXT|DIMENSION|LEADER|ATTRIB)\b/u.test(raw)) {
+            try {
+                const mod = await resolveAndImport('dxf-parser');
+                const Parser = mod.default ?? mod;
+                parsed = new Parser().parseSync(raw);
+            }
+            catch {
+                warnings.push('dxf-parser 解析失败，已使用 DXF 文本结构抽取回退');
+            }
         }
-        const layers = this.matchAll(raw, /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+        const layers = this.matchAll(raw, /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/gu).filter(value => this.isUsableCadName(value)).slice(0, 300);
         const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 5000);
-        const blocks = this.matchAll(raw, /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+        const blocks = this.matchAll(raw, /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/gu).filter(value => this.isUsableCadName(value)).slice(0, 300);
         const entityTypes = this.matchAll(raw, /(?:^|\r?\n)\s*0\s*\r?\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
         const uniqueLayers = Array.from(new Set(layers));
         const uniqueBlocks = Array.from(new Set(blocks));
@@ -491,7 +510,9 @@ export class ContentExtractor {
         metadata.entityTypeCount = uniqueEntityTypes.length;
         metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
         metadata.parsedByDxfParser = Boolean(parsed);
-        const characterDataCount = this.countCadCharacterData([...textEntities.map(annotation => annotation.text), ...uniqueLayers, ...uniqueBlocks]);
+        // 判空口径只统计标注文本：图层/块名是 CAD 内部结构信息，图纸「空数据」= 无文字标注。
+        // 把图层/块名计入字符数会让空图纸（仅图层结构、无任何标注）错误入库
+        const characterDataCount = this.countCadCharacterData(textEntities.map(annotation => annotation.text));
         if (characterDataCount < MIN_CAD_CHARACTER_DATA) {
             // 图纸无字符数据（无文字标注、图层/块名均为内部默认值），不入库
             metadata.contentCoverage = 'cad_no_extractable_text';
@@ -526,20 +547,38 @@ export class ContentExtractor {
         return [`图纸节点: ${fileName}`, ...annotations.map(item => item.text)];
     }
     extractDxfTextAnnotations(raw) {
-        const entities = raw.split(/(?:^|\r?\n)\s*0\s*\r?\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
+        // ATTRIB（块属性）是门窗表/材料表/标题栏数据的载体（块插入时的属性值实体），
+        // 此前遗漏导致整表数据丢失（真实图纸回归：门窗表仅剩零散标注）
+        const entities = raw.split(/(?:^|\r?\n)\s*0\s*\r?\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER|ATTRIB)/u.test(section.trim()));
         return entities.flatMap(section => {
-            const text = this.cleanCadReadableText(/(?:^|\r?\n)\s*(?:1|3)\s*\r?\n([^\r\n]+)/u.exec(section)?.[1] ?? '');
+            const entityType = section.trim().split(/\s+/u)[0];
+            let rawText;
+            if (entityType === 'ATTRIB') {
+                // 属性实体：组码 2 是属性标签名（门窗表/材料表的列名），组码 1 是属性值，
+                // 标签与值是独立语义字段（「型号 M1021」「高度 2100」），空格连接而非续段拼接
+                const tag = /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim() ?? '';
+                const value = /(?:^|\r?\n)\s*1\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim() ?? '';
+                rawText = `${tag} ${value}`;
+            }
+            else {
+                // MTEXT 多段文本：组码 1 是首段（≤255 字符），组码 3 是后续续段（每段 ≤250 字符），
+                // 必须按序拼接才是完整文字（此前只取首个组码 1/3，多段标注丢失大半内容）；
+                // DIMENSION 组码 1/3 是显式尺寸文字与后缀，TEXT/LEADER 组码 3 罕见但同按序收集
+                rawText = [...section.matchAll(/(?:^|\r?\n)\s*(?:1|3)\s*\r?\n([^\r\n]+)/gu)].map(match => match[1]).join('');
+            }
+            const text = this.cleanCadReadableText(rawText);
             if (!text || !this.isReadableCadValue(text))
                 return [];
             // 图层/块名同样要过可读性过滤：DXF 里 GBK 误读（Ïä¹ñ）或纯数字/内部
             // 标识（11、AcDb...）会直接混进节点文本，不合格时置空由语义节点回退
             const layer = /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim() ?? '';
-            const block = /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim() ?? '';
+            // ATTRIB 的组码 2 是属性标签名而非块名，不可当块名使用
+            const block = entityType === 'ATTRIB' ? '' : /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim() ?? '';
             return [{
                     text,
                     layer: layer && this.isReadableCadValue(layer) ? layer : undefined,
                     block: block && this.isReadableCadValue(block) ? block : undefined,
-                    entityType: section.trim().split(/\s+/u)[0],
+                    entityType,
                     x: Number(/(?:^|\r?\n)\s*10\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
                     y: Number(/(?:^|\r?\n)\s*20\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
                 }].map(item => ({ ...item, x: Number.isFinite(item.x) ? item.x : undefined, y: Number.isFinite(item.y) ? item.y : undefined }));
@@ -834,7 +873,8 @@ export class ContentExtractor {
         ].join('\n');
     }
     extractDelimitedText(file) {
-        const raw = fs.readFileSync(file.absolutePath, 'utf8');
+        const decoded = decodeTextBuffer(fs.readFileSync(file.absolutePath));
+        const raw = decoded.text;
         const delimiter = file.format === 'tsv' ? '\t' : ',';
         const rows = raw.split(/\r?\n/u).filter(line => line.trim().length > 0);
         const header = rows[0] ? this.parseDelimitedLine(rows[0], delimiter) : [];
@@ -850,6 +890,7 @@ export class ContentExtractor {
                 extractionMode: 'delimited_text_structured',
                 semanticExtractionMode: 'delimited_markdown_table',
                 vectorizable: true,
+                encoding: decoded.encoding,
                 delimiter: file.format === 'tsv' ? 'tab' : 'comma',
                 rowCount: rows.length,
                 columnCount: header.length,

package/dist/extraction/text-encoding.d.ts

@@ -0,0 +1,21 @@
+/**
+ * 文本编码自动检测与符号字体私用区（PUA）规范化。
+ *
+ * 背景：
+ * - 知识库常混入 GBK 编码文本（中文 CAD 日志、导出 DXF、旧 CSV 等），按 UTF-8 读取会产生大
+ *   量 U+FFFD 替换符乱码；Node 的 TextDecoder 内置 gbk 解码器，无需额外依赖。
+ * - CAD SHX 字体与 PDF 符号字体（Wingdings 等）的字符在提取时会被映射到 Unicode 私用区
+ *   码位（U+E000-U+F8FF），直接入库后检索/阅读均为不可见乱码，需映射回标准符号。
+ */
+/** 将符号字体私用区字符映射回标准 Unicode 符号，正常文本中不出现 PUA，替换是安全的 */
+export declare function normalizeSymbolicPua(value: string): string;
+/**
+ * 文本编码自动检测解码：BOM（UTF-16LE/BE）→ UTF-8 → GBK。
+ * UTF-8 解码替换符率高于 2% 时判定为 GBK 等非 UTF-8 编码；
+ * GBK 对绝大多数字节序列都能成功解码，因此再比较两种解码结果的中文占比，
+ * 只有 GBK 中文语义明显更强时才采用 GBK，避免把含个别非法字节的 UTF-8 文本误转。
+ */
+export declare function decodeTextBuffer(buffer: Buffer): {
+    text: string;
+    encoding: string;
+};

package/dist/extraction/text-encoding.js

@@ -0,0 +1,62 @@
+/**
+ * 文本编码自动检测与符号字体私用区（PUA）规范化。
+ *
+ * 背景：
+ * - 知识库常混入 GBK 编码文本（中文 CAD 日志、导出 DXF、旧 CSV 等），按 UTF-8 读取会产生大
+ *   量 U+FFFD 替换符乱码；Node 的 TextDecoder 内置 gbk 解码器，无需额外依赖。
+ * - CAD SHX 字体与 PDF 符号字体（Wingdings 等）的字符在提取时会被映射到 Unicode 私用区
+ *   码位（U+E000-U+F8FF），直接入库后检索/阅读均为不可见乱码，需映射回标准符号。
+ */
+/** 符号字体私用区 → 标准 Unicode 符号映射（已用实际 PDF/DXF 字形渲染验证） */
+const SYMBOLIC_PUA_MAP = {
+    0xE000: 'Φ', // AutoCAD SHX 字体直径符号 %%c 经 DXF 转换的私用区码位
+    0xE002: 'Φ', // AutoCAD SHX 字体直径符号 %%c 的另一私用区码位（不同 SHX 字体）
+    0xF052: '●', // Wingdings 2 码位 0x52 实心圆（招标文件选项标记/绿建星级标记）
+    0xF0A3: '□', // Wingdings 2 码位 0xA3 空心方框（招标文件选项框）
+};
+/** 将符号字体私用区字符映射回标准 Unicode 符号，正常文本中不出现 PUA，替换是安全的 */
+export function normalizeSymbolicPua(value) {
+    let result = '';
+    let dirty = false;
+    for (const char of value) {
+        const mapped = SYMBOLIC_PUA_MAP[char.charCodeAt(0)];
+        if (mapped) {
+            result += mapped;
+            dirty = true;
+        }
+        else {
+            result += char;
+        }
+    }
+    return dirty ? result : value;
+}
+/**
+ * 文本编码自动检测解码：BOM（UTF-16LE/BE）→ UTF-8 → GBK。
+ * UTF-8 解码替换符率高于 2% 时判定为 GBK 等非 UTF-8 编码；
+ * GBK 对绝大多数字节序列都能成功解码，因此再比较两种解码结果的中文占比，
+ * 只有 GBK 中文语义明显更强时才采用 GBK，避免把含个别非法字节的 UTF-8 文本误转。
+ */
+export function decodeTextBuffer(buffer) {
+    // UTF-16 BOM 检测（部分 Windows 工具导出的 DXF/CSV 为 UTF-16）
+    if (buffer.length >= 2 && buffer[0] === 0xFF && buffer[1] === 0xFE) {
+        return { text: new TextDecoder('utf-16le').decode(buffer).replace(/^\uFEFF/u, ''), encoding: 'utf-16le' };
+    }
+    if (buffer.length >= 2 && buffer[0] === 0xFE && buffer[1] === 0xFF) {
+        return { text: new TextDecoder('utf-16be').decode(buffer).replace(/^\uFEFF/u, ''), encoding: 'utf-16be' };
+    }
+    const utf8Text = new TextDecoder('utf-8', { fatal: false }).decode(buffer);
+    const replacementRatio = (utf8Text.match(/\uFFFD/gu) ?? []).length / Math.max(1, utf8Text.length);
+    if (replacementRatio <= 0.02)
+        return { text: utf8Text, encoding: 'utf-8' };
+    try {
+        const gbkText = new TextDecoder('gbk', { fatal: true }).decode(buffer);
+        const cjkRatio = (value) => (value.match(/[\u4E00-\u9FFF]/gu) ?? []).length / Math.max(1, value.length);
+        if (replacementRatio > 0.3 || cjkRatio(gbkText) > cjkRatio(utf8Text)) {
+            return { text: gbkText, encoding: 'gbk' };
+        }
+        return { text: utf8Text, encoding: 'utf-8' };
+    }
+    catch {
+        return { text: utf8Text, encoding: 'utf-8' };
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.3.5",
+  "version": "4.3.6",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",