@customize-agent/knowledge 4.0.8 → 4.0.9

package/dist/chunking/text-chunker.js

@@ -324,7 +324,7 @@ export class TextChunker {
     withHeader(text, file, config) {
         if (!config.headerInjection)
             return text;
-        return `文件: ${file.relativePath}\n类型: ${file.category}/${file.format}\n\n${text}`;
+        return `资料类型: ${file.category}/${file.format}\n\n${text}`;
     }
     createChunk(index, candidate, file, metadata) {
         const text = candidate.text.trim();

package/dist/core/index-state-store.js

@@ -278,7 +278,7 @@ export class IndexStateStore {
       `).all(matchQuery, limit * 8);
             return rows
                 .map(row => {
-                const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
+                const keyword = this.scoreChunkDetailed(`${String(row.section_title ?? '')}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
                 const bm25Score = this.bm25ToPositiveScore(Number(row.bm25_score));
                 return this.rowToChunk(row, keyword.keywordScore + bm25Score, { ...keyword, bm25Score });
             })
@@ -299,7 +299,7 @@ export class IndexStateStore {
     `).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), limit * 6);
         return rows
             .map(row => {
-            const keyword = this.scoreChunkDetailed(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
+            const keyword = this.scoreChunkDetailed(`${String(row.section_title ?? '')}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms);
             return this.rowToChunk(row, keyword.keywordScore, keyword);
         })
             .filter(row => row.score > 0)

package/dist/extraction/content-extractor.d.ts

@@ -16,6 +16,9 @@ export declare class ContentExtractor {
     private extractTextClipping;
     private swapUtf16Bytes;
     private extractReadableFragments;
+    private cleanCadReadableText;
+    private isReadableCadValue;
+    private cleanExtractedText;
     private textScore;
     private extractCad;
     private extractDxf;

package/dist/extraction/content-extractor.js

@@ -5,6 +5,8 @@ import { tmpdir } from 'node:os';
 import { pathToFileURL } from 'node:url';
 import { ExternalExtractorRegistry } from './external-extractor.js';
 import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
+const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb\w+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*)\b/giu;
+const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous)$/iu;
 /** 文件内容提取器，支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
 export class ContentExtractor {
     externalExtractors;
@@ -102,7 +104,7 @@ export class ContentExtractor {
             }
         }
         return {
-            text: text.trim(),
+            text: this.cleanExtractedText(text, file).trim(),
             metadata,
             warnings,
             extractionTimeMs: Date.now() - start,
@@ -177,9 +179,36 @@ export class ContentExtractor {
         return value
             .replace(/[^\p{L}\p{N}\p{P}\p{S}\s]/gu, '\n')
             .split(/[\r\n]+/u)
-            .map(line => line.replace(/\s+/gu, ' ').trim())
+            .map(line => this.cleanCadReadableText(line))
             .filter(line => line.length >= 2 && /[\p{L}\p{N}]/u.test(line));
     }
+    cleanCadReadableText(value) {
+        return value
+            .replace(CAD_INTERNAL_TOKEN_RE, '')
+            .replace(/\b(?:LINE|LWPOLYLINE|POLYLINE|INSERT|HATCH|CIRCLE|ARC|DIMENSION|TEXT|MTEXT)\b/giu, '')
+            .replace(/\s+/gu, ' ')
+            .trim();
+    }
+    isReadableCadValue(value) {
+        const cleaned = this.cleanCadReadableText(value);
+        return cleaned.length >= 2 && !CAD_INTERNAL_LINE_RE.test(cleaned) && /[\p{Script=Han}\p{Letter}\d]/u.test(cleaned);
+    }
+    cleanExtractedText(value, file) {
+        const normalized = [...value]
+            .filter(char => {
+            const code = char.charCodeAt(0);
+            return code === 9 || code === 10 || code === 13 || code >= 32;
+        })
+            .join('');
+        if (file.category !== 'cad')
+            return normalized.replace(/\n{3,}/gu, '\n\n');
+        return normalized
+            .split(/\r?\n/u)
+            .map(line => this.cleanCadReadableText(line))
+            .filter(line => line && !CAD_INTERNAL_LINE_RE.test(line))
+            .join('\n')
+            .replace(/\n{3,}/gu, '\n\n');
+    }
     textScore(value) {
         const cjk = (value.match(/[\p{Script=Han}]/gu) ?? []).length;
         const alnum = (value.match(/[\p{L}\p{N}]/gu) ?? []).length;
@@ -202,9 +231,9 @@ export class ContentExtractor {
         }
         if (file.format === 'autocad' && ext === '.dxf') {
             const raw = fs.readFileSync(file.absolutePath, 'utf8');
-            const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
+            const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
             const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 500);
-            const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
+            const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
             const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
             const uniqueLayers = Array.from(new Set(layers));
             const uniqueBlocks = Array.from(new Set(blocks));
@@ -278,7 +307,7 @@ export class ContentExtractor {
             if (result.text.trim())
                 return result;
         }
-        const readable = this.extractBinaryReadableFragments(file.absolutePath).slice(0, 800);
+        const readable = this.extractBinaryReadableFragments(file.absolutePath).filter(value => this.isReadableCadValue(value)).slice(0, 5000);
         metadata.extractionMode = 'builtin_cad_readable_fragments';
         metadata.contentCoverage = readable.length > 0 ? 'cad_readable_text_fragments' : 'metadata';
         metadata.stringCount = readable.length;
@@ -303,10 +332,10 @@ export class ContentExtractor {
         catch {
             warnings.push('dxf-parser 解析失败，已使用 DXF 文本结构抽取回退');
         }
-        const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
-        const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 800);
-        const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
-        const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
+        const layers = this.matchAll(raw, /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+        const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 5000);
+        const blocks = this.matchAll(raw, /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/gu).filter(value => this.isReadableCadValue(value)).slice(0, 300);
+        const entityTypes = this.matchAll(raw, /(?:^|\r?\n)\s*0\s*\r?\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
         const uniqueLayers = Array.from(new Set(layers));
         const uniqueBlocks = Array.from(new Set(blocks));
         const uniqueEntityTypes = Array.from(new Set(entityTypes));
@@ -350,18 +379,18 @@ export class ContentExtractor {
         });
     }
     extractDxfTextAnnotations(raw) {
-        const entities = raw.split(/\n\s*0\s*\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
+        const entities = raw.split(/(?:^|\r?\n)\s*0\s*\r?\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
         return entities.flatMap(section => {
-            const text = /\n\s*(?:1|3)\s*\n([^\n]+)/u.exec(section)?.[1]?.trim();
-            if (!text)
+            const text = this.cleanCadReadableText(/(?:^|\r?\n)\s*(?:1|3)\s*\r?\n([^\r\n]+)/u.exec(section)?.[1] ?? '');
+            if (!text || !this.isReadableCadValue(text))
                 return [];
             return [{
                     text,
-                    layer: /\n\s*8\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
-                    block: /\n\s*2\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
+                    layer: /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim(),
+                    block: /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim(),
                     entityType: section.trim().split(/\s+/u)[0],
-                    x: Number(/\n\s*10\s*\n([^\n]+)/u.exec(section)?.[1]),
-                    y: Number(/\n\s*20\s*\n([^\n]+)/u.exec(section)?.[1]),
+                    x: Number(/(?:^|\r?\n)\s*10\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
+                    y: Number(/(?:^|\r?\n)\s*20\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
                 }].map(item => ({ ...item, x: Number.isFinite(item.x) ? item.x : undefined, y: Number.isFinite(item.y) ? item.y : undefined }));
         });
     }
@@ -1165,7 +1194,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
             if (pages.length > 0) {
                 const combined = pages.join('\n\n');
                 if (combined.trim())
-                    return combined.slice(0, 250_000);
+                    return combined;
             }
         }
         catch (e) {
@@ -1179,7 +1208,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
             if (pdfParse) {
                 const result = await pdfParse(buffer);
                 if (result.text.trim())
-                    return result.text.slice(0, 250_000);
+                    return result.text;
             }
         }
         catch {
@@ -1450,15 +1479,8 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
         return lines.filter(line => !line.endsWith(':'));
     }
     metadataOnlyText(file) {
-        const fileName = path.basename(file.relativePath);
-        const directory = path.dirname(file.relativePath);
-        const searchableName = fileName.replace(/[_\-.]+/gu, ' ');
         return [
-            `文件名: ${fileName}`,
-            `文件路径: ${file.relativePath}`,
-            `所在目录: ${directory === '.' ? 'knowledgeBase' : directory}`,
-            `可搜索名称: ${searchableName}`,
-            `文件类型: ${file.category}/${file.format}`,
+            `资料类型: ${file.category}/${file.format}`,
             `MIME: ${file.mimeType}`,
             `文件大小: ${file.fileSize} bytes`,
         ].join('\n');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.8",
+  "version": "4.0.9",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",