@customize-agent/knowledge 3.0.1 → 3.0.3

package/dist/classification/classifier.js

@@ -50,7 +50,7 @@ export class FileClassifier {
             ['.pdf', 'document', 'pdf'], ['.docx', 'document', 'office'], ['.doc', 'document', 'office'], ['.rtf', 'document', 'office'], ['.odt', 'document', 'office'],
             ['.pptx', 'document', 'presentation'], ['.ppt', 'document', 'presentation'], ['.odp', 'document', 'presentation'],
             ['.md', 'document', 'markdown'], ['.markdown', 'document', 'markdown'], ['.mdx', 'document', 'markdown'],
-            ['.txt', 'document', 'plaintext'], ['.rst', 'document', 'plaintext'], ['.asciidoc', 'document', 'plaintext'], ['.tex', 'document', 'plaintext'],
+            ['.txt', 'document', 'plaintext'], ['.textclipping', 'document', 'text_clipping'], ['.rst', 'document', 'plaintext'], ['.asciidoc', 'document', 'plaintext'], ['.tex', 'document', 'plaintext'],
             ['.epub', 'document', 'ebook'], ['.mobi', 'document', 'ebook'],
             ['.xlsx', 'spreadsheet', 'excel'], ['.xls', 'spreadsheet', 'excel'], ['.xlsm', 'spreadsheet', 'excel'],
             ['.csv', 'spreadsheet', 'csv'], ['.tsv', 'spreadsheet', 'csv'], ['.tab', 'spreadsheet', 'csv'], ['.ods', 'spreadsheet', 'opendoc'],

package/dist/core/knowledge-base-manager.d.ts

@@ -90,6 +90,13 @@ export declare class KnowledgeBaseManager {
     uploadFile(fileName: string, content: Buffer, targetRelativePath?: string, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
         vectorMode?: 'sync' | 'defer';
     }): Promise<DiffResult>;
+    uploadFiles(files: Array<{
+        fileName: string;
+        content: Buffer;
+        targetRelativePath?: string;
+    }>, onProgress?: (progress: KnowledgeIndexProgress) => void, options?: {
+        vectorMode?: 'sync' | 'defer';
+    }): Promise<DiffResult>;
     listFailedFiles(): DiffResult['skippedFiles'];
     removeFile(relativePath: string): Promise<void>;
     tagFile(relativePath: string, tags: string[]): void;

package/dist/core/knowledge-base-manager.js

@@ -368,11 +368,16 @@ export class KnowledgeBaseManager {
         return targetRelativePath ?? this.defaultUploadRelativePath(fileName);
     }
     async uploadFile(fileName, content, targetRelativePath, onProgress, options = {}) {
+        return this.uploadFiles([{ fileName, content, targetRelativePath }], onProgress, options);
+    }
+    async uploadFiles(files, onProgress, options = {}) {
         this.initialize();
-        const relativePath = this.getUploadRelativePath(fileName, targetRelativePath);
-        const targetPath = this.resolveKbRelativePath(relativePath);
-        fs.mkdirSync(path.dirname(targetPath), { recursive: true });
-        fs.writeFileSync(targetPath, content);
+        for (const file of files) {
+            const relativePath = this.getUploadRelativePath(file.fileName, file.targetRelativePath);
+            const targetPath = this.resolveKbRelativePath(relativePath);
+            fs.mkdirSync(path.dirname(targetPath), { recursive: true });
+            fs.writeFileSync(targetPath, file.content);
+        }
         return this.incrementalIndex({ onProgress, vectorMode: options.vectorMode });
     }
     listFailedFiles() {

package/dist/extraction/content-extractor.d.ts

@@ -11,14 +11,25 @@ export declare class ContentExtractor {
     constructor(externalExtractors?: ExternalExtractorRegistry);
     extract(file: ClassifiedFile): Promise<ExtractionResult>;
     private tryExternalExtractor;
+    private extractTextClipping;
+    private swapUtf16Bytes;
+    private extractReadableFragments;
+    private textScore;
     private extractCad;
+    private extractDxf;
+    private tryConvertDwgToDxf;
     private extractCadMesh;
+    private getTempRoot;
+    private extractBinaryReadableFragments;
     private extractBinaryStrings;
     private extractData;
     private extractDiagram;
     private parseDelimitedLine;
     private extractDelimitedText;
     private extractOfficeDocument;
+    private extractRtf;
+    private extractLegacyWordDocument;
+    private extractLegacyOfficeBinary;
     private extractSpreadsheet;
     private extractOfficeZip;
     private extractArchive;

package/dist/extraction/content-extractor.js

@@ -1,6 +1,7 @@
 import { spawnSync } from 'node:child_process';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { tmpdir } from 'node:os';
 import { ExternalExtractorRegistry } from './external-extractor.js';
 import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
 export class ContentExtractor {
@@ -24,7 +25,7 @@ export class ContentExtractor {
             warnings.push(...external.warnings);
         }
         else if (file.category === 'cad') {
-            const result = this.extractCad(file);
+            const result = await this.extractCad(file);
             text = result.text;
             Object.assign(metadata, result.metadata);
             warnings.push(...result.warnings);
@@ -83,6 +84,12 @@ export class ContentExtractor {
             Object.assign(metadata, result.metadata);
             warnings.push(...result.warnings);
         }
+        else if (file.format === 'text_clipping') {
+            const result = this.extractTextClipping(file);
+            text = result.text;
+            Object.assign(metadata, result.metadata);
+            warnings.push(...result.warnings);
+        }
         else if (this.isTextReadable(file)) {
             text = fs.readFileSync(file.absolutePath, 'utf8');
             metadata.extractionMode = 'plain_text';
@@ -133,10 +140,67 @@ export class ContentExtractor {
         }
         return undefined;
     }
-    extractCad(file) {
+    extractTextClipping(file) {
+        const buffer = fs.readFileSync(file.absolutePath);
+        const candidates = [
+            buffer.toString('utf16le'),
+            this.swapUtf16Bytes(buffer).toString('utf16le'),
+            buffer.toString('utf8'),
+            ...this.extractBinaryStrings(file.absolutePath),
+        ];
+        const fragments = candidates.flatMap(candidate => this.extractReadableFragments(candidate));
+        const unique = Array.from(new Set(fragments))
+            .filter(fragment => fragment.length >= 2 && !/^bplist\d+/u.test(fragment))
+            .sort((a, b) => this.textScore(b) - this.textScore(a))
+            .slice(0, 50);
+        const text = unique.join('\n');
+        return {
+            text: text ? [this.metadataOnlyText(file), text].join('\n') : this.metadataOnlyText(file),
+            metadata: {
+                extractionMode: 'builtin_text_clipping',
+                vectorizable: true,
+                contentCoverage: text ? 'text_clipping_payload' : 'metadata',
+                fragmentCount: unique.length,
+            },
+            warnings: text ? [] : ['未从 textClipping 中提取到剪贴文本，仅入库元数据'],
+        };
+    }
+    swapUtf16Bytes(buffer) {
+        const swapped = Buffer.from(buffer);
+        for (let i = 0; i + 1 < swapped.length; i += 2) {
+            const first = swapped[i] ?? 0;
+            swapped[i] = swapped[i + 1] ?? 0;
+            swapped[i + 1] = first;
+        }
+        return swapped;
+    }
+    extractReadableFragments(value) {
+        return value
+            .replace(/[^\p{L}\p{N}\p{P}\p{S}\s]/gu, '\n')
+            .split(/[\r\n]+/u)
+            .map(line => line.replace(/\s+/gu, ' ').trim())
+            .filter(line => line.length >= 2 && /[\p{L}\p{N}]/u.test(line));
+    }
+    textScore(value) {
+        const cjk = (value.match(/[\p{Script=Han}]/gu) ?? []).length;
+        const alnum = (value.match(/[\p{L}\p{N}]/gu) ?? []).length;
+        return cjk * 4 + alnum + Math.min(value.length, 200) / 20;
+    }
+    async extractCad(file) {
         const metadata = { extractionMode: 'builtin_cad_structural', vectorizable: true };
         const warnings = [];
         const ext = path.extname(file.absolutePath).toLowerCase();
+        if (ext === '.dxf')
+            return await this.extractDxf(file, fs.readFileSync(file.absolutePath, 'utf8'), metadata);
+        if (ext === '.dwg') {
+            const converted = await this.tryConvertDwgToDxf(file.absolutePath);
+            if (converted?.dxfText) {
+                const parsed = await this.extractDxf(file, converted.dxfText, { ...metadata, extractionMode: converted.tool, convertedFrom: 'dwg' });
+                parsed.warnings.push(...converted.warnings);
+                return parsed;
+            }
+            warnings.push(...(converted?.warnings ?? ['未检测到可用 DWG→DXF 转换器，使用内置图纸可读文本抽取']));
+        }
         if (file.format === 'autocad' && ext === '.dxf') {
             const raw = fs.readFileSync(file.absolutePath, 'utf8');
             const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
@@ -213,18 +277,82 @@ export class ContentExtractor {
             if (result.text.trim())
                 return result;
         }
-        const binaryStrings = this.extractBinaryStrings(file.absolutePath).slice(0, 500);
-        metadata.extractionMode = 'builtin_cad_binary_strings';
-        metadata.contentCoverage = binaryStrings.length > 0 ? 'cad_binary_strings' : 'metadata';
-        metadata.stringCount = binaryStrings.length;
-        if (binaryStrings.length === 0)
-            warnings.push(`${file.format} 内置 CAD 解析器未提取到可用文本，未入库`);
+        const readable = this.extractBinaryReadableFragments(file.absolutePath).slice(0, 800);
+        metadata.extractionMode = 'builtin_cad_readable_fragments';
+        metadata.contentCoverage = readable.length > 0 ? 'cad_readable_text_fragments' : 'metadata';
+        metadata.stringCount = readable.length;
+        if (readable.length === 0)
+            warnings.push(`${file.format} 内置 CAD 解析器未提取到可用文本，已入库文件元数据用于检索`);
+        else
+            warnings.push(`${file.format} 未检测到专业 DWG 转换器，已使用内置可读标注/标题块抽取；如需完整图纸结构，请安装 ODA File Converter 或 LibreDWG 并配置外部解析器`);
+        return {
+            text: readable.length > 0 ? [this.metadataOnlyText(file), `CAD 图纸可读标注/标题块/属性:\n${readable.join('\n')}`].join('\n') : this.metadataOnlyText(file),
+            metadata,
+            warnings,
+        };
+    }
+    async extractDxf(file, raw, metadata) {
+        const warnings = [];
+        let parsed;
+        try {
+            const mod = await resolveAndImport('dxf-parser');
+            const Parser = mod.default ?? mod;
+            parsed = new Parser().parseSync(raw);
+        }
+        catch {
+            warnings.push('dxf-parser 解析失败，已使用 DXF 文本结构抽取回退');
+        }
+        const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
+        const textEntities = this.matchAll(raw, /\n\s*(?:1|3)\s*\n([^\n]+)/gu).slice(0, 800);
+        const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
+        const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
+        const uniqueLayers = Array.from(new Set(layers));
+        const uniqueBlocks = Array.from(new Set(blocks));
+        const uniqueEntityTypes = Array.from(new Set(entityTypes));
+        metadata.layerCount = uniqueLayers.length;
+        metadata.layerNames = uniqueLayers.slice(0, 80);
+        metadata.textEntityCount = textEntities.length;
+        metadata.blockCount = uniqueBlocks.length;
+        metadata.blockNames = uniqueBlocks.slice(0, 80);
+        metadata.entityTypeCount = uniqueEntityTypes.length;
+        metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
+        metadata.contentCoverage = 'dxf_layers_blocks_entities_text';
+        metadata.parsedByDxfParser = Boolean(parsed);
         return {
-            text: binaryStrings.length > 0 ? [this.metadataOnlyText(file), `CAD 二进制字符串/标题块:\n${binaryStrings.join('\n')}`].join('\n') : '',
+            text: [
+                this.metadataOnlyText(file),
+                `CAD DXF 图层: ${uniqueLayers.join(', ')}`,
+                `CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
+                `CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
+                `CAD DXF 标注/文本:\n${textEntities.join('\n')}`,
+            ].join('\n'),
             metadata,
             warnings,
         };
     }
+    async tryConvertDwgToDxf(filePath) {
+        const tmpDir = fs.mkdtempSync(path.join(this.getTempRoot(), 'customize-dwg-'));
+        const outputPath = path.join(tmpDir, `${path.basename(filePath, path.extname(filePath))}.dxf`);
+        try {
+            const customCmd = process.env.CUSTOMIZE_DWG_TO_DXF_CMD;
+            if (customCmd) {
+                const command = customCmd.replace(/\{input\}/gu, filePath).replace(/\{output\}/gu, outputPath);
+                const result = spawnSync(command, { shell: true, encoding: 'utf8', timeout: 120_000 });
+                if (result.status === 0 && fs.existsSync(outputPath))
+                    return { dxfText: fs.readFileSync(outputPath, 'utf8'), tool: 'external_dwg_to_dxf', warnings: [] };
+                return { tool: 'external_dwg_to_dxf', warnings: [`CUSTOMIZE_DWG_TO_DXF_CMD 转换失败: ${result.stderr || result.stdout || result.error?.message || 'unknown error'}`] };
+            }
+            for (const bin of ['dwgread', 'dwg2dxf']) {
+                const result = spawnSync(bin, bin === 'dwgread' ? ['-O', 'DXF', '-o', outputPath, filePath] : [filePath, outputPath], { encoding: 'utf8', timeout: 120_000 });
+                if (result.status === 0 && fs.existsSync(outputPath))
+                    return { dxfText: fs.readFileSync(outputPath, 'utf8'), tool: bin, warnings: [] };
+            }
+            return { tool: 'builtin_fallback', warnings: ['未检测到可用 DWG→DXF 转换器（dwgread/dwg2dxf/CUSTOMIZE_DWG_TO_DXF_CMD），使用内置图纸可读文本抽取'] };
+        }
+        finally {
+            fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+    }
     extractCadMesh(file, ext, metadata) {
         const warnings = [];
         if (['.obj', '.gltf'].includes(ext)) {
@@ -258,13 +386,25 @@ export class ContentExtractor {
         metadata.contentCoverage = binaryStrings.length > 0 ? 'mesh_binary_strings' : 'metadata';
         return { text: binaryStrings.length > 0 ? [this.metadataOnlyText(file), `Mesh 二进制字符串:\n${binaryStrings.join('\n')}`].join('\n') : '', metadata, warnings };
     }
-    extractBinaryStrings(filePath) {
+    getTempRoot() {
+        return process.env.CUSTOMIZE_TMPDIR || process.env.TMPDIR || tmpdir();
+    }
+    extractBinaryReadableFragments(filePath) {
         const buffer = fs.readFileSync(filePath);
-        const raw = buffer.toString('latin1');
-        return Array.from(raw.matchAll(/[A-Za-z0-9_ .:\-/\\\u4e00-\u9fa5]{4,}/gu), match => match[0].trim())
-            .filter(value => value.length >= 4 && !/^\d+$/u.test(value))
+        const candidates = [
+            buffer.toString('utf8'),
+            buffer.toString('utf16le'),
+            this.swapUtf16Bytes(buffer).toString('utf16le'),
+            buffer.toString('latin1'),
+        ];
+        return Array.from(new Set(candidates.flatMap(candidate => this.extractReadableFragments(candidate))))
+            .filter(value => value.length >= 3 && !/^\d+$/u.test(value))
+            .sort((a, b) => this.textScore(b) - this.textScore(a))
             .slice(0, 2_000);
     }
+    extractBinaryStrings(filePath) {
+        return this.extractBinaryReadableFragments(filePath);
+    }
     extractData(file) {
         const raw = fs.readFileSync(file.absolutePath, 'utf8');
         const metadata = { extractionMode: 'structured_data', vectorizable: true };
@@ -378,7 +518,14 @@ export class ContentExtractor {
         };
     }
     async extractOfficeDocument(file) {
-        if (path.extname(file.absolutePath).toLowerCase() === '.docx') {
+        const ext = path.extname(file.absolutePath).toLowerCase();
+        if (ext === '.rtf')
+            return this.extractRtf(file);
+        if (ext === '.doc')
+            return this.extractLegacyWordDocument(file);
+        if (ext === '.ppt')
+            return this.extractLegacyOfficeBinary(file);
+        if (ext === '.docx') {
             try {
                 const mammoth = await resolveAndImport('mammoth');
                 const result = await mammoth.extractRawText({ path: file.absolutePath });
@@ -397,7 +544,50 @@ export class ContentExtractor {
         }
         return this.extractOfficeZip(file);
     }
+    extractRtf(file) {
+        const raw = fs.readFileSync(file.absolutePath, 'utf8');
+        const text = raw
+            .replace(/\\'[0-9a-fA-F]{2}/gu, ' ')
+            .replace(/\\[a-zA-Z]+-?\d* ?/gu, ' ')
+            .replace(/[{}]/gu, ' ')
+            .replace(/\s+/gu, ' ')
+            .trim();
+        return {
+            text,
+            metadata: { extractionMode: 'builtin_rtf_text', vectorizable: true, contentCoverage: 'rtf_text' },
+            warnings: text ? [] : ['RTF 解析未提取到正文，未入库'],
+        };
+    }
+    async extractLegacyWordDocument(file) {
+        try {
+            const mod = await resolveAndImport('word-extractor');
+            const WordExtractor = mod.default ?? mod;
+            const document = await new WordExtractor().extract(file.absolutePath);
+            const text = document.getBody().trim();
+            if (text) {
+                return {
+                    text,
+                    metadata: { extractionMode: 'builtin_word_extractor', vectorizable: true, contentCoverage: 'legacy_word_full_text', textLength: text.length },
+                    warnings: [],
+                };
+            }
+        }
+        catch {
+            // fallback below
+        }
+        return this.extractLegacyOfficeBinary(file);
+    }
+    extractLegacyOfficeBinary(file) {
+        const strings = this.extractBinaryStrings(file.absolutePath).slice(0, 1_000);
+        const text = strings.join('\n').trim();
+        return {
+            text,
+            metadata: { extractionMode: 'builtin_legacy_office_binary_strings', vectorizable: true, contentCoverage: 'legacy_office_binary_strings', stringCount: strings.length },
+            warnings: text ? [] : ['旧版 Office 二进制文件未提取到正文，未入库'],
+        };
+    }
     async extractSpreadsheet(file) {
+        const ext = path.extname(file.absolutePath).toLowerCase();
         try {
             const XLSX = await resolveAndImport('xlsx');
             const workbook = XLSX.readFile(file.absolutePath, { cellDates: true, cellFormula: true, cellNF: true, cellStyles: true });
@@ -442,8 +632,12 @@ export class ContentExtractor {
             }
         }
         catch {
+            if (ext === '.xls')
+                return this.extractLegacyOfficeBinary(file);
             // fallback below
         }
+        if (ext === '.xls')
+            return this.extractLegacyOfficeBinary(file);
         return this.extractOfficeZip(file);
     }
     async extractOfficeZip(file) {
@@ -461,13 +655,14 @@ export class ContentExtractor {
                     texts.push(`${entry.name}: ${stripped.slice(0, 8_000)}`);
             }
             metadata.entryCount = Object.keys(zip.files).length;
-            metadata.contentCoverage = texts.length > 0 ? 'office_zip_xml_text' : 'metadata_filename';
-            return { text: [this.metadataOnlyText(file), ...texts].join('\n'), metadata, warnings: texts.length ? [] : ['未从 Office 压缩结构中提取到正文，已跳过入库'] };
+            metadata.contentCoverage = texts.length > 0 ? 'office_zip_xml_text' : 'office_zip_empty_text';
+            return { text: texts.join('\n'), metadata, warnings: texts.length ? [] : ['Office/表格/演示文件未提取到正文，未入库'] };
         }
         catch (error) {
             metadata.extractionMode = 'office_zip_failed';
             metadata.parseError = error instanceof Error ? error.message : String(error);
-            return { text: '', metadata, warnings: ['Office/表格/演示文件解析失败，内置解析器未提取到正文，未入库'] };
+            metadata.contentCoverage = 'office_zip_failed';
+            return { text: '', metadata, warnings: [`Office/表格/演示文件解析失败: ${metadata.parseError}，未入库`] };
         }
     }
     async extractArchive(file) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "3.0.1",
+  "version": "3.0.3",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",
@@ -32,15 +32,17 @@
     "knowledge-base"
   ],
   "dependencies": {
+    "@napi-rs/canvas": "^0.1.82",
     "better-sqlite3": "^12.10.0",
-    "sqlite-vec": "^0.1.9",
+    "dxf-parser": "^1.1.2",
     "fast-glob": "^3.3.3",
     "jszip": "^3.10.1",
-    "@napi-rs/canvas": "^0.1.82",
     "mammoth": "^1.12.0",
     "pdf-parse": "^2.4.5",
     "pdfjs-dist": "^5.4.394",
+    "sqlite-vec": "^0.1.9",
     "tesseract.js": "^7.0.0",
+    "word-extractor": "^1.0.4",
     "xlsx": "^0.18.5"
   },
   "devDependencies": {