@customize-agent/knowledge 4.0.37 → 4.0.38

package/dist/chunking/text-chunker.d.ts

@@ -28,7 +28,7 @@ export declare class TextChunker {
     chunk(text: string, file: ClassifiedFile, metadata?: Record<string, unknown>): TextChunk[];
     private createCandidates;
     private createTextCandidates;
-    private createTableCandidates;
+    private createMixedTableCandidates;
     private createDataCandidates;
     private createCodeCandidates;
     private normalizeCodeLanguage;
@@ -44,6 +44,7 @@ export declare class TextChunker {
     private splitByWindow;
     private splitBySentenceBoundary;
     private isMarkdownTable;
+    private hasMarkdownTableBlock;
     private extractMarkdownTableBlocks;
     private extractMarkdownTableRowRange;
     private splitMarkdownTable;

package/dist/chunking/text-chunker.js

@@ -50,8 +50,8 @@ export class TextChunker {
         return candidates.map((candidate, index) => this.createChunk(index, candidate, file, metadata));
     }
     createCandidates(text, file, config) {
-        if (file.category === 'spreadsheet')
-            return this.createTableCandidates(text, config);
+        if (file.category === 'spreadsheet' || this.hasMarkdownTableBlock(text))
+            return this.createMixedTableCandidates(text, file.category, config);
         if (file.category === 'data')
             return this.createDataCandidates(text, config);
         if (file.category === 'code')
@@ -85,33 +85,52 @@ export class TextChunker {
         });
         return candidates;
     }
-    createTableCandidates(text, config) {
-        if (this.isMarkdownTable(text)) {
-            const tableBlocks = this.extractMarkdownTableBlocks(text);
-            if (tableBlocks.length > 0) {
-                return tableBlocks.flatMap((block, parentIndex) => {
-                    const sectionTitle = this.extractSectionTitle(block.text) ?? '表格数据';
-                    return this.splitMarkdownTable(block.text, config.maxChunkSize).map((part, childIndex) => {
-                        const localStart = Math.max(0, block.text.indexOf(part.slice(0, 40)));
-                        const startChar = block.startChar + localStart;
-                        return {
-                            text: part,
-                            startChar,
-                            endChar: startChar + part.length,
-                            sectionTitle,
-                            titlePath: sectionTitle,
-                            kind: 'table',
-                            parentId: `table-${parentIndex}`,
-                            parentIndex,
-                            childIndex,
-                            rowRange: this.extractMarkdownTableRowRange(part),
-                            parentText: block.text,
-                        };
-                    });
+    createMixedTableCandidates(text, category, config) {
+        const tableBlocks = this.extractMarkdownTableBlocks(text);
+        if (tableBlocks.length === 0)
+            return this.createTextCandidates(text, category, config);
+        const candidates = [];
+        let cursor = 0;
+        tableBlocks.forEach((block, parentIndex) => {
+            const beforeStart = cursor;
+            const before = text.slice(beforeStart, block.startChar).trim();
+            if (before) {
+                candidates.push(...this.createTextCandidates(before, category, config).map(candidate => ({
+                    ...candidate,
+                    startChar: candidate.startChar + beforeStart,
+                    endChar: candidate.endChar + beforeStart,
+                })));
+            }
+            const sectionTitle = this.extractSectionTitle(block.text) ?? '表格数据';
+            for (const [childIndex, part] of this.splitMarkdownTable(block.text, config.maxChunkSize).entries()) {
+                const localStart = Math.max(0, block.text.indexOf(part.slice(0, 40)));
+                const startChar = block.startChar + localStart;
+                candidates.push({
+                    text: part,
+                    startChar,
+                    endChar: startChar + part.length,
+                    sectionTitle,
+                    titlePath: sectionTitle,
+                    kind: 'table',
+                    parentId: `table-${parentIndex}`,
+                    parentIndex,
+                    childIndex,
+                    rowRange: this.extractMarkdownTableRowRange(part),
+                    parentText: block.text,
                 });
             }
+            cursor = block.startChar + block.text.length;
+        });
+        const afterStart = cursor;
+        const after = text.slice(afterStart).trim();
+        if (after) {
+            candidates.push(...this.createTextCandidates(after, category, config).map(candidate => ({
+                ...candidate,
+                startChar: candidate.startChar + afterStart,
+                endChar: candidate.endChar + afterStart,
+            })));
         }
-        return this.createTextCandidates(text, 'spreadsheet', config);
+        return candidates;
     }
     createDataCandidates(text, config) {
         const sections = text.split(/\n(?=[\w.[\]-]+[:：]\s)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
@@ -331,6 +350,9 @@ export class TextChunker {
         const lines = text.trim().split(/\r?\n/u);
         return lines.length >= 3 && lines.some(line => /^\s*\|?\s*:?-{3,}:?\s*\|/u.test(line));
     }
+    hasMarkdownTableBlock(text) {
+        return /^\s*\|?\s*:?-{3,}:?\s*\|/mu.test(text);
+    }
     extractMarkdownTableBlocks(text) {
         const lines = text.split(/\r?\n/u);
         const lineStarts = [];

package/dist/extraction/content-extractor.d.ts

@@ -41,9 +41,15 @@ export declare class ContentExtractor {
     private extractRtf;
     private extractLegacyWordDocument;
     private extractLegacyOfficeBinary;
+    private extractLegacyOfficeBinaryWithWarnings;
+    private isZipOpenXmlFile;
+    private isOleCompoundFile;
+    private extractOleOfficeDocument;
     private findMergedCellValue;
     private extractSpreadsheet;
     private extractDocxStyleTreeMarkdown;
+    private extractDocxTableRows;
+    private extractDocxCellText;
     private docxHeadingLevel;
     private extractOfficeZip;
     private extractRasterImage;

package/dist/extraction/content-extractor.js

@@ -653,6 +653,10 @@ export class ContentExtractor {
     }
     async extractOfficeDocument(file) {
         const ext = path.extname(file.absolutePath).toLowerCase();
+        const isZip = this.isZipOpenXmlFile(file.absolutePath);
+        const isOle = this.isOleCompoundFile(file.absolutePath);
+        if (isOle)
+            return this.extractOleOfficeDocument(file, ext === '.docx' ? ['文件扩展名为 .docx，但真实格式为旧版 OLE/CFB Office 复合文档，已按旧版 Office 解析'] : []);
         if (ext === '.rtf')
             return this.extractRtf(file);
         if (ext === '.doc')
@@ -660,6 +664,8 @@ export class ContentExtractor {
         if (ext === '.ppt')
             return this.extractLegacyOfficeBinary(file);
         if (ext === '.docx') {
+            if (!isZip)
+                return this.extractLegacyOfficeBinaryWithWarnings(file, ['文件扩展名为 .docx，但未检测到 OpenXML ZIP 文件头，已降级为二进制可读文本抽取']);
             try {
                 const styledMarkdown = await this.extractDocxStyleTreeMarkdown(file.absolutePath);
                 if (styledMarkdown.trim()) {
@@ -732,6 +738,37 @@ export class ContentExtractor {
             warnings: text ? [] : ['旧版 Office 二进制文件未提取到正文，未入库'],
         };
     }
+    extractLegacyOfficeBinaryWithWarnings(file, warnings) {
+        const result = this.extractLegacyOfficeBinary(file);
+        return { ...result, warnings: [...warnings, ...result.warnings] };
+    }
+    isZipOpenXmlFile(filePath) {
+        const header = fs.readFileSync(filePath).subarray(0, 4);
+        return header.length >= 4 && header[0] === 0x50 && header[1] === 0x4b && [0x03, 0x05, 0x07].includes(header[2] ?? -1);
+    }
+    isOleCompoundFile(filePath) {
+        const signature = fs.readFileSync(filePath).subarray(0, 8);
+        return signature.equals(Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]));
+    }
+    async extractOleOfficeDocument(file, warnings = []) {
+        const word = await this.extractLegacyWordDocument(file);
+        if (word.text.trim() && word.metadata.extractionMode === 'builtin_word_extractor') {
+            return {
+                text: word.text,
+                metadata: { ...word.metadata, realOfficeContainer: 'ole_cfb' },
+                warnings: [...warnings, ...word.warnings],
+            };
+        }
+        const spreadsheet = await this.extractSpreadsheet(file);
+        if (spreadsheet.text.trim() && spreadsheet.metadata.extractionMode !== 'office_zip_failed') {
+            return {
+                text: spreadsheet.text,
+                metadata: { ...spreadsheet.metadata, realOfficeContainer: 'ole_cfb' },
+                warnings: [...warnings, ...spreadsheet.warnings],
+            };
+        }
+        return this.extractLegacyOfficeBinaryWithWarnings(file, warnings);
+    }
     findMergedCellValue(sheet, merges, row, col, XLSX) {
         const merge = merges.find(item => row >= item.s.r && row <= item.e.r && col >= item.s.c && col <= item.e.c);
         if (!merge)
@@ -814,6 +851,7 @@ export class ContentExtractor {
         // 在 xml 中 w:body 的一级子节点通常是 w:p 和 w:tbl
         const elements = Array.from(docXml.matchAll(/<(w:p|w:tbl)[\s>][\s\S]*?<\/\1>/gu), match => ({ tag: match[1], xml: match[0] }));
         const lines = [];
+        let tableIndex = 0;
         for (const { tag, xml } of elements) {
             if (tag === 'w:p') {
                 const texts = Array.from(xml.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
@@ -826,34 +864,58 @@ export class ContentExtractor {
                 lines.push(`${level > 0 ? `${'#'.repeat(level)} ` : ''}${texts.trim()}`);
             }
             else if (tag === 'w:tbl') {
-                const trList = Array.from(xml.matchAll(/<w:tr[\s>][\s\S]*?<\/w:tr>/gu), match => match[0]);
-                const rows = [];
-                for (const tr of trList) {
-                    const tcList = Array.from(tr.matchAll(/<w:tc[\s>][\s\S]*?<\/w:tc>/gu), match => match[0]);
-                    const cols = [];
-                    for (const tc of tcList) {
-                        const tcText = Array.from(tc.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
-                        cols.push(tcText.trim().replace(/\r?\n/gu, ' '));
-                    }
-                    rows.push(cols);
-                }
+                tableIndex += 1;
+                const rows = this.extractDocxTableRows(xml);
                 if (rows.length > 0) {
-                    const maxCols = Math.max(...rows.map(r => r.length));
-                    // 补齐列数
-                    rows.forEach(r => { while (r.length < maxCols)
-                        r.push(''); });
+                    const maxCols = Math.max(...rows.map(row => row.length), 1);
+                    rows.forEach(row => { while (row.length < maxCols)
+                        row.push(''); });
                     const header = rows[0] || Array(maxCols).fill('');
-                    const mdTable = [
-                        `| ${header.join(' | ')} |`,
-                        `| ${header.map(() => '---').join(' | ')} |`,
-                        ...rows.slice(1).map(row => `| ${row.join(' | ')} |`)
-                    ].join('\n');
-                    lines.push(mdTable);
+                    const mdTable = this.toMarkdownTable(header, rows.slice(1));
+                    const declarations = rows.slice(1).flatMap((row, rowIndex) => row.map((value, colIndex) => {
+                        const column = header[colIndex] || `COL${colIndex + 1}`;
+                        return `表${tableIndex}.R${rowIndex + 2}C${colIndex + 1} ${column}: ${value}`;
+                    })).filter(line => !line.endsWith(': '));
+                    lines.push([`DOCX 表格 ${tableIndex}`, mdTable, '表格路径声明', ...declarations].join('\n'));
                 }
             }
         }
         return lines.join('\n\n');
     }
+    extractDocxTableRows(tableXml) {
+        const activeVMerges = new Map();
+        return Array.from(tableXml.matchAll(/<w:tr[\s>][\s\S]*?<\/w:tr>/gu), match => match[0]).map(tr => {
+            const row = [];
+            for (const tc of Array.from(tr.matchAll(/<w:tc[\s>][\s\S]*?<\/w:tc>/gu), match => match[0])) {
+                const gridSpan = Math.max(1, Number(/<w:gridSpan\s+w:val="(\d+)"/u.exec(tc)?.[1] ?? 1));
+                const vMerge = /<w:vMerge(?:\s+w:val="([^"]+)")?\s*\/?/u.exec(tc)?.[1] ?? (/<w:vMerge\b/u.test(tc) ? 'continue' : undefined);
+                const cellText = this.extractDocxCellText(tc);
+                const colIndex = row.length;
+                const value = vMerge === 'continue' ? (activeVMerges.get(colIndex) ?? cellText) : cellText;
+                if (vMerge === 'restart' || (vMerge && cellText))
+                    activeVMerges.set(colIndex, cellText);
+                if (!vMerge)
+                    activeVMerges.delete(colIndex);
+                for (let index = 0; index < gridSpan; index++)
+                    row.push(index === 0 ? value : '');
+            }
+            return row;
+        }).filter(row => row.some(Boolean));
+    }
+    extractDocxCellText(cellXml) {
+        return Array.from(cellXml.matchAll(/<w:p[\s>][\s\S]*?<\/w:p>/gu), match => match[0])
+            .map(paragraph => Array.from(paragraph.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>|<w:tab\s*\/>|<w:br\s*\/>/gu), match => {
+            if (match[1] != null)
+                return this.stripXml(match[1]);
+            if (match[0].startsWith('<w:tab'))
+                return ' ';
+            return '\n';
+        }).join('').trim())
+            .filter(Boolean)
+            .join(' / ')
+            .replace(/\s+/gu, ' ')
+            .trim();
+    }
     docxHeadingLevel(style, bold, size, text) {
         const normalized = style.toLowerCase();
         const heading = /heading(\d)|标题(\d)|h(\d)/iu.exec(normalized);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.37",
+  "version": "4.0.38",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",