@customize-agent/knowledge 4.0.10 → 4.0.11

package/dist/chunking/text-chunker.d.ts

@@ -37,6 +37,7 @@ export declare class TextChunker {
     private collectBraceBalancedBlocks;
     private splitCodeByStructuralFallback;
     private splitIntoSections;
+    private mergeSmallSections;
     private mergeLeadingHeader;
     private recursiveSplit;
     private mergeParts;

package/dist/chunking/text-chunker.js

@@ -15,8 +15,8 @@ const RECURSIVE_SEPARATORS = [
     /\n(?=#{1,6}\s)/u,
     /\n{2,}/u,
     /\n(?=(?:第[一二三四五六七八九十百千万\d]+[章节条]|[一二三四五六七八九十]+、|\d+[.)、]))/u,
-    /(?<=[。！？；])\s*/u,
-    /(?<=[，、])\s*/u,
+    /(?<=[。！？；])\s+/u,
+    /(?<=[，、])\s+/u,
     /\s+/u,
 ];
 const LANGUAGE_ROUTER = {
@@ -45,7 +45,8 @@ export class TextChunker {
             return [];
         const config = DEFAULT_CONFIGS[file.category];
         const normalized = this.withHeader(source, file, config);
-        const candidates = this.enforceCandidateLimit(this.createCandidates(normalized, file, config), config);
+        const rawCandidates = this.createCandidates(normalized, file, config);
+        const candidates = this.enforceCandidateLimit(rawCandidates, config);
         return candidates.map((candidate, index) => this.createChunk(index, candidate, file, metadata));
     }
     createCandidates(text, file, config) {
@@ -58,7 +59,7 @@ export class TextChunker {
         return this.createTextCandidates(text, file.category, config);
     }
     createTextCandidates(text, category, config) {
-        const sections = this.splitIntoSections(text, category);
+        const sections = this.mergeSmallSections(this.splitIntoSections(text, category), config);
         const candidates = [];
         const titlePaths = this.buildTitlePaths(sections);
         sections.forEach((section, parentIndex) => {
@@ -215,6 +216,29 @@ export class TextChunker {
             return { text: section, startChar, title: this.extractSectionTitle(section) };
         });
     }
+    mergeSmallSections(sections, config) {
+        const merged = [];
+        let current;
+        const targetTokens = Math.max(80, Math.floor(config.maxChunkSize * 0.75));
+        for (const section of sections) {
+            if (!current) {
+                current = { ...section };
+                continue;
+            }
+            const candidateText = `${current.text}\n\n${section.text}`;
+            const currentTokens = this.estimateTokens(current.text);
+            if (currentTokens < targetTokens && this.estimateTokens(candidateText) <= config.maxChunkSize) {
+                current = { ...current, text: candidateText, title: current.title ?? section.title };
+            }
+            else {
+                merged.push(current);
+                current = { ...section };
+            }
+        }
+        if (current)
+            merged.push(current);
+        return merged;
+    }
     mergeLeadingHeader(sections) {
         if (sections.length < 2)
             return sections;
@@ -237,7 +261,7 @@ export class TextChunker {
         if (!separator)
             return this.splitBySentenceBoundary(text, maxTokens);
         const parts = text.split(separator).map(part => part.trim()).filter(Boolean);
-        if (parts.length <= 1)
+        if (parts.length <= 1 || parts.some(part => part === text))
             return this.recursiveSplit(text, maxTokens, separatorIndex + 1);
         return parts.flatMap(part => this.recursiveSplit(part, maxTokens, separatorIndex + 1));
     }

package/dist/extraction/content-extractor.d.ts

@@ -58,6 +58,7 @@ export declare class ContentExtractor {
     private formatBoundingBox;
     private validateRasterImage;
     private extractPdf;
+    private shouldAugmentPdfWithOcr;
     private extractScannedPdfOcr;
     private extractPdfText;
     private normalizedTextLength;

package/dist/extraction/content-extractor.js

@@ -1063,7 +1063,16 @@ try {
             if (text.trim()) {
                 metadata.contentCoverage = 'pdf_text_streams_layout_markdown';
                 metadata.pdfExtractor = 'pdfjs-dist';
-                return { text: [this.metadataOnlyText(file), this.toMarkdownDocument(text)].join('\n\n'), metadata, warnings };
+                const markdownText = this.toMarkdownDocument(text);
+                if (!this.shouldAugmentPdfWithOcr(markdownText))
+                    return { text: [this.metadataOnlyText(file), markdownText].join('\n\n'), metadata, warnings };
+                const ocr = await this.extractScannedPdfOcr(file);
+                if (ocr.text.trim()) {
+                    metadata.contentCoverage = 'pdf_text_streams_plus_ocr';
+                    metadata.ocrAugmented = true;
+                    return { text: [this.metadataOnlyText(file), markdownText, '## PDF OCR 备用识别文本', ocr.text].join('\n\n'), metadata: { ...metadata, ocr: ocr.metadata }, warnings: [...warnings, ...ocr.warnings] };
+                }
+                return { text: [this.metadataOnlyText(file), markdownText].join('\n\n'), metadata: { ...metadata, ocrRecommended: true, ocrReason: ocr.metadata.ocrReason ?? 'pdf_text_low_quality' }, warnings: [...warnings, ...ocr.warnings] };
             }
         }
         catch (error) {
@@ -1086,6 +1095,17 @@ try {
             warnings: [...warnings, 'PDF 正文暂未提取到文本，已索引文件名、路径和类型元数据', ...ocr.warnings],
         };
     }
+    shouldAugmentPdfWithOcr(text) {
+        const normalizedLength = this.normalizedTextLength(text);
+        if (normalizedLength < 1200)
+            return true;
+        const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
+        if (lines.length === 0)
+            return true;
+        const shortLineRatio = lines.filter(line => line.length <= 12).length / lines.length;
+        const cjkCount = (text.match(/[\p{Script=Han}]/gu) ?? []).length;
+        return shortLineRatio > 0.65 && cjkCount < 1200;
+    }
     async extractScannedPdfOcr(file) {
         const metadata = {
             extractionMode: 'pdf_page_ocr_embedded',
@@ -1201,22 +1221,23 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
         // 第二层：pdf-parse（兼容旧版 PDF），与 pdfjs 结果互补，避免单一解析器漏字
         try {
             const mod = await resolveAndImport('pdf-parse');
-            const pdfParse = mod.default;
+            const pdfParse = typeof mod === 'function'
+                ? mod
+                : mod.default;
             if (pdfParse) {
                 const result = await pdfParse(buffer);
                 const parseText = result.text.trim();
                 if (pdfjsText && parseText && this.normalizedTextLength(parseText) > this.normalizedTextLength(pdfjsText) * 1.08)
                     return [pdfjsText, '## PDF 备用解析文本', parseText].join('\n\n');
-                if (pdfjsText)
-                    return pdfjsText;
-                if (parseText)
+                if (parseText && !pdfjsText)
                     return parseText;
             }
         }
         catch {
-            if (pdfjsText)
-                return pdfjsText;
+            // pdfjs 结果已可用时忽略备用解析器失败
         }
+        if (pdfjsText)
+            return pdfjsText;
         // 第三层：raw regex 回退（未压缩的古老 PDF）
         const raw = buffer.toString('latin1');
         const matches = Array.from(raw.matchAll(/\(([^()]{2,500})\)\s*T[jJ]/gu), match => match[1] ?? '')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.10",
+  "version": "4.0.11",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",