@customize-agent/knowledge 4.0.17 → 4.0.19

package/dist/chunking/text-chunker.js

@@ -295,6 +295,18 @@ export class TextChunker {
         return chunks.flatMap(chunk => this.estimateTokens(chunk) > maxTokens ? this.splitByWindow(chunk, maxTokens, overlapTokens) : [chunk]);
     }
     splitByWindow(text, maxTokens, overlapTokens = 0) {
+        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > maxTokens * 3) {
+            const maxChars = Math.max(1, maxTokens * 3);
+            const overlapChars = Math.max(0, Math.min(Math.floor(maxChars / 2), overlapTokens * 3));
+            const step = Math.max(1, maxChars - overlapChars);
+            const chunks = [];
+            for (let start = 0; start < text.length; start += step) {
+                chunks.push(text.slice(start, start + maxChars).trim());
+                if (start + maxChars >= text.length)
+                    break;
+            }
+            return chunks.filter(Boolean);
+        }
         const tokens = this.tokenizer.encode(text);
         if (tokens.length <= maxTokens)
             return [text.trim()].filter(Boolean);
@@ -470,6 +482,8 @@ export class TextChunker {
         return text.slice(Math.max(0, text.length - chars));
     }
     estimateTokens(text) {
+        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > 2000)
+            return Math.max(1, Math.ceil(text.length / 3));
         return Math.max(1, this.tokenizer.countTokens(text));
     }
 }

package/dist/core/knowledge-base-manager.d.ts

@@ -180,6 +180,7 @@ export declare class KnowledgeBaseManager {
     private statRelativePaths;
     private moveUploadedFile;
     private hasUsableContent;
+    private isMetadataOnlyNonBlocking;
     private defaultUploadRelativePath;
     private validateUploadRelativePath;
     private resolveKbRelativePath;

package/dist/core/knowledge-base-manager.js

@@ -148,8 +148,9 @@ export class KnowledgeBaseManager {
                 extraction.metadata.textLength = extraction.text.length;
                 if (!this.hasUsableContent(extraction.text, extraction.metadata)) {
                     const reason = extraction.warnings[0] ?? '未解析出可用于模型的正文内容，已跳过向量化';
+                    const metadataOnly = this.isMetadataOnlyNonBlocking(file, extraction.metadata);
                     diff.skippedFiles.push({ file, reason });
-                    this.updateJobsForFile(file.relativePath, 'ERROR', 100, reason, reason);
+                    this.updateJobsForFile(file.relativePath, metadataOnly ? 'SUCCESS' : 'ERROR', 100, reason, metadataOnly ? undefined : reason);
                     this.store.upsertRecord({
                         relativePath: file.relativePath,
                         category: file.category,
@@ -161,9 +162,9 @@ export class KnowledgeBaseManager {
                         collectionName,
                         indexedAt: now,
                         lastVerifiedAt: now,
-                        status: 'error',
-                        errorMessage: reason,
-                        metadataJson: JSON.stringify({ mimeType: file.mimeType, warnings: extraction.warnings }),
+                        status: metadataOnly ? 'active' : 'error',
+                        errorMessage: metadataOnly ? undefined : reason,
+                        metadataJson: JSON.stringify({ mimeType: file.mimeType, ...extraction.metadata, warnings: extraction.warnings, metadataOnly }),
                     });
                     continue;
                 }
@@ -973,6 +974,10 @@ export class KnowledgeBaseManager {
             return false;
         return text.trim().length > 0;
     }
+    isMetadataOnlyNonBlocking(file, metadata) {
+        const coverage = String(metadata.contentCoverage ?? '');
+        return file.category === 'image' && ['image_too_small_for_ocr', 'ocr_no_text'].includes(coverage);
+    }
     defaultUploadRelativePath(fileName) {
         const classification = this.classifier.classifyVirtual(fileName);
         const configDirs = this.projectConfig?.categoryDirs ?? DEFAULT_CATEGORY_DIRS;

package/dist/extraction/content-extractor.d.ts

@@ -60,6 +60,8 @@ export declare class ContentExtractor {
     private cleanOcrText;
     /** 加载图片像素数据（依赖 sharp） */
     private loadImagePixels;
+    private readImageDimensions;
+    private isTooSmallForOcr;
     private extractPdfText;
     private normalizedTextLength;
     private toPdfTextItem;

package/dist/extraction/content-extractor.js

@@ -906,6 +906,15 @@ export class ContentExtractor {
             metadata.parseError = validationError;
             return { text: '', metadata, warnings: [`图片文件无效或不完整：${validationError}，未入库`] };
         }
+        const dimensions = await this.readImageDimensions(file.absolutePath);
+        if (dimensions) {
+            metadata.imageWidth = dimensions.width;
+            metadata.imageHeight = dimensions.height;
+            if (this.isTooSmallForOcr(dimensions.width, dimensions.height)) {
+                metadata.contentCoverage = 'image_too_small_for_ocr';
+                return { text: this.metadataOnlyText(file), metadata, warnings: [`图片尺寸过小（${dimensions.width}x${dimensions.height}），已跳过 OCR 并仅索引元数据`] };
+            }
+        }
         // 1. 尝试外部 PaddleOCR 命令（CUSTOMIZE_PADDLE_OCR_CMD）
         const paddleExternal = await this.tryPaddleOcrLayout(file.absolutePath);
         if (paddleExternal) {
@@ -918,6 +927,12 @@ export class ContentExtractor {
         let imageData;
         try {
             imageData = await this.loadImagePixels(file.absolutePath);
+            metadata.imageWidth = imageData.width;
+            metadata.imageHeight = imageData.height;
+            if (this.isTooSmallForOcr(imageData.width, imageData.height)) {
+                metadata.contentCoverage = 'image_too_small_for_ocr';
+                return { text: this.metadataOnlyText(file), metadata, warnings: [`图片尺寸过小（${imageData.width}x${imageData.height}），已跳过 OCR 并仅索引元数据`] };
+            }
         }
         catch (e) {
             metadata.contentCoverage = 'image_decode_failed';
@@ -944,7 +959,7 @@ export class ContentExtractor {
                 };
             }
             metadata.contentCoverage = 'ocr_no_text';
-            warnings.push(`${metadata.ocrProvider} 未识别到文字，未入库`);
+            warnings.push(`${metadata.ocrProvider} 未识别到文字，未生成可检索文本切片`);
             return { text: '', metadata, warnings };
         }
         catch (e) {
@@ -1094,10 +1109,21 @@ export class ContentExtractor {
         for (let i = 0; i < pageImages.length; i++) {
             const imgPath = pageImages[i];
             try {
+                const dimensions = await this.readImageDimensions(imgPath);
+                if (!dimensions) {
+                    failedPages.push({ page: i + 1, reason: 'image_dimensions_unavailable' });
+                    warnings.push(`PDF 第 ${i + 1} 页渲染图片无法读取尺寸，已跳过 OCR`);
+                    continue;
+                }
+                if (this.isTooSmallForOcr(dimensions.width, dimensions.height)) {
+                    failedPages.push({ page: i + 1, reason: `image_too_small_for_ocr_${dimensions.width}x${dimensions.height}` });
+                    warnings.push(`PDF 第 ${i + 1} 页渲染图片尺寸过小（${dimensions.width}x${dimensions.height}），已跳过 OCR`);
+                    continue;
+                }
                 const provider = await getOcrProvider();
                 const ocrResult = await provider.recognize({
                     data: new Uint8Array(0),
-                    width: 0, height: 0, channels: 0,
+                    width: dimensions.width, height: dimensions.height, channels: 0,
                     filePath: imgPath,
                 });
                 const ocrText = this.cleanOcrText(ocrResult.text);
@@ -1219,6 +1245,22 @@ export class ContentExtractor {
         const { data, info } = await sharpFn(filePath).raw().toBuffer({ resolveWithObject: true });
         return { data: new Uint8Array(data), width: info.width, height: info.height };
     }
+    async readImageDimensions(filePath) {
+        try {
+            const sharpMod = await resolveAndImport('sharp');
+            const sharpFn = sharpMod.default ?? sharpMod;
+            const metadata = await sharpFn(filePath).metadata();
+            const width = Number(metadata.width ?? 0);
+            const height = Number(metadata.height ?? 0);
+            return width > 0 && height > 0 ? { width, height } : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    isTooSmallForOcr(width, height) {
+        return width < 8 || height < 8 || width * height < 128;
+    }
     async extractPdfText(buffer) {
         let pdfjsText = '';
         // 第一层：pdfjs-dist 文本提取（处理压缩内容流、CJK 字体、现代 PDF）

package/dist/extraction/ocr-providers.d.ts

@@ -48,6 +48,8 @@ export declare class TesseractJsProvider implements OcrProvider {
         filePath?: string;
     }): Promise<OcrResult>;
     getWarnings(): string[];
+    private readImageDimensions;
+    private isTooSmallForOcr;
     private getWorker;
     private createReusableWorker;
     dispose(): Promise<void>;

package/dist/extraction/ocr-providers.js

@@ -44,9 +44,14 @@ export class TesseractJsProvider {
     async recognize(input) {
         let pngPath;
         let tmpDir = null;
+        let width = input.width;
+        let height = input.height;
         // 如果传了 filePath，直接使用；否则 raw pixels → PNG
         if (input.filePath && fs.existsSync(input.filePath)) {
             pngPath = input.filePath;
+            const dimensions = await this.readImageDimensions(pngPath);
+            width = dimensions?.width ?? width;
+            height = dimensions?.height ?? height;
         }
         else {
             const sharpMod = await resolveAndImport('sharp');
@@ -61,6 +66,9 @@ export class TesseractJsProvider {
                 .withMetadata({ density: 288 })
                 .png().toFile(pngPath);
         }
+        if (this.isTooSmallForOcr(width, height)) {
+            return { text: '', confidence: 0, regions: [], warnings: [`image too small for OCR: ${width}x${height}`] };
+        }
         try {
             const worker = await this.getWorker();
             const result = await worker.recognize(pngPath);
@@ -88,6 +96,22 @@ export class TesseractJsProvider {
     getWarnings() {
         return [...new Set(this.warnings)].slice(-20);
     }
+    async readImageDimensions(filePath) {
+        try {
+            const sharpMod = await resolveAndImport('sharp');
+            const sharpFn = sharpMod.default ?? sharpMod;
+            const metadata = await sharpFn(filePath).metadata();
+            const width = Number(metadata.width ?? 0);
+            const height = Number(metadata.height ?? 0);
+            return width > 0 && height > 0 ? { width, height } : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    isTooSmallForOcr(width, height) {
+        return width < 8 || height < 8 || width * height < 128;
+    }
     async getWorker() {
         if (this.worker)
             return this.worker;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.17",
+  "version": "4.0.19",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",