@customize-agent/knowledge 4.0.18 → 4.0.19

package/dist/chunking/text-chunker.js

@@ -295,6 +295,18 @@ export class TextChunker {
         return chunks.flatMap(chunk => this.estimateTokens(chunk) > maxTokens ? this.splitByWindow(chunk, maxTokens, overlapTokens) : [chunk]);
     }
     splitByWindow(text, maxTokens, overlapTokens = 0) {
+        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > maxTokens * 3) {
+            const maxChars = Math.max(1, maxTokens * 3);
+            const overlapChars = Math.max(0, Math.min(Math.floor(maxChars / 2), overlapTokens * 3));
+            const step = Math.max(1, maxChars - overlapChars);
+            const chunks = [];
+            for (let start = 0; start < text.length; start += step) {
+                chunks.push(text.slice(start, start + maxChars).trim());
+                if (start + maxChars >= text.length)
+                    break;
+            }
+            return chunks.filter(Boolean);
+        }
         const tokens = this.tokenizer.encode(text);
         if (tokens.length <= maxTokens)
             return [text.trim()].filter(Boolean);
@@ -470,6 +482,8 @@ export class TextChunker {
         return text.slice(Math.max(0, text.length - chars));
     }
     estimateTokens(text) {
+        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > 2000)
+            return Math.max(1, Math.ceil(text.length / 3));
         return Math.max(1, this.tokenizer.countTokens(text));
     }
 }

package/dist/core/knowledge-base-manager.d.ts

@@ -180,6 +180,7 @@ export declare class KnowledgeBaseManager {
     private statRelativePaths;
     private moveUploadedFile;
     private hasUsableContent;
+    private isMetadataOnlyNonBlocking;
     private defaultUploadRelativePath;
     private validateUploadRelativePath;
     private resolveKbRelativePath;

package/dist/core/knowledge-base-manager.js

@@ -148,8 +148,9 @@ export class KnowledgeBaseManager {
                 extraction.metadata.textLength = extraction.text.length;
                 if (!this.hasUsableContent(extraction.text, extraction.metadata)) {
                     const reason = extraction.warnings[0] ?? '未解析出可用于模型的正文内容，已跳过向量化';
+                    const metadataOnly = this.isMetadataOnlyNonBlocking(file, extraction.metadata);
                     diff.skippedFiles.push({ file, reason });
-                    this.updateJobsForFile(file.relativePath, 'ERROR', 100, reason, reason);
+                    this.updateJobsForFile(file.relativePath, metadataOnly ? 'SUCCESS' : 'ERROR', 100, reason, metadataOnly ? undefined : reason);
                     this.store.upsertRecord({
                         relativePath: file.relativePath,
                         category: file.category,
@@ -161,9 +162,9 @@ export class KnowledgeBaseManager {
                         collectionName,
                         indexedAt: now,
                         lastVerifiedAt: now,
-                        status: 'error',
-                        errorMessage: reason,
-                        metadataJson: JSON.stringify({ mimeType: file.mimeType, warnings: extraction.warnings }),
+                        status: metadataOnly ? 'active' : 'error',
+                        errorMessage: metadataOnly ? undefined : reason,
+                        metadataJson: JSON.stringify({ mimeType: file.mimeType, ...extraction.metadata, warnings: extraction.warnings, metadataOnly }),
                     });
                     continue;
                 }
@@ -973,6 +974,10 @@ export class KnowledgeBaseManager {
             return false;
         return text.trim().length > 0;
     }
+    isMetadataOnlyNonBlocking(file, metadata) {
+        const coverage = String(metadata.contentCoverage ?? '');
+        return file.category === 'image' && ['image_too_small_for_ocr', 'ocr_no_text'].includes(coverage);
+    }
     defaultUploadRelativePath(fileName) {
         const classification = this.classifier.classifyVirtual(fileName);
         const configDirs = this.projectConfig?.categoryDirs ?? DEFAULT_CATEGORY_DIRS;

package/dist/extraction/content-extractor.js

@@ -906,6 +906,15 @@ export class ContentExtractor {
             metadata.parseError = validationError;
             return { text: '', metadata, warnings: [`图片文件无效或不完整：${validationError}，未入库`] };
         }
+        const dimensions = await this.readImageDimensions(file.absolutePath);
+        if (dimensions) {
+            metadata.imageWidth = dimensions.width;
+            metadata.imageHeight = dimensions.height;
+            if (this.isTooSmallForOcr(dimensions.width, dimensions.height)) {
+                metadata.contentCoverage = 'image_too_small_for_ocr';
+                return { text: this.metadataOnlyText(file), metadata, warnings: [`图片尺寸过小（${dimensions.width}x${dimensions.height}），已跳过 OCR 并仅索引元数据`] };
+            }
+        }
         // 1. 尝试外部 PaddleOCR 命令（CUSTOMIZE_PADDLE_OCR_CMD）
         const paddleExternal = await this.tryPaddleOcrLayout(file.absolutePath);
         if (paddleExternal) {
@@ -918,10 +927,10 @@ export class ContentExtractor {
         let imageData;
         try {
             imageData = await this.loadImagePixels(file.absolutePath);
+            metadata.imageWidth = imageData.width;
+            metadata.imageHeight = imageData.height;
             if (this.isTooSmallForOcr(imageData.width, imageData.height)) {
                 metadata.contentCoverage = 'image_too_small_for_ocr';
-                metadata.imageWidth = imageData.width;
-                metadata.imageHeight = imageData.height;
                 return { text: this.metadataOnlyText(file), metadata, warnings: [`图片尺寸过小（${imageData.width}x${imageData.height}），已跳过 OCR 并仅索引元数据`] };
             }
         }
@@ -950,7 +959,7 @@ export class ContentExtractor {
                 };
             }
             metadata.contentCoverage = 'ocr_no_text';
-            warnings.push(`${metadata.ocrProvider} 未识别到文字，未入库`);
+            warnings.push(`${metadata.ocrProvider} 未识别到文字，未生成可检索文本切片`);
             return { text: '', metadata, warnings };
         }
         catch (e) {
@@ -1101,7 +1110,12 @@ export class ContentExtractor {
             const imgPath = pageImages[i];
             try {
                 const dimensions = await this.readImageDimensions(imgPath);
-                if (dimensions && this.isTooSmallForOcr(dimensions.width, dimensions.height)) {
+                if (!dimensions) {
+                    failedPages.push({ page: i + 1, reason: 'image_dimensions_unavailable' });
+                    warnings.push(`PDF 第 ${i + 1} 页渲染图片无法读取尺寸，已跳过 OCR`);
+                    continue;
+                }
+                if (this.isTooSmallForOcr(dimensions.width, dimensions.height)) {
                     failedPages.push({ page: i + 1, reason: `image_too_small_for_ocr_${dimensions.width}x${dimensions.height}` });
                     warnings.push(`PDF 第 ${i + 1} 页渲染图片尺寸过小（${dimensions.width}x${dimensions.height}），已跳过 OCR`);
                     continue;
@@ -1109,7 +1123,7 @@ export class ContentExtractor {
                 const provider = await getOcrProvider();
                 const ocrResult = await provider.recognize({
                     data: new Uint8Array(0),
-                    width: dimensions?.width ?? 0, height: dimensions?.height ?? 0, channels: 0,
+                    width: dimensions.width, height: dimensions.height, channels: 0,
                     filePath: imgPath,
                 });
                 const ocrText = this.cleanOcrText(ocrResult.text);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.18",
+  "version": "4.0.19",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",