@customize-agent/knowledge 4.0.20 → 4.0.22

package/dist/chunking/text-chunker.d.ts

@@ -54,5 +54,6 @@ export declare class TextChunker {
     private kindForCategory;
     private extractSectionTitle;
     private takeOverlap;
+    private hasLongUnbrokenSegment;
     private estimateTokens;
 }

package/dist/chunking/text-chunker.js

@@ -295,7 +295,7 @@ export class TextChunker {
         return chunks.flatMap(chunk => this.estimateTokens(chunk) > maxTokens ? this.splitByWindow(chunk, maxTokens, overlapTokens) : [chunk]);
     }
     splitByWindow(text, maxTokens, overlapTokens = 0) {
-        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > Math.max(2000, maxTokens * 3)) {
+        if (this.hasLongUnbrokenSegment(text) && text.length > Math.max(2000, maxTokens * 3)) {
             const maxChars = Math.max(1, maxTokens);
             const overlapChars = Math.max(0, Math.min(Math.floor(maxChars / 2), overlapTokens));
             const step = Math.max(1, maxChars - overlapChars);
@@ -452,7 +452,7 @@ export class TextChunker {
                 startChar: candidate.startChar,
                 endChar: candidate.endChar,
                 splitStrategy: 'recursive_parent_child_v2',
-                parentText: candidate.parentText, // <=== 原始父块内容
+                parentText: candidate.childIndex === 0 ? candidate.parentText : undefined,
             },
         };
     }
@@ -481,9 +481,12 @@ export class TextChunker {
         const chars = overlapTokens * 4;
         return text.slice(Math.max(0, text.length - chars));
     }
+    hasLongUnbrokenSegment(text) {
+        return /[^\s\n\r\t。？！；;.!?，、]{2001,}/u.test(text);
+    }
     estimateTokens(text) {
-        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > 2000)
-            return Math.max(1, Math.ceil(text.length / 3));
+        if (this.hasLongUnbrokenSegment(text))
+            return text.length;
         return Math.max(1, this.tokenizer.countTokens(text));
     }
 }

package/dist/extraction/ocr-providers.js

@@ -11,6 +11,13 @@ import { fileURLToPath } from 'node:url';
 import { resolveAndImport, resolvePackage } from './module-resolver.js';
 // ─── 路径工具 ───────────────────────────────────────────────────
 const knowledgeDir = path.dirname(fileURLToPath(import.meta.url));
+const OCR_NATIVE_NOISE_PATTERNS = [/^Image too small to scale!!/u, /^Line cannot be recognized!!$/u];
+const OCR_NOISE_SUPPRESSION_KEY = Symbol.for('customize-agent.ocr-noise-suppression');
+function ocrNoiseSuppressionState() {
+    const globalState = globalThis;
+    globalState[OCR_NOISE_SUPPRESSION_KEY] ??= { depth: 0 };
+    return globalState[OCR_NOISE_SUPPRESSION_KEY];
+}
 function tessdataDir() {
     if (process.env.TESSDATA_PREFIX)
         return process.env.TESSDATA_PREFIX;
@@ -19,6 +26,44 @@ function tessdataDir() {
         return pkg;
     return pkg;
 }
+function isNativeOcrNoise(chunk) {
+    const text = Buffer.isBuffer(chunk) ? chunk.toString('utf8') : chunk instanceof Uint8Array ? Buffer.from(chunk).toString('utf8') : typeof chunk === 'string' ? chunk : '';
+    if (!text)
+        return false;
+    const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
+    return lines.length > 0 && lines.every(line => OCR_NATIVE_NOISE_PATTERNS.some(pattern => pattern.test(line)));
+}
+async function suppressNativeOcrNoise(operation) {
+    const state = ocrNoiseSuppressionState();
+    if (state.depth === 0) {
+        state.stdout = process.stdout.write;
+        state.stderr = process.stderr.write;
+        const filter = (original) => function write(chunk, ...args) {
+            if (isNativeOcrNoise(chunk)) {
+                const callback = args.find((arg) => typeof arg === 'function');
+                if (callback)
+                    process.nextTick(callback);
+                return true;
+            }
+            return original.call(this, chunk, ...args);
+        };
+        process.stdout.write = filter(state.stdout);
+        process.stderr.write = filter(state.stderr);
+    }
+    state.depth += 1;
+    try {
+        return await operation();
+    }
+    finally {
+        state.depth -= 1;
+        if (state.depth === 0 && state.stdout && state.stderr) {
+            process.stdout.write = state.stdout;
+            process.stderr.write = state.stderr;
+            state.stdout = undefined;
+            state.stderr = undefined;
+        }
+    }
+}
 // ─── Tesseract.js Provider（主 OCR 引擎，跨平台） ──────────────
 export class TesseractJsProvider {
     id = 'tesseract.js';
@@ -71,7 +116,7 @@ export class TesseractJsProvider {
         }
         try {
             const worker = await this.getWorker();
-            const result = await worker.recognize(pngPath);
+            const result = await suppressNativeOcrNoise(() => worker.recognize(pngPath));
             const text = (result.data.text ?? '').trim();
             const lines = (result.data.lines ?? []);
             const regions = lines

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.20",
+  "version": "4.0.22",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",