@customize-agent/knowledge 4.0.19 → 4.0.21

package/dist/chunking/text-chunker.d.ts

@@ -54,5 +54,6 @@ export declare class TextChunker {
     private kindForCategory;
     private extractSectionTitle;
     private takeOverlap;
+    private hasLongUnbrokenSegment;
     private estimateTokens;
 }

package/dist/chunking/text-chunker.js

@@ -295,9 +295,9 @@ export class TextChunker {
         return chunks.flatMap(chunk => this.estimateTokens(chunk) > maxTokens ? this.splitByWindow(chunk, maxTokens, overlapTokens) : [chunk]);
     }
     splitByWindow(text, maxTokens, overlapTokens = 0) {
-        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > maxTokens * 3) {
-            const maxChars = Math.max(1, maxTokens * 3);
-            const overlapChars = Math.max(0, Math.min(Math.floor(maxChars / 2), overlapTokens * 3));
+        if (this.hasLongUnbrokenSegment(text) && text.length > Math.max(2000, maxTokens * 3)) {
+            const maxChars = Math.max(1, maxTokens);
+            const overlapChars = Math.max(0, Math.min(Math.floor(maxChars / 2), overlapTokens));
             const step = Math.max(1, maxChars - overlapChars);
             const chunks = [];
             for (let start = 0; start < text.length; start += step) {
@@ -452,7 +452,7 @@ export class TextChunker {
                 startChar: candidate.startChar,
                 endChar: candidate.endChar,
                 splitStrategy: 'recursive_parent_child_v2',
-                parentText: candidate.parentText, // <=== 原始父块内容
+                parentText: candidate.childIndex === 0 ? candidate.parentText : undefined,
             },
         };
     }
@@ -481,9 +481,12 @@ export class TextChunker {
         const chars = overlapTokens * 4;
         return text.slice(Math.max(0, text.length - chars));
     }
+    hasLongUnbrokenSegment(text) {
+        return /[^\s\n\r\t。？！；;.!?，、]{2001,}/u.test(text);
+    }
     estimateTokens(text) {
-        if (!/[\s\n\r\t。？！；;.!?，、]/u.test(text) && text.length > 2000)
-            return Math.max(1, Math.ceil(text.length / 3));
+        if (this.hasLongUnbrokenSegment(text))
+            return text.length;
         return Math.max(1, this.tokenizer.countTokens(text));
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.19",
+  "version": "4.0.21",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",