doc2vec 2.9.1 → 2.9.2

package/dist/content-processor.js

@@ -1760,9 +1760,9 @@ class ContentProcessor {
     }
     async chunkMarkdown(markdown, sourceConfig, url) {
         const logger = this.logger.child('chunker');
-        // --- Configuration ---
-        const MAX_TOKENS = 1000;
-        const MIN_TOKENS = 150; // 💡 Merges "OpenAI-compatible" sentence into the next block
+        // --- Configuration (character-based, ~4 chars ≈ 1 token) ---
+        const MAX_CHARS = 4000;
+        const MIN_CHARS = 600; // 💡 Merges short sections into the next block
         const OVERLAP_PERCENT = 0.1; // 10% overlap for large splits
         const chunks = [];
         const lines = markdown.split("\n");
@@ -1824,29 +1824,27 @@ class ContentProcessor {
         };
         /**
          * Flushes the current buffer into the chunks array.
-         * Uses sub-splitting logic if the buffer exceeds MAX_TOKENS.
+         * Uses sub-splitting logic if the buffer exceeds MAX_CHARS.
          */
         const flushBuffer = (force = false) => {
             const trimmedBuffer = buffer.trim();
             if (!trimmedBuffer)
                 return;
-            const tokenCount = utils_1.Utils.tokenize(trimmedBuffer).length;
+            const charCount = trimmedBuffer.length;
             // 💡 SEMANTIC MERGING
             // If the current section is too short (like just a title or a one-liner),
             // we don't flush yet unless it's the end of the file (force=true).
-            if (tokenCount < MIN_TOKENS && !force) {
+            if (charCount < MIN_CHARS && !force) {
                 return;
             }
             // Compute the appropriate topic hierarchy for merged content
             const topicHierarchy = computeTopicHierarchy();
-            if (tokenCount > MAX_TOKENS) {
-                // 💡 RECURSIVE OVERLAP SPLITTING
+            if (charCount > MAX_CHARS) {
+                // 💡 OVERLAP SPLITTING
                 // If the section is a massive guide, split it but keep headers on every sub-piece.
-                const tokens = utils_1.Utils.tokenize(trimmedBuffer);
-                const overlapSize = Math.floor(MAX_TOKENS * OVERLAP_PERCENT);
-                for (let i = 0; i < tokens.length; i += (MAX_TOKENS - overlapSize)) {
-                    const subTokens = tokens.slice(i, i + MAX_TOKENS);
-                    const subContent = subTokens.join("");
+                const overlapSize = Math.floor(MAX_CHARS * OVERLAP_PERCENT);
+                for (let i = 0; i < trimmedBuffer.length; i += (MAX_CHARS - overlapSize)) {
+                    const subContent = trimmedBuffer.slice(i, i + MAX_CHARS);
                     chunks.push(createDocumentChunk(subContent, topicHierarchy));
                 }
             }
@@ -1870,9 +1868,9 @@ class ContentProcessor {
                     .replace(/\[\]\(#[^)]*\)/g, "") // Remove [](#anchor) patterns  
                     .trim();
                 // Check if we should merge with previous content
-                const currentTokenCount = utils_1.Utils.tokenize(buffer.trim()).length;
-                const hasBufferContent = currentTokenCount > 0;
-                const bufferIsSmall = currentTokenCount < MIN_TOKENS;
+                const currentCharCount = buffer.trim().length;
+                const hasBufferContent = currentCharCount > 0;
+                const bufferIsSmall = currentCharCount < MIN_CHARS;
                 // Only merge if:
                 // 1. Buffer has content and is small
                 // 2. Buffer has tracked headings (we're merging sections, not just content)
@@ -1898,7 +1896,7 @@ class ContentProcessor {
             else {
                 buffer += `${line}\n`;
                 // Safety valve: if a single section is huge, flush it periodically
-                if (utils_1.Utils.tokenize(buffer).length >= MAX_TOKENS) {
+                if (buffer.length >= MAX_CHARS) {
                     flushBuffer();
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.9.1",
+    "version": "2.9.2",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",