doc2vec 2.9.0 → 2.9.2

package/dist/content-processor.js

@@ -1707,7 +1707,10 @@ class ContentProcessor {
         if (lang) {
             try {
                 const codeChunker = await this.getCodeChunker(lang, sourceConfig.chunk_size);
-                chunks = await codeChunker.chunk(code);
+                chunks = await Promise.race([
+                    codeChunker.chunk(code),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error(`CodeChunker timed out after 30s for ${normalizedPath || url}`)), 30000))
+                ]);
             }
             catch (error) {
                 logger.warn(`CodeChunker failed for ${normalizedPath || url}, falling back to token chunking:`, error);
@@ -1757,9 +1760,9 @@ class ContentProcessor {
     }
     async chunkMarkdown(markdown, sourceConfig, url) {
         const logger = this.logger.child('chunker');
-        // --- Configuration ---
-        const MAX_TOKENS = 1000;
-        const MIN_TOKENS = 150; // 💡 Merges "OpenAI-compatible" sentence into the next block
+        // --- Configuration (character-based, ~4 chars ≈ 1 token) ---
+        const MAX_CHARS = 4000;
+        const MIN_CHARS = 600; // 💡 Merges short sections into the next block
         const OVERLAP_PERCENT = 0.1; // 10% overlap for large splits
         const chunks = [];
         const lines = markdown.split("\n");
@@ -1821,29 +1824,27 @@ class ContentProcessor {
         };
         /**
          * Flushes the current buffer into the chunks array.
-         * Uses sub-splitting logic if the buffer exceeds MAX_TOKENS.
+         * Uses sub-splitting logic if the buffer exceeds MAX_CHARS.
          */
         const flushBuffer = (force = false) => {
             const trimmedBuffer = buffer.trim();
             if (!trimmedBuffer)
                 return;
-            const tokenCount = utils_1.Utils.tokenize(trimmedBuffer).length;
+            const charCount = trimmedBuffer.length;
             // 💡 SEMANTIC MERGING
             // If the current section is too short (like just a title or a one-liner),
             // we don't flush yet unless it's the end of the file (force=true).
-            if (tokenCount < MIN_TOKENS && !force) {
+            if (charCount < MIN_CHARS && !force) {
                 return;
             }
             // Compute the appropriate topic hierarchy for merged content
             const topicHierarchy = computeTopicHierarchy();
-            if (tokenCount > MAX_TOKENS) {
-                // 💡 RECURSIVE OVERLAP SPLITTING
+            if (charCount > MAX_CHARS) {
+                // 💡 OVERLAP SPLITTING
                 // If the section is a massive guide, split it but keep headers on every sub-piece.
-                const tokens = utils_1.Utils.tokenize(trimmedBuffer);
-                const overlapSize = Math.floor(MAX_TOKENS * OVERLAP_PERCENT);
-                for (let i = 0; i < tokens.length; i += (MAX_TOKENS - overlapSize)) {
-                    const subTokens = tokens.slice(i, i + MAX_TOKENS);
-                    const subContent = subTokens.join("");
+                const overlapSize = Math.floor(MAX_CHARS * OVERLAP_PERCENT);
+                for (let i = 0; i < trimmedBuffer.length; i += (MAX_CHARS - overlapSize)) {
+                    const subContent = trimmedBuffer.slice(i, i + MAX_CHARS);
                     chunks.push(createDocumentChunk(subContent, topicHierarchy));
                 }
             }
@@ -1867,9 +1868,9 @@ class ContentProcessor {
                     .replace(/\[\]\(#[^)]*\)/g, "") // Remove [](#anchor) patterns  
                     .trim();
                 // Check if we should merge with previous content
-                const currentTokenCount = utils_1.Utils.tokenize(buffer.trim()).length;
-                const hasBufferContent = currentTokenCount > 0;
-                const bufferIsSmall = currentTokenCount < MIN_TOKENS;
+                const currentCharCount = buffer.trim().length;
+                const hasBufferContent = currentCharCount > 0;
+                const bufferIsSmall = currentCharCount < MIN_CHARS;
                 // Only merge if:
                 // 1. Buffer has content and is small
                 // 2. Buffer has tracked headings (we're merging sections, not just content)
@@ -1895,7 +1896,7 @@ class ContentProcessor {
             else {
                 buffer += `${line}\n`;
                 // Safety valve: if a single section is huge, flush it periodically
-                if (utils_1.Utils.tokenize(buffer).length >= MAX_TOKENS) {
+                if (buffer.length >= MAX_CHARS) {
                     flushBuffer();
                 }
             }

package/dist/doc2vec.js

@@ -1530,7 +1530,7 @@ class Doc2Vec {
             const response = await this.openai.embeddings.create({
                 model: this.embeddingModel,
                 input: safeTexts,
-            });
+            }, { timeout: 60000 });
             logger.debug(`Successfully created ${response.data.length} embeddings`);
             return response.data.map(d => d.embedding);
         }
@@ -1555,5 +1555,7 @@ if (require.main === module) {
         process.exit(1);
     }
     const doc2Vec = new Doc2Vec(configPath);
-    doc2Vec.run().catch(console.error);
+    doc2Vec.run()
+        .then(() => process.exit(0))
+        .catch((err) => { console.error(err); process.exit(1); });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.9.0",
+    "version": "2.9.2",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",