@bike4mind/cli 0.2.62 → 0.2.63-fix-chunk-token-limit-overflow.21805

package/dist/{chunk-PJFESKK6.js → chunk-5LZS5CVJ.js}

@@ -2,7 +2,7 @@
 import {
   BadRequestError,
   secureParameters
-} from "./chunk-EO2Y5GFY.js";
+} from "./chunk-EPIYC3LA.js";
 import {
   CompletionApiUsageTransaction,
   GenericCreditDeductTransaction,

package/dist/{chunk-VJLPCIK2.js → chunk-C7N7VYXQ.js}

@@ -3,7 +3,7 @@
 // package.json
 var package_default = {
   name: "@bike4mind/cli",
-  version: "0.2.62",
+  version: "0.2.63-fix-chunk-token-limit-overflow.21805+c654aa55d",
   type: "module",
   description: "Interactive CLI tool for Bike4Mind with ReAct agents",
   license: "UNLICENSED",
@@ -118,11 +118,11 @@ var package_default = {
     zustand: "^4.5.4"
   },
   devDependencies: {
-    "@bike4mind/agents": "0.4.0",
-    "@bike4mind/common": "2.75.0",
-    "@bike4mind/mcp": "1.33.20",
-    "@bike4mind/services": "2.68.2",
-    "@bike4mind/utils": "2.16.1",
+    "@bike4mind/agents": "0.4.1-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/common": "2.75.1-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/mcp": "1.33.21-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/services": "2.68.3-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/utils": "2.16.2-fix-chunk-token-limit-overflow.21805+c654aa55d",
     "@types/better-sqlite3": "^7.6.13",
     "@types/jsonwebtoken": "^9.0.4",
     "@types/node": "^22.9.0",
@@ -139,7 +139,7 @@ var package_default = {
   optionalDependencies: {
     "@vscode/ripgrep": "^1.17.1"
   },
-  gitHead: "faae6cba90cc104961ee4dbef74dbe51d599b9dc"
+  gitHead: "c654aa55d9c000f9b4c8a1cd92c6dbd5e0095ccc"
 };
 
 // src/utils/updateChecker.ts

package/dist/{chunk-EO2Y5GFY.js → chunk-EPIYC3LA.js}

@@ -9747,18 +9747,24 @@ var SmartChunker = class {
     }
     this.logger.updateMetadata({ mimeType });
     this.logger.log(`Chunking file with type: ${mimeType}`);
+    let chunks;
     switch (mimeType) {
       case SupportedFabFileMimeTypes.CSV:
-        return this.chunkCSV(content);
+        chunks = await this.chunkCSV(content);
+        break;
       case SupportedFabFileMimeTypes.PDF:
-        return this.chunkPDF(content);
+        chunks = await this.chunkPDF(content);
+        break;
       case SupportedFabFileMimeTypes.JSON:
-        return this.chunkJSON(content);
+        chunks = await this.chunkJSON(content);
+        break;
       case SupportedFabFileMimeTypes.DOCX:
-        return this.chunkDOCX(content);
+        chunks = await this.chunkDOCX(content);
+        break;
       case SupportedFabFileMimeTypes.XLS:
       case SupportedFabFileMimeTypes.XLSX:
-        return this.chunkExcel(content);
+        chunks = await this.chunkExcel(content);
+        break;
       case SupportedFabFileMimeTypes.PNG:
       case SupportedFabFileMimeTypes.JPG:
       case SupportedFabFileMimeTypes.WEBP:
@@ -9774,14 +9780,17 @@ var SmartChunker = class {
       case SupportedFabFileMimeTypes.RUBY:
       case SupportedFabFileMimeTypes.SH:
       case SupportedFabFileMimeTypes.BASH:
-        return this.chunkText(content.toString());
+        chunks = await this.chunkText(content.toString());
+        break;
       default:
         if (mimeType && mimeType.startsWith("text/")) {
-          return this.chunkText(content.toString());
+          chunks = await this.chunkText(content.toString());
+          break;
         }
         this.logger.error(`Unsupported file type: ${mimeType}`);
         return [];
     }
+    return this.validateAndResplitChunks(chunks);
   }
   // Fetches the content of a file from storage as a Buffer
   async fetchFileContent(file) {
@@ -9995,6 +10004,19 @@ var SmartChunker = class {
         for (const word of words) {
           const wordWithSpace = word + " ";
           const wordTokens = await this.countTokens(wordWithSpace);
+          if (wordTokens > this.chunkTokenLimit) {
+            if (subChunk.trim().length > 0) {
+              chunks.push({
+                text: subChunk.trim(),
+                tokenCount: subChunkTokens
+              });
+              subChunk = "";
+              subChunkTokens = 0;
+            }
+            const wordChunks = await this.splitOversizedSegment(word);
+            chunks.push(...wordChunks);
+            continue;
+          }
           if (subChunkTokens + wordTokens > this.chunkTokenLimit) {
             if (subChunk.trim().length > 0) {
               chunks.push({
@@ -10142,6 +10164,104 @@ var SmartChunker = class {
     console.log("Skipping image chunking as AI models can accept file image urls");
     return chunks;
   }
+  /**
+   * Encode text into token IDs. Uses tiktoken for OpenAI models,
+   * falls back to character-based splitting for VoyageAI/Bedrock.
+   */
+  async encodeTokens(text) {
+    if (isEmbeddingModel(this.model, OpenAIEmbeddingModel)) {
+      await this.initializeEncoder();
+      return Array.from(this.encoder.encode(text));
+    }
+    const charsPerToken = isEmbeddingModel(this.model, VoyageAIEmbeddingModel) ? 3.7 : 4;
+    const groupSize = Math.max(1, Math.round(charsPerToken));
+    const tokens = [];
+    for (let i = 0; i < text.length; i += groupSize) {
+      tokens.push(i);
+    }
+    return tokens;
+  }
+  /**
+   * Decode token IDs back to text. Uses tiktoken for OpenAI models,
+   * falls back to character-based reconstruction for VoyageAI/Bedrock.
+   *
+   * For non-OpenAI models, originalText is REQUIRED — the pseudo-token IDs from
+   * encodeTokens() are character offsets, so decoding reconstructs by slicing
+   * the original string. Returns '' if originalText is omitted for non-OpenAI models.
+   *
+   * Note: splitOversizedSegment() uses character slicing directly for non-OpenAI
+   * models and does not call this method, so this is only used in the OpenAI path today.
+   */
+  async decodeTokens(tokens, originalText) {
+    if (isEmbeddingModel(this.model, OpenAIEmbeddingModel)) {
+      await this.initializeEncoder();
+      const decoded = this.encoder.decode(new Uint32Array(tokens));
+      if (typeof decoded === "string")
+        return decoded;
+      return new TextDecoder().decode(decoded);
+    }
+    if (!originalText)
+      return "";
+    const charsPerToken = isEmbeddingModel(this.model, VoyageAIEmbeddingModel) ? 3.7 : 4;
+    const groupSize = Math.max(1, Math.round(charsPerToken));
+    const startIdx = tokens[0] ?? 0;
+    const endIdx = (tokens[tokens.length - 1] ?? 0) + groupSize;
+    return originalText.slice(startIdx, Math.min(endIdx, originalText.length));
+  }
+  /**
+   * Split an oversized text segment using encode-slice-decode for guaranteed correct splitting.
+   * Works with any model: tiktoken for OpenAI, character-based for others.
+   */
+  async splitOversizedSegment(text) {
+    const chunks = [];
+    if (isEmbeddingModel(this.model, OpenAIEmbeddingModel)) {
+      const encoded = await this.encodeTokens(text);
+      for (let j = 0; j < encoded.length; j += this.chunkTokenLimit) {
+        const segmentTokens = encoded.slice(j, j + this.chunkTokenLimit);
+        const segment = await this.decodeTokens(segmentTokens);
+        if (segment.trim().length > 0) {
+          chunks.push({ text: segment, tokenCount: segmentTokens.length });
+        }
+      }
+    } else {
+      const charsPerToken = isEmbeddingModel(this.model, VoyageAIEmbeddingModel) ? 3.7 : 4;
+      const charsPerChunk = Math.floor(this.chunkTokenLimit * charsPerToken);
+      for (let j = 0; j < text.length; j += charsPerChunk) {
+        const segment = text.slice(j, j + charsPerChunk);
+        if (segment.trim().length > 0) {
+          const tokenCount = await this.countTokens(segment);
+          chunks.push({ text: segment, tokenCount });
+        }
+      }
+    }
+    return chunks;
+  }
+  /**
+   * Post-chunking validation: re-split any chunks that still exceed the token limit.
+   * Bounded to max 3 passes to prevent infinite loops.
+   */
+  async validateAndResplitChunks(chunks) {
+    let result = chunks;
+    for (let pass = 0; pass < 3; pass++) {
+      let allValid = true;
+      const validated = [];
+      for (const chunk of result) {
+        const actualTokens = await this.countTokens(chunk.text);
+        if (actualTokens > this.chunkTokenLimit) {
+          allValid = false;
+          this.logger.warn(`Chunk exceeds limit (${actualTokens} > ${this.chunkTokenLimit}), re-splitting (pass ${pass + 1})`);
+          const resplit = await this.splitOversizedSegment(chunk.text);
+          validated.push(...resplit);
+        } else {
+          validated.push({ ...chunk, tokenCount: actualTokens });
+        }
+      }
+      result = validated;
+      if (allValid)
+        break;
+    }
+    return result.filter((c) => c.text.trim().length > 0);
+  }
   // Counts the number of tokens in the given text using the appropriate tokenization method
   async countTokens(text) {
     if (isEmbeddingModel(this.model, OpenAIEmbeddingModel)) {

package/dist/{chunk-Y7K4HI6L.js → chunk-UZUHPHZC.js}

@@ -7,7 +7,7 @@ import {
   getSettingsMap,
   getSettingsValue,
   secureParameters
-} from "./chunk-EO2Y5GFY.js";
+} from "./chunk-EPIYC3LA.js";
 import {
   KnowledgeType,
   SupportedFabFileMimeTypes

package/dist/{chunk-2J4HB7EB.js → chunk-WBE7SQUB.js}

@@ -6,7 +6,7 @@ import {
   getSettingsByNames,
   obfuscateApiKey,
   secureParameters
-} from "./chunk-EO2Y5GFY.js";
+} from "./chunk-EPIYC3LA.js";
 import {
   ApiKeyType,
   MementoTier,

package/dist/{chunk-MNBT2VFX.js → chunk-YHHCU4PX.js}

@@ -4,7 +4,7 @@ import {
   getOpenWeatherKey,
   getSerperKey,
   getWolframAlphaKey
-} from "./chunk-2J4HB7EB.js";
+} from "./chunk-WBE7SQUB.js";
 import {
   assertPathAllowed,
   isPathAllowed
@@ -20,7 +20,7 @@ import {
   OpenAIBackend,
   OpenAIImageService,
   XAIImageService
-} from "./chunk-EO2Y5GFY.js";
+} from "./chunk-EPIYC3LA.js";
 import {
   Logger
 } from "./chunk-PFBYGCOW.js";

package/dist/commands/doctorCommand.js

@@ -3,7 +3,7 @@ import {
   fetchLatestVersion,
   forceCheckForUpdate,
   package_default
-} from "../chunk-VJLPCIK2.js";
+} from "../chunk-C7N7VYXQ.js";
 
 // src/commands/doctorCommand.ts
 import { execSync } from "child_process";

package/dist/commands/headlessCommand.js

@@ -36,14 +36,14 @@ import {
   isReadOnlyTool,
   loadContextFiles,
   setWebSocketToolExecutor
-} from "../chunk-MNBT2VFX.js";
+} from "../chunk-YHHCU4PX.js";
 import "../chunk-BDQBOLYG.js";
-import "../chunk-2J4HB7EB.js";
+import "../chunk-WBE7SQUB.js";
 import "../chunk-GQGOWACU.js";
 import "../chunk-LTLJRF6I.js";
-import "../chunk-PJFESKK6.js";
-import "../chunk-Y7K4HI6L.js";
-import "../chunk-EO2Y5GFY.js";
+import "../chunk-5LZS5CVJ.js";
+import "../chunk-UZUHPHZC.js";
+import "../chunk-EPIYC3LA.js";
 import "../chunk-PFBYGCOW.js";
 import "../chunk-BPFEGDC7.js";
 import {

package/dist/commands/updateCommand.js

@@ -2,7 +2,7 @@
 import {
   forceCheckForUpdate,
   package_default
-} from "../chunk-VJLPCIK2.js";
+} from "../chunk-C7N7VYXQ.js";
 
 // src/commands/updateCommand.ts
 import { execSync } from "child_process";

package/dist/{create-4Q7IAG7H.js → create-C4VEEEYR.js}

@@ -2,8 +2,8 @@
 import {
   createFabFile,
   createFabFileSchema
-} from "./chunk-Y7K4HI6L.js";
-import "./chunk-EO2Y5GFY.js";
+} from "./chunk-UZUHPHZC.js";
+import "./chunk-EPIYC3LA.js";
 import "./chunk-PFBYGCOW.js";
 import "./chunk-JW3JRHH7.js";
 export {

package/dist/index.js

@@ -48,16 +48,16 @@ import {
   setWebSocketToolExecutor,
   substituteArguments,
   warmFileCache
-} from "./chunk-MNBT2VFX.js";
+} from "./chunk-YHHCU4PX.js";
 import "./chunk-BDQBOLYG.js";
-import "./chunk-2J4HB7EB.js";
+import "./chunk-WBE7SQUB.js";
 import "./chunk-GQGOWACU.js";
 import "./chunk-LTLJRF6I.js";
-import "./chunk-PJFESKK6.js";
-import "./chunk-Y7K4HI6L.js";
+import "./chunk-5LZS5CVJ.js";
+import "./chunk-UZUHPHZC.js";
 import {
   OllamaBackend
-} from "./chunk-EO2Y5GFY.js";
+} from "./chunk-EPIYC3LA.js";
 import "./chunk-PFBYGCOW.js";
 import "./chunk-BPFEGDC7.js";
 import {
@@ -67,7 +67,7 @@ import {
 import {
   checkForUpdate,
   package_default
-} from "./chunk-VJLPCIK2.js";
+} from "./chunk-C7N7VYXQ.js";
 import {
   selectActiveBackgroundAgents,
   useCliStore

package/dist/{mementoService-5VAXTB4G.js → mementoService-N4IM6QAC.js}

@@ -2,8 +2,8 @@
 import {
   findMostSimilarMemento,
   getRelevantMementos
-} from "./chunk-2J4HB7EB.js";
-import "./chunk-EO2Y5GFY.js";
+} from "./chunk-WBE7SQUB.js";
+import "./chunk-EPIYC3LA.js";
 import "./chunk-PFBYGCOW.js";
 import "./chunk-JW3JRHH7.js";
 export {

package/dist/{src-NYW3P73W.js → src-F4KZCAA2.js}

@@ -146,7 +146,7 @@ import {
   validateUrlForFetch,
   warmUpSettingsCache,
   withRetry
-} from "./chunk-EO2Y5GFY.js";
+} from "./chunk-EPIYC3LA.js";
 import {
   Logger,
   NotificationDeduplicator,

package/dist/{subtractCredits-ZDMCQ6R5.js → subtractCredits-D4KEM6VU.js}

@@ -2,8 +2,8 @@
 import {
   SubtractCreditsSchema,
   subtractCredits
-} from "./chunk-PJFESKK6.js";
-import "./chunk-EO2Y5GFY.js";
+} from "./chunk-5LZS5CVJ.js";
+import "./chunk-EPIYC3LA.js";
 import "./chunk-PFBYGCOW.js";
 import "./chunk-JW3JRHH7.js";
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bike4mind/cli",
-  "version": "0.2.62",
+  "version": "0.2.63-fix-chunk-token-limit-overflow.21805+c654aa55d",
   "type": "module",
   "description": "Interactive CLI tool for Bike4Mind with ReAct agents",
   "license": "UNLICENSED",
@@ -115,11 +115,11 @@
     "zustand": "^4.5.4"
   },
   "devDependencies": {
-    "@bike4mind/agents": "0.4.0",
-    "@bike4mind/common": "2.75.0",
-    "@bike4mind/mcp": "1.33.20",
-    "@bike4mind/services": "2.68.2",
-    "@bike4mind/utils": "2.16.1",
+    "@bike4mind/agents": "0.4.1-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/common": "2.75.1-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/mcp": "1.33.21-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/services": "2.68.3-fix-chunk-token-limit-overflow.21805+c654aa55d",
+    "@bike4mind/utils": "2.16.2-fix-chunk-token-limit-overflow.21805+c654aa55d",
     "@types/better-sqlite3": "^7.6.13",
     "@types/jsonwebtoken": "^9.0.4",
     "@types/node": "^22.9.0",
@@ -136,5 +136,5 @@
   "optionalDependencies": {
     "@vscode/ripgrep": "^1.17.1"
   },
-  "gitHead": "faae6cba90cc104961ee4dbef74dbe51d599b9dc"
+  "gitHead": "c654aa55d9c000f9b4c8a1cd92c6dbd5e0095ccc"
 }