@compilr-dev/agents 0.3.11 → 0.3.13

package/dist/utils/tokenizer.js

@@ -0,0 +1,93 @@
+/**
+ * Token counting utility using tiktoken (cl100k_base encoding)
+ *
+ * Replaces the rough `Math.ceil(charCount / 4)` heuristic with actual
+ * BPE tokenization for accurate context window management.
+ */
+import { getEncoding } from 'js-tiktoken';
+// Lazy-initialized encoder (loaded on first use)
+let encoder = null;
+function getEncoder() {
+    if (!encoder) {
+        encoder = getEncoding('cl100k_base');
+    }
+    return encoder;
+}
+/**
+ * Maximum text length for tiktoken encoding.
+ * Beyond this, fall back to chars/4 heuristic to avoid pathological BPE performance.
+ */
+const MAX_TIKTOKEN_LENGTH = 10000;
+/**
+ * Minimum length at which we check for low-entropy (repetitive) content.
+ * Repetitive single-character strings like 'a'.repeat(5000) cause O(n²+)
+ * performance in js-tiktoken's BPE merge loop.
+ */
+const ENTROPY_CHECK_THRESHOLD = 2000;
+/**
+ * Check if a string is low-entropy (highly repetitive), which causes
+ * pathological BPE performance in js-tiktoken.
+ *
+ * Samples characters from the string and checks unique character ratio.
+ * A ratio below 0.01 (e.g., 'aaaa...') triggers the heuristic fallback.
+ */
+function isLowEntropy(text) {
+    // Sample up to 200 characters evenly across the string
+    const sampleSize = Math.min(200, text.length);
+    const step = Math.max(1, Math.floor(text.length / sampleSize));
+    const seen = new Set();
+    for (let i = 0; i < text.length && seen.size < 20; i += step) {
+        seen.add(text[i]);
+    }
+    // If unique chars / length ratio is very low, it's repetitive
+    return seen.size / Math.min(sampleSize, text.length) < 0.02;
+}
+/**
+ * Count tokens in a text string using tiktoken.
+ * Falls back to chars/4 heuristic for very large or pathologically repetitive strings.
+ */
+export function countTokens(text) {
+    if (!text)
+        return 0;
+    if (text.length > MAX_TIKTOKEN_LENGTH) {
+        return Math.ceil(text.length / 4);
+    }
+    // Detect low-entropy strings that cause O(n²+) BPE performance
+    if (text.length >= ENTROPY_CHECK_THRESHOLD && isLowEntropy(text)) {
+        return Math.ceil(text.length / 4);
+    }
+    return getEncoder().encode(text).length;
+}
+/**
+ * Count tokens across an array of messages
+ *
+ * Extracts all text content from messages (text blocks, tool inputs/results,
+ * thinking blocks) and counts tokens using tiktoken.
+ */
+export function countMessageTokens(messages) {
+    const parts = [];
+    for (const msg of messages) {
+        if (typeof msg.content === 'string') {
+            parts.push(msg.content);
+        }
+        else {
+            for (const block of msg.content) {
+                switch (block.type) {
+                    case 'text':
+                        parts.push(block.text);
+                        break;
+                    case 'tool_use':
+                        parts.push(JSON.stringify(block.input));
+                        break;
+                    case 'tool_result':
+                        parts.push(typeof block.content === 'string' ? block.content : JSON.stringify(block.content));
+                        break;
+                    case 'thinking':
+                        parts.push(block.thinking);
+                        break;
+                }
+            }
+        }
+    }
+    return countTokens(parts.join(' '));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@compilr-dev/agents",
-  "version": "0.3.11",
+  "version": "0.3.13",
   "description": "Lightweight multi-LLM agent library for building CLI AI assistants",
   "type": "module",
   "main": "dist/index.js",
@@ -77,7 +77,8 @@
     "vitest": "^4.0.18"
   },
   "dependencies": {
-    "@google/genai": "^1.38.0"
+    "@google/genai": "^1.38.0",
+    "js-tiktoken": "^1.0.21"
   },
   "overrides": {
     "hono": "^4.11.7"