@softerist/heuristic-mcp 3.0.17 → 3.1.0

package/lib/settings-editor.js

@@ -734,21 +734,28 @@ export function upsertMcpServerEntryInToml(text, serverName, serverConfig) {
   return `${withTrailingNewline}${newline}${section}${newline}`;
 }
 
-export function setMcpServerDisabledInToml(text, serverName, disabled) {
-  const source = String(text || '');
-  const sectionName = `mcp_servers.${serverName}`;
-  const range = findTomlSectionRange(source, sectionName);
-
-  if (!range) {
-    return source;
-  }
-
-  const sectionBlock = source.slice(range.start, range.end);
-  const newline = detectNewline(sectionBlock || '\n');
-  const disabledLine = `disabled = ${disabled ? 'true' : 'false'}`;
-  const updatedSection = /^\s*disabled\s*=.*$/m.test(sectionBlock)
-    ? sectionBlock.replace(/^\s*disabled\s*=.*$/m, disabledLine)
-    : `${sectionBlock.trimEnd()}${newline}${disabledLine}${newline}`;
-
-  return `${source.slice(0, range.start)}${updatedSection}${source.slice(range.end)}`;
-}
+export function setMcpServerDisabledInToml(text, serverName, disabled) {
+  const source = String(text || '');
+  const sectionName = `mcp_servers.${serverName}`;
+  const range = findTomlSectionRange(source, sectionName);
+
+  if (!range) {
+    return source;
+  }
+
+  const sectionBlock = source.slice(range.start, range.end);
+  const newline = detectNewline(sectionBlock || '\n');
+  if (disabled) {
+    const disabledLine = `disabled = true`;
+    const updatedSection = /^\s*disabled\s*=.*$/m.test(sectionBlock)
+      ? sectionBlock.replace(/^\s*disabled\s*=.*$/m, disabledLine)
+      : `${sectionBlock.trimEnd()}${newline}${disabledLine}${newline}`;
+
+    return `${source.slice(0, range.start)}${updatedSection}${source.slice(range.end)}`;
+  }
+
+  const cleanedSection = sectionBlock
+    .replace(/^\s*disabled\s*=.*$/m, '')
+    .replace(/\n\s*\n$/, '\n');
+  return `${source.slice(0, range.start)}${cleanedSection}${source.slice(range.end)}`;
+}

package/lib/slice-normalize.js

@@ -1,19 +1,13 @@
-/**
- * Slice and L2-normalize a vector for MRL (Matryoshka Representation Learning).
- * If targetDim is null/undefined or >= vector length, returns the original vector unchanged.
- * @param {Float32Array} vector - The full embedding vector
- * @param {number|null} targetDim - Target dimension (64/128/256/512/768 or null)
- * @returns {Float32Array} - Sliced and normalized vector, or original if no slicing
- */
+
 export function sliceAndNormalize(vector, targetDim) {
   if (!targetDim || targetDim >= vector.length) {
     return vector;
   }
 
-  // Slice to target dimension
+  
   const sliced = vector.slice(0, targetDim);
 
-  // L2 normalize the sliced vector
+  
   let sumSquares = 0;
   for (let i = 0; i < targetDim; i++) {
     sumSquares += sliced[i] * sliced[i];
@@ -29,13 +23,9 @@ export function sliceAndNormalize(vector, targetDim) {
   return sliced;
 }
 
-/**
- * Convert any array-like to Float32Array (always creates a copy).
- * @param {ArrayLike<number>} vector - Input vector
- * @returns {Float32Array} - Copy as Float32Array
- */
+
 export function toFloat32Array(vector) {
-  // Always create a copy to ensure we have a unique buffer
-  // and avoid issues with reusable WASM memory views
+  
+  
   return new Float32Array(vector);
 }

package/lib/tokenizer.js

@@ -1,24 +1,13 @@
-/**
- * Token estimation and limits for embedding models
- *
- * Performance:
- * - O(1) model lookups with precomputed maps
- * - Zero regex / Zero allocations in hot loop
- * - Proper LRU cache eviction
- * - Optimized Unicode whitespace detection (ordered by probability)
- * - Eliminated double toLowerCase() calls
- * - Type-safe guard rails on all public APIs
- * - Branchless special character counting
- */
+
 
 const IS_TEST_ENV = process.env.VITEST === 'true' || process.env.NODE_ENV === 'test';
 
 const MODEL_TOKEN_LIMITS_RAW = {
-  // NOTE: While jina-embeddings-v2-base-code supports 8192 tokens, ONNX runtime
-  // allocates O(n²) memory for attention. Using 512 tokens for optimal speed
-  // with 4 ONNX threads (~1.5GB RAM, fastest inference).
+  
+  
+  
   'jinaai/jina-embeddings-v2-base-code': 512,
-  default: 512, // Safe default for BERT-like models
+  default: 512, 
 };
 
 export const MODEL_TOKEN_LIMITS = IS_TEST_ENV
@@ -27,34 +16,26 @@ export const MODEL_TOKEN_LIMITS = IS_TEST_ENV
 
 const DEFAULT_LIMIT = MODEL_TOKEN_LIMITS.default ?? 512;
 
-/**
- * Precomputed case-insensitive lookup
- */
+
 const MODEL_LIMITS_LC = new Map();
 for (const [k, v] of Object.entries(MODEL_TOKEN_LIMITS)) {
   MODEL_LIMITS_LC.set(k.toLowerCase(), v);
 }
 
-/**
- * Internal helper: get model limit from pre-normalized key
- * Avoids double toLowerCase() when called from cache flow
- * @param {string} lowerName - Pre-normalized lowercase model name
- * @param {*} originalName - Original model name (may not be a string)
- * @returns {number} Token limit
- */
+
 function getModelTokenLimitFromLower(lowerName, originalName) {
-  // Fast path: try exact match first (only if original is a string)
+  
   if (typeof originalName === 'string') {
     const direct = MODEL_TOKEN_LIMITS[originalName];
     if (direct !== undefined) return direct;
   }
 
-  // Slow path: use pre-normalized key
+  
   const exact = MODEL_LIMITS_LC.get(lowerName);
   if (exact !== undefined) return exact;
 
-  // Heuristics for common models (use conservative limits for ONNX speed)
-  // 512 tokens = fastest, 1024 = 4x more compute due to O(n²) attention
+  
+  
   if (
     lowerName.includes('jina') ||
     lowerName.includes('nomic') ||
@@ -72,13 +53,9 @@ function getModelTokenLimitFromLower(lowerName, originalName) {
   return DEFAULT_LIMIT;
 }
 
-/**
- * Get the maximum token limit for a given model
- * @param {string} modelName - The model name
- * @returns {number} Maximum tokens supported by the model
- */
+
 export function getModelTokenLimit(modelName) {
-  // Guard clause for non-string or empty inputs
+  
   if (typeof modelName !== 'string' || modelName.length === 0) return DEFAULT_LIMIT;
 
   const direct = MODEL_TOKEN_LIMITS[modelName];
@@ -87,22 +64,15 @@ export function getModelTokenLimit(modelName) {
   const lower = modelName.toLowerCase();
   return getModelTokenLimitFromLower(lower, modelName);
 }
-/**
- * LRU cache for chunking parameters
- * @type {Map<string, {maxTokens: number, targetTokens: number, overlapTokens: number}>}
- */
+
 import { CHUNKING_PARAMS_CACHE_SIZE as MAX_CACHE_SIZE } from './constants.js';
 const chunkingParamsCache = new Map();
 
-/**
- * Get chunking parameters for a model
- * @param {string} modelName - The model name
- * @returns {{maxTokens: number, targetTokens: number, overlapTokens: number}}
- */
+
 export function getChunkingParams(modelName) {
   const key = typeof modelName === 'string' && modelName.length ? modelName.toLowerCase() : '';
 
-  // Fast path for invalid inputs: don't consume cache slots
+  
   if (key === '') {
     const maxTokens = DEFAULT_LIMIT;
     const targetTokens = Math.trunc(maxTokens * 0.85);
@@ -110,10 +80,10 @@ export function getChunkingParams(modelName) {
     return { maxTokens, targetTokens, overlapTokens };
   }
 
-  // LRU pattern: delete-and-reinsert to mark as most recently used.
-  // Note: This creates minor GC pressure due to Map key reallocation, but is
-  // acceptable for MAX_CACHE_SIZE=100. For larger caches (1000+), consider
-  // a doubly-linked-list LRU implementation for O(1) access without reallocation.
+  
+  
+  
+  
   const cached = chunkingParamsCache.get(key);
   if (cached) {
     chunkingParamsCache.delete(key);
@@ -121,14 +91,14 @@ export function getChunkingParams(modelName) {
     return cached;
   }
 
-  // Cache miss: compute new params (avoid double toLowerCase)
+  
   const maxTokens = getModelTokenLimitFromLower(key, modelName);
   const targetTokens = Math.trunc(maxTokens * 0.85);
   const overlapTokens = Math.trunc(targetTokens * 0.18);
 
   const params = { maxTokens, targetTokens, overlapTokens };
 
-  // LRU eviction: remove oldest entry if at capacity
+  
   if (chunkingParamsCache.size >= MAX_CACHE_SIZE) {
     const oldestKey = chunkingParamsCache.keys().next().value;
     chunkingParamsCache.delete(oldestKey);
@@ -138,66 +108,43 @@ export function getChunkingParams(modelName) {
   return params;
 }
 
-/**
- * ASCII whitespace lookup table
- */
+
 const WS = new Uint8Array(128);
-WS[9] = 1; // \t (horizontal tab)
-WS[10] = 1; // \n (line feed)
-WS[11] = 1; // \v (vertical tab)
-WS[12] = 1; // \f (form feed)
-WS[13] = 1; // \r (carriage return)
-WS[32] = 1; // space
-
-/**
- * ASCII special character lookup table
- */
+WS[9] = 1; 
+WS[10] = 1; 
+WS[11] = 1; 
+WS[12] = 1; 
+WS[13] = 1; 
+WS[32] = 1; 
+
+
 const SPECIAL = new Uint8Array(128);
 const SPECIAL_CHARS = '{}()[];:,.<>!=+-*/%&|^~@#$"\'`\\';
 for (let i = 0; i < SPECIAL_CHARS.length; i++) {
   SPECIAL[SPECIAL_CHARS.charCodeAt(i)] = 1;
 }
 
-/**
- * Calculate token count for a word of given length
- * This function will be inlined by V8
- * @param {number} len - Word length in characters
- * @returns {number} Estimated token count
- */
+
 function calcWordTokens(len) {
   if (len <= 4) return 1;
   if (len <= 10) return 2;
-  return (len + 3) >> 2; // ceil(len / 4)
+  return (len + 3) >> 2; 
 }
 
-/**
- * Estimate token count for text (conservative estimate for code)
- *
- * Performance optimizations:
- * - No regex (pure integer comparisons)
- * - No string allocations (charCodeAt only)
- * - Inlined word token calculation
- * - Unicode checks ordered by frequency
- * - Branchless special character counting
- *
- * @param {string} text - The text to estimate tokens for
- * @param {object} [options]
- * @param {boolean} [options.includeSpecialTokens=true] - Whether to include [CLS]/[SEP]
- * @returns {number} Estimated token count
- */
+
 export function estimateTokens(text, { includeSpecialTokens = true } = {}) {
-  // Type-safe guard: prevents crashes from non-string inputs
+  
   if (typeof text !== 'string' || text.length === 0) return 0;
 
   const len = text.length;
-  let tokenCount = includeSpecialTokens ? 2 : 0; // [CLS] + [SEP]
+  let tokenCount = includeSpecialTokens ? 2 : 0; 
   let specialCount = 0;
   let wordStart = -1;
 
   for (let i = 0; i < len; i++) {
     const code = text.charCodeAt(i);
 
-    // ASCII fast path (most common for code)
+    
     if (code < 128) {
       if (WS[code]) {
         if (wordStart !== -1) {
@@ -205,28 +152,28 @@ export function estimateTokens(text, { includeSpecialTokens = true } = {}) {
           wordStart = -1;
         }
       } else {
-        // Branchless: add 0 or 1 based on SPECIAL[code]
+        
         specialCount += SPECIAL[code];
         if (wordStart === -1) wordStart = i;
       }
       continue;
     }
 
-    // Unicode whitespace: ordered by frequency for real-world text
-    // Note: Includes legacy 0x180E for tokenization compatibility even though
-    // modern JS \s doesn't consider it whitespace (ES2016+)
+    
+    
+    
     const isUnicodeWS =
-      code === 0x00a0 || // NBSP (most common)
-      code === 0x202f || // NARROW NO-BREAK SPACE
-      (code >= 0x2000 && code <= 0x200a) || // EN QUAD..HAIR SPACE
-      code === 0x3000 || // IDEOGRAPHIC SPACE (CJK)
-      code === 0x2028 || // LINE SEPARATOR
-      code === 0x2029 || // PARAGRAPH SEPARATOR
-      code === 0x205f || // MEDIUM MATHEMATICAL SPACE
-      code === 0x1680 || // OGHAM SPACE MARK
-      code === 0x180e || // MONGOLIAN VOWEL SEPARATOR (legacy)
-      code === 0x0085 || // NEXT LINE (NEL)
-      code === 0xfeff; // ZERO WIDTH NO-BREAK SPACE / BOM
+      code === 0x00a0 || 
+      code === 0x202f || 
+      (code >= 0x2000 && code <= 0x200a) || 
+      code === 0x3000 || 
+      code === 0x2028 || 
+      code === 0x2029 || 
+      code === 0x205f || 
+      code === 0x1680 || 
+      code === 0x180e || 
+      code === 0x0085 || 
+      code === 0xfeff; 
 
     if (isUnicodeWS) {
       if (wordStart !== -1) {
@@ -234,8 +181,8 @@ export function estimateTokens(text, { includeSpecialTokens = true } = {}) {
         wordStart = -1;
       }
     } else {
-      // Non-ASCII, non-whitespace (e.g., CJK, emojis, accented chars)
-      // Conservative estimate: treat each as 1 token
+      
+      
       if (wordStart !== -1) {
         tokenCount += calcWordTokens(i - wordStart);
         wordStart = -1;
@@ -244,12 +191,12 @@ export function estimateTokens(text, { includeSpecialTokens = true } = {}) {
     }
   }
 
-  // Flush final word
+  
   if (wordStart !== -1) {
     tokenCount += calcWordTokens(len - wordStart);
   }
 
-  // Add ~50% of special chars as tokens
+  
   tokenCount += specialCount >> 1;
 
   return tokenCount;