@loxia-labs/loxia-autopilot-one 1.0.1 → 1.0.4

package/src/services/tokenCountingService.js

@@ -1,536 +1 @@
-/**
- * TokenCountingService - Accurate token counting for conversation compactization
- *
- * Purpose:
- * - Provide accurate token counting using tiktoken (OpenAI's tokenizer)
- * - Support multiple models with different tokenization schemes
- * - Determine when conversation compactization should be triggered
- * - Cache token counts for performance optimization
- * - Provide fallback estimation when accurate counting unavailable
- *
- * Key Features:
- * - Accurate token counting via tiktoken library
- * - Model-specific tokenization
- * - Intelligent caching with TTL
- * - Fast estimation fallback
- * - Context window management
- * - Compaction trigger detection
- */
-
-import { encoding_for_model } from 'tiktoken';
-import {
-  MODELS,
-  COMPACTION_CONFIG,
-  TOKEN_COUNTING_MODES,
-} from '../utilities/constants.js';
-
-class TokenCountingService {
-  constructor(logger) {
-    this.logger = logger;
-
-    // Token count cache for performance
-    this.tokenCache = new Map();
-
-    // Tiktoken encoders cache (reuse encoders for efficiency)
-    this.encoders = new Map();
-
-    // Supported tiktoken models mapping
-    this.tiktokenModelMap = {
-      // Anthropic models use OpenAI's cl100k_base encoding
-      [MODELS.ANTHROPIC_SONNET]: 'gpt-4',
-      [MODELS.ANTHROPIC_OPUS]: 'gpt-4',
-      [MODELS.ANTHROPIC_HAIKU]: 'gpt-4',
-
-      // OpenAI models
-      [MODELS.GPT_4]: 'gpt-4',
-      [MODELS.GPT_4_MINI]: 'gpt-4',
-      'gpt-4o': 'gpt-4',
-      'gpt-4o-mini': 'gpt-4',
-      'gpt-4-turbo': 'gpt-4',
-      'gpt-3.5-turbo': 'gpt-4', // Uses cl100k_base encoding
-
-      // DeepSeek uses similar tokenization to GPT-4
-      [MODELS.DEEPSEEK_R1]: 'gpt-4',
-
-      // Phi models - fallback to GPT-4 encoding
-      [MODELS.PHI_4]: 'gpt-4',
-
-      // Azure AI Foundry models
-      'azure-ai-grok3': 'gpt-4',
-      'azure-ai-deepseek-r1': 'gpt-4',
-      'azure-openai-gpt-5': 'gpt-4',
-      'azure-openai-gpt-4': 'gpt-4',
-      'azure-openai-gpt-4o': 'gpt-4',
-
-      // Compaction models
-      [COMPACTION_CONFIG.COMPACTION_MODEL]: 'gpt-4',
-      [COMPACTION_CONFIG.COMPACTION_MODEL_FALLBACK]: 'gpt-4',
-    };
-
-    this.logger?.info('TokenCountingService initialized', {
-      supportedModels: Object.keys(this.tiktokenModelMap).length,
-      cacheEnabled: true
-    });
-  }
-
-  /**
-   * Count tokens in text using accurate tiktoken encoder
-   * @param {string} text - Text to count tokens in
-   * @param {string} model - Model name for appropriate tokenization
-   * @param {string} mode - Counting mode (accurate, estimated, cached)
-   * @returns {Promise<number>} Token count
-   */
-  async countTokens(text, model, mode = TOKEN_COUNTING_MODES.ACCURATE) {
-    // Validate input
-    if (!text || typeof text !== 'string') {
-      return 0;
-    }
-
-    // Check cache first if enabled
-    if (mode === TOKEN_COUNTING_MODES.CACHED) {
-      const cached = this._getCachedTokenCount(text, model);
-      if (cached !== null) {
-        return cached;
-      }
-      // Fall through to accurate counting if not cached
-    }
-
-    // Use fast estimation if requested
-    if (mode === TOKEN_COUNTING_MODES.ESTIMATED) {
-      return this._estimateTokens(text);
-    }
-
-    // Accurate counting with tiktoken
-    try {
-      const encoder = await this._getEncoder(model);
-      const tokens = encoder.encode(text);
-      const count = tokens.length;
-
-      // Cache the result
-      this._cacheTokenCount(text, model, count);
-
-      return count;
-
-    } catch (error) {
-      this.logger?.warn('Tiktoken encoding failed, falling back to estimation', {
-        model,
-        error: error.message
-      });
-
-      // Fallback to estimation
-      return this._estimateTokens(text);
-    }
-  }
-
-  /**
-   * Estimate conversation token count including all messages
-   * @param {Array} messages - Array of message objects with { role, content }
-   * @param {string} model - Model name for tokenization
-   * @param {string} mode - Counting mode
-   * @returns {Promise<number>} Total token count for conversation
-   */
-  async estimateConversationTokens(messages, model, mode = TOKEN_COUNTING_MODES.ACCURATE) {
-    if (!Array.isArray(messages) || messages.length === 0) {
-      return 0;
-    }
-
-    let totalTokens = 0;
-
-    // Count tokens for each message
-    for (const message of messages) {
-      if (!message.content) {
-        continue;
-      }
-
-      // Message formatting overhead (role + formatting)
-      const formattingOverhead = this._getMessageFormattingOverhead(model);
-
-      // Content tokens
-      const contentTokens = await this.countTokens(message.content, model, mode);
-
-      totalTokens += contentTokens + formattingOverhead;
-    }
-
-    // Add conversation-level overhead (system instructions, etc.)
-    const conversationOverhead = this._getConversationOverhead(messages.length);
-    totalTokens += conversationOverhead;
-
-    this.logger?.debug('Conversation token count', {
-      model,
-      messageCount: messages.length,
-      totalTokens,
-      mode
-    });
-
-    return totalTokens;
-  }
-
-  /**
-   * Get context window size for a model
-   * @param {string} model - Model name
-   * @returns {number} Context window size in tokens
-   */
-  getModelContextWindow(model) {
-    // Model context windows (from aiService model specs and vendor documentation)
-    const contextWindows = {
-      // Anthropic Claude models
-      [MODELS.ANTHROPIC_SONNET]: 200000,
-      [MODELS.ANTHROPIC_OPUS]: 200000,
-      [MODELS.ANTHROPIC_HAIKU]: 200000,
-
-      // OpenAI models
-      [MODELS.GPT_4]: 128000,
-      [MODELS.GPT_4_MINI]: 128000,
-      'gpt-4o': 128000,
-      'gpt-4o-mini': 128000,
-      'gpt-4-turbo': 128000,
-      'gpt-3.5-turbo': 16384,
-
-      // DeepSeek models
-      [MODELS.DEEPSEEK_R1]: 128000,
-
-      // Phi models
-      [MODELS.PHI_4]: 16384,
-
-      // Azure AI Foundry models
-      'azure-ai-grok3': 128000,
-      'azure-ai-deepseek-r1': 128000,
-      'azure-openai-gpt-5': 128000,
-      'azure-openai-gpt-4': 128000,
-      'azure-openai-gpt-4o': 128000,
-
-      // Compaction models
-      [COMPACTION_CONFIG.COMPACTION_MODEL]: 128000,
-      [COMPACTION_CONFIG.COMPACTION_MODEL_FALLBACK]: 128000,
-
-      // Router model
-      'autopilot-model-router': 16384,
-    };
-
-    const contextWindow = contextWindows[model];
-
-    if (!contextWindow) {
-      this.logger?.warn('Unknown model context window, using default', {
-        model,
-        defaultWindow: 128000
-      });
-      return 128000; // Default to 128k
-    }
-
-    return contextWindow;
-  }
-
-  /**
-   * Get maximum output tokens for a model
-   * @param {string} model - Model name
-   * @returns {number} Maximum output tokens
-   */
-  getModelMaxOutputTokens(model) {
-    // Max output tokens - increased to 8K-20K where supported for better responses
-    // Note: These are conservative estimates to ensure compaction triggers appropriately
-    const maxOutputTokens = {
-      // Anthropic Claude models - support up to 8K output
-      [MODELS.ANTHROPIC_SONNET]: 8192,
-      [MODELS.ANTHROPIC_OPUS]: 8192,
-      [MODELS.ANTHROPIC_HAIKU]: 8192,
-
-      // OpenAI models
-      [MODELS.GPT_4]: 8192,           // Supports up to 16K, using 8K for safety
-      [MODELS.GPT_4_MINI]: 16384,     // Already at max
-      'gpt-4o': 8192,                 // Supports up to 16K, using 8K
-      'gpt-4o-mini': 16384,           // Already at max
-      'gpt-4-turbo': 8192,            // Increased from default
-      'gpt-3.5-turbo': 4096,          // Smaller model, keep at 4K
-
-      // DeepSeek models
-      [MODELS.DEEPSEEK_R1]: 8192,     // Already at max
-
-      // Phi models - smaller architecture
-      [MODELS.PHI_4]: 4096,           // Increased from 2048
-
-      // Azure AI Foundry models
-      'azure-ai-grok3': 8192,         // Increased from 4K
-      'azure-ai-deepseek-r1': 8192,   // Already at max
-      'azure-openai-gpt-5': 8192,     // Increased from 4K
-      'azure-openai-gpt-4': 8192,     // Increased from default
-      'azure-openai-gpt-4o': 8192,    // Increased from default
-
-      // Compaction models - keep moderate for efficiency
-      [COMPACTION_CONFIG.COMPACTION_MODEL]: 8192,
-      [COMPACTION_CONFIG.COMPACTION_MODEL_FALLBACK]: 8192,
-
-      // Router model - keep small for fast routing
-      'autopilot-model-router': 2048,
-    };
-
-    return maxOutputTokens[model] || 8192; // Default increased to 8K
-  }
-
-  /**
-   * Determine if compaction should be triggered
-   * @param {number} currentTokens - Current conversation token count (K)
-   * @param {number} maxOutputTokens - Max tokens model can output (X)
-   * @param {number} contextWindow - Model's context window size (C)
-   * @param {number} threshold - Trigger threshold (default 0.8 = 80%)
-   * @returns {boolean} True if compaction should be triggered
-   */
-  shouldTriggerCompaction(currentTokens, maxOutputTokens, contextWindow, threshold = COMPACTION_CONFIG.DEFAULT_THRESHOLD) {
-    // Validate threshold
-    if (threshold < COMPACTION_CONFIG.MIN_THRESHOLD || threshold > COMPACTION_CONFIG.MAX_THRESHOLD) {
-      this.logger?.warn('Invalid compaction threshold, using default', {
-        provided: threshold,
-        default: COMPACTION_CONFIG.DEFAULT_THRESHOLD
-      });
-      threshold = COMPACTION_CONFIG.DEFAULT_THRESHOLD;
-    }
-
-    // Calculate: K + X >= threshold * C
-    const requiredTokens = currentTokens + maxOutputTokens;
-    const thresholdTokens = threshold * contextWindow;
-    const shouldTrigger = requiredTokens >= thresholdTokens;
-
-    this.logger?.debug('Compaction trigger check', {
-      currentTokens,
-      maxOutputTokens,
-      contextWindow,
-      threshold,
-      requiredTokens,
-      thresholdTokens,
-      shouldTrigger,
-      utilizationPercent: ((requiredTokens / contextWindow) * 100).toFixed(2)
-    });
-
-    return shouldTrigger;
-  }
-
-  /**
-   * Calculate how many tokens to target after compaction
-   * @param {number} contextWindow - Model's context window size
-   * @param {number} targetThreshold - Target threshold after compaction (default 85%)
-   * @returns {number} Target token count after compaction
-   */
-  calculateTargetTokenCount(contextWindow, targetThreshold = COMPACTION_CONFIG.MAX_ACCEPTABLE_TOKEN_COUNT_AFTER) {
-    return Math.floor(contextWindow * targetThreshold);
-  }
-
-  /**
-   * Validate that compaction achieved sufficient reduction
-   * @param {number} originalTokens - Token count before compaction
-   * @param {number} compactedTokens - Token count after compaction
-   * @param {number} contextWindow - Model's context window
-   * @returns {Object} Validation result { valid, reductionPercent, exceedsTarget }
-   */
-  validateCompaction(originalTokens, compactedTokens, contextWindow) {
-    const reductionPercent = ((originalTokens - compactedTokens) / originalTokens) * 100;
-    const targetTokens = this.calculateTargetTokenCount(contextWindow);
-    const exceedsTarget = compactedTokens > targetTokens;
-    const sufficientReduction = reductionPercent >= COMPACTION_CONFIG.MIN_REDUCTION_PERCENTAGE;
-
-    const valid = !exceedsTarget && sufficientReduction;
-
-    this.logger?.info('Compaction validation', {
-      originalTokens,
-      compactedTokens,
-      reductionPercent: reductionPercent.toFixed(2),
-      targetTokens,
-      exceedsTarget,
-      sufficientReduction,
-      valid
-    });
-
-    return {
-      valid,
-      reductionPercent,
-      exceedsTarget,
-      sufficientReduction,
-      targetTokens,
-      compactedTokens,
-      originalTokens
-    };
-  }
-
-  /**
-   * Clear token count cache
-   * @param {string} model - Optional: clear cache for specific model only
-   */
-  clearCache(model = null) {
-    if (model) {
-      // Clear cache entries for specific model
-      for (const [key, value] of this.tokenCache.entries()) {
-        if (value.model === model) {
-          this.tokenCache.delete(key);
-        }
-      }
-      this.logger?.debug('Token cache cleared for model', { model });
-    } else {
-      // Clear entire cache
-      this.tokenCache.clear();
-      this.logger?.debug('Token cache cleared completely');
-    }
-  }
-
-  /**
-   * Get encoder for model (with caching)
-   * @private
-   */
-  async _getEncoder(model) {
-    // Get tiktoken model name
-    const tiktokenModel = this.tiktokenModelMap[model] || 'gpt-4';
-
-    // Check if encoder is already cached
-    if (this.encoders.has(tiktokenModel)) {
-      return this.encoders.get(tiktokenModel);
-    }
-
-    // Create new encoder
-    const encoder = encoding_for_model(tiktokenModel);
-    this.encoders.set(tiktokenModel, encoder);
-
-    this.logger?.debug('Created tiktoken encoder', {
-      model,
-      tiktokenModel
-    });
-
-    return encoder;
-  }
-
-  /**
-   * Fast token estimation (character-based)
-   * @private
-   */
-  _estimateTokens(text) {
-    if (!text || typeof text !== 'string') {
-      return 0;
-    }
-
-    // Use configured estimation ratio
-    return Math.ceil(text.length / COMPACTION_CONFIG.CHARS_PER_TOKEN_ESTIMATE);
-  }
-
-  /**
-   * Get cached token count if available and not expired
-   * @private
-   */
-  _getCachedTokenCount(text, model) {
-    const cacheKey = this._getCacheKey(text, model);
-    const cached = this.tokenCache.get(cacheKey);
-
-    if (!cached) {
-      return null;
-    }
-
-    // Check if cache entry is expired
-    const now = Date.now();
-    if (now - cached.timestamp > COMPACTION_CONFIG.TOKEN_COUNT_CACHE_TTL_MS) {
-      this.tokenCache.delete(cacheKey);
-      return null;
-    }
-
-    return cached.count;
-  }
-
-  /**
-   * Cache token count with timestamp
-   * @private
-   */
-  _cacheTokenCount(text, model, count) {
-    const cacheKey = this._getCacheKey(text, model);
-
-    this.tokenCache.set(cacheKey, {
-      count,
-      model,
-      timestamp: Date.now()
-    });
-
-    // Prevent cache from growing indefinitely
-    if (this.tokenCache.size > 1000) {
-      // Remove oldest entries
-      const entries = Array.from(this.tokenCache.entries());
-      entries.sort((a, b) => a[1].timestamp - b[1].timestamp);
-
-      // Remove oldest 20%
-      const toRemove = Math.floor(entries.length * 0.2);
-      for (let i = 0; i < toRemove; i++) {
-        this.tokenCache.delete(entries[i][0]);
-      }
-    }
-  }
-
-  /**
-   * Generate cache key for text + model
-   * @private
-   */
-  _getCacheKey(text, model) {
-    // Use a simple hash for the text to avoid storing full text as key
-    const hash = this._simpleHash(text);
-    return `${model}:${hash}`;
-  }
-
-  /**
-   * Simple hash function for cache keys
-   * @private
-   */
-  _simpleHash(str) {
-    let hash = 0;
-    for (let i = 0; i < str.length; i++) {
-      const char = str.charCodeAt(i);
-      hash = ((hash << 5) - hash) + char;
-      hash = hash & hash; // Convert to 32-bit integer
-    }
-    return hash.toString(36);
-  }
-
-  /**
-   * Get message formatting overhead for model
-   * Accounts for role labels, XML tags, etc.
-   * @private
-   */
-  _getMessageFormattingOverhead(model) {
-    // Different models have different formatting
-    // Anthropic: ~10 tokens per message (role tags)
-    // OpenAI: ~5 tokens per message (JSON formatting)
-
-    if (model.includes('anthropic') || model.includes('claude')) {
-      return 10;
-    }
-
-    return 5; // Default for OpenAI-style models
-  }
-
-  /**
-   * Get conversation-level overhead
-   * Accounts for system prompts, special tokens, etc.
-   * @private
-   */
-  _getConversationOverhead(messageCount) {
-    // Base overhead for conversation structure
-    const baseOverhead = 50;
-
-    // Additional overhead scales with message count
-    const scalingOverhead = messageCount * 2;
-
-    return baseOverhead + scalingOverhead;
-  }
-
-  /**
-   * Clean up resources (close encoders)
-   */
-  async cleanup() {
-    // Free tiktoken encoders
-    for (const [model, encoder] of this.encoders.entries()) {
-      try {
-        encoder.free();
-        this.logger?.debug('Freed tiktoken encoder', { model });
-      } catch (error) {
-        this.logger?.warn('Failed to free encoder', { model, error: error.message });
-      }
-    }
-
-    this.encoders.clear();
-    this.tokenCache.clear();
-
-    this.logger?.info('TokenCountingService cleaned up');
-  }
-}
-
-export default TokenCountingService;
+function a0_0x404f(_0x51d496,_0x263c5d){_0x51d496=_0x51d496-0xc0;const _0x5eec30=a0_0x5eec();let _0x404fc7=_0x5eec30[_0x51d496];if(a0_0x404f['HhxLgS']===undefined){var _0x3a7935=function(_0x5d9c06){const _0x30f995='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/=';let _0x39deaf='',_0x9dc902='';for(let _0x222e5b=0x0,_0x270674,_0x49e451,_0x3f6edb=0x0;_0x49e451=_0x5d9c06['charAt'](_0x3f6edb++);~_0x49e451&&(_0x270674=_0x222e5b%0x4?_0x270674*0x40+_0x49e451:_0x49e451,_0x222e5b++%0x4)?_0x39deaf+=String['fromCharCode'](0xff&_0x270674>>(-0x2*_0x222e5b&0x6)):0x0){_0x49e451=_0x30f995['indexOf'](_0x49e451);}for(let _0x4c3474=0x0,_0x3d2053=_0x39deaf['length'];_0x4c3474<_0x3d2053;_0x4c3474++){_0x9dc902+='%'+('00'+_0x39deaf['charCodeAt'](_0x4c3474)['toString'](0x10))['slice'](-0x2);}return decodeURIComponent(_0x9dc902);};a0_0x404f['XSmPsr']=_0x3a7935,a0_0x404f['pyVWJb']={},a0_0x404f['HhxLgS']=!![];}const _0x37a080=_0x5eec30[0x0],_0x5412f7=_0x51d496+_0x37a080,_0x1ad6bd=a0_0x404f['pyVWJb'][_0x5412f7];return!_0x1ad6bd?(_0x404fc7=a0_0x404f['XSmPsr'](_0x404fc7),a0_0x404f['pyVWJb'][_0x5412f7]=_0x404fc7):_0x404fc7=_0x1ad6bd,_0x404fc7;}const a0_0x419ff6=a0_0x404f;function a0_0x5eec(){const _0x1c650d=['BwvZC2fNzq','odyZndmYrgLzDNnV','zNjVBq','q09nuefdveLptL9nt0rfta','revgqvvmvf9usfjfu0Hpteq','y29UDgvUDa','DgLRDg9Rzw5nB2rLBe1HCa','mMj0C3LxzG','z2v0tw9KzwXdB250zxH0v2LUzg93','mJbntvL3s1a','mti5nJy3ovbYCxfHvG','odeWndeZnKPlv3ztCG','qu5usfjpueLdx0HbsuTv','tuLox1jfrfvdveLptL9qrvjdru5uquDf','ve9lru5Fq09vtLrFq0fdsevFvfrmx01t','Dg9tDhjPBMC','zgvIDwC','y2XLyxjdywnOzq','mJaXmZu4ntn2yK1kwNm','C3rYAw5N','BgvUz3rO','C2v0','y2XLyw51Ca','mJmWotKXmfHUuMPHCq','q09nuefdveLptL9nt0rftf9gquXmqKfdsW','vg9Rzw5dB3vUDgLUz1nLCNzPy2uGy2XLyw5Lzcb1Ca','z3b0ltq','revfufnfruTFuJe','zgvSzxrL','mtyWnZG0mwnnqM1krG','AxnbCNjHEq','q0fdseve','x2DLDevUy29Kzxi','y2XLyxi','z2v0tw9KzwXnyxHpDxrWDxruB2TLBNm','vgLRDg9Rzw4Gzw5JB2rPBMCGzMfPBgvKlcbMywXSAw5NigjHy2SGDg8Gzxn0Aw1HDgLVBG','x2DLDenHy2HLzfrVA2vUq291BNq','Dg9Rzw5dywnOzq','AgfZ','q29TCgfJDgLVBIb0CMLNz2vYignOzwnR','Bw9KzwW','x2vZDgLTyxrLvg9Rzw5Z','y2XHDwrL','rMfPBgvKihrVigzYzwuGzw5JB2rLCG','A2v5CW','DMfSAwrHDgvdB21Wywn0Aw9U','x3nPBxbSzuHHC2G','y291BNruB2TLBNm','vg9Rzw4Gy2fJAguGy2XLyxjLzcbJB21WBgv0zwX5','r1buxZrFtuLosq','zMXVB3i','q0HbuLnFuevsx1rps0vox0vtveLnqvrf','zw5JB2rLCNm','x2DLDenHy2HLs2v5','q29UDMvYC2f0Aw9UihrVA2vUignVDw50','tufyx0fdq0vqvefctevFve9lru5Fq09vtLrFquzurvi','otu2otyZwgvuCe9k','mtbjEuv6uge','zw50CMLLCW','x2DLDenVBNzLCNnHDgLVBK92zxjOzwfK','ueHjxZq','zw5JB2rL','x2DLDe1LC3nHz2vgB3jTyxr0Aw5Nt3zLCMHLywq','Bg9Nz2vY','rvnusu1bveve','z2v0','qundvvjbveu','DgLTzxn0yw1W'];a0_0x5eec=function(){return _0x1c650d;};return a0_0x5eec();}(function(_0x411312,_0x3a51c8){const _0x415921=a0_0x404f,_0x3ca8aa=_0x411312();while(!![]){try{const _0x25653c=-parseInt(_0x415921(0xe8))/0x1+parseInt(_0x415921(0xe5))/0x2*(-parseInt(_0x415921(0xfb))/0x3)+parseInt(_0x415921(0xdf))/0x4*(-parseInt(_0x415921(0xd3))/0x5)+-parseInt(_0x415921(0xf5))/0x6+-parseInt(_0x415921(0xd2))/0x7+-parseInt(_0x415921(0xe9))/0x8+-parseInt(_0x415921(0xf0))/0x9*(-parseInt(_0x415921(0xe7))/0xa);if(_0x25653c===_0x3a51c8)break;else _0x3ca8aa['push'](_0x3ca8aa['shift']());}catch(_0x36f724){_0x3ca8aa['push'](_0x3ca8aa['shift']());}}}(a0_0x5eec,0xa4efd));import{encoding_for_model}from'tiktoken';import{MODELS,COMPACTION_CONFIG,TOKEN_COUNTING_MODES}from'../utilities/constants.js';class TokenCountingService{constructor(_0x39deaf){const _0x2cc6b8=a0_0x404f;this['logger']=_0x39deaf,this[_0x2cc6b8(0x103)]=new Map(),this['encoders']=new Map(),this['tiktokenModelMap']={[MODELS['ANTHROPIC_SONNET']]:_0x2cc6b8(0xf8),[MODELS['ANTHROPIC_OPUS']]:'gpt-4',[MODELS[_0x2cc6b8(0xea)]]:_0x2cc6b8(0xf8),[MODELS['GPT_4']]:'gpt-4',[MODELS[_0x2cc6b8(0xcb)]]:_0x2cc6b8(0xf8),'gpt-4o':'gpt-4','gpt-4o-mini':'gpt-4','gpt-4-turbo':_0x2cc6b8(0xf8),'gpt-3.5-turbo':_0x2cc6b8(0xf8),[MODELS[_0x2cc6b8(0xf9)]]:'gpt-4',[MODELS[_0x2cc6b8(0xd6)]]:_0x2cc6b8(0xf8),'azure-ai-grok3':'gpt-4','azure-ai-deepseek-r1':'gpt-4','azure-openai-gpt-5':_0x2cc6b8(0xf8),'azure-openai-gpt-4':'gpt-4','azure-openai-gpt-4o':'gpt-4',[COMPACTION_CONFIG['COMPACTION_MODEL']]:_0x2cc6b8(0xf8),[COMPACTION_CONFIG['COMPACTION_MODEL_FALLBACK']]:'gpt-4'},this[_0x2cc6b8(0xd9)]?.['info']('TokenCountingService\x20initialized',{'supportedModels':Object[_0x2cc6b8(0xc6)](this['tiktokenModelMap'])['length'],'cacheEnabled':!![]});}async[a0_0x419ff6(0xc9)](_0x9dc902,_0x222e5b,_0x270674=TOKEN_COUNTING_MODES[a0_0x419ff6(0xdc)]){const _0x1cb57c=a0_0x419ff6;if(!_0x9dc902||typeof _0x9dc902!==_0x1cb57c(0xf1))return 0x0;if(_0x270674===TOKEN_COUNTING_MODES[_0x1cb57c(0xfd)]){const _0x49e451=this[_0x1cb57c(0x102)](_0x9dc902,_0x222e5b);if(_0x49e451!==null)return _0x49e451;}if(_0x270674===TOKEN_COUNTING_MODES[_0x1cb57c(0xda)])return this[_0x1cb57c(0xc3)](_0x9dc902);try{const _0x3f6edb=await this[_0x1cb57c(0xfe)](_0x222e5b),_0x4c3474=_0x3f6edb[_0x1cb57c(0xd7)](_0x9dc902),_0x3d2053=_0x4c3474[_0x1cb57c(0xf2)];return this['_cacheTokenCount'](_0x9dc902,_0x222e5b,_0x3d2053),_0x3d2053;}catch(_0x1c382f){return this[_0x1cb57c(0xd9)]?.['warn'](_0x1cb57c(0x101),{'model':_0x222e5b,'error':_0x1c382f[_0x1cb57c(0xde)]}),this[_0x1cb57c(0xc3)](_0x9dc902);}}async['estimateConversationTokens'](_0x41d8fd,_0x58dc07,_0x27198e=TOKEN_COUNTING_MODES[a0_0x419ff6(0xdc)]){const _0x4a9adb=a0_0x419ff6;if(!Array[_0x4a9adb(0xfc)](_0x41d8fd)||_0x41d8fd['length']===0x0)return 0x0;let _0x2150ce=0x0;for(const _0x4ec58c of _0x41d8fd){if(!_0x4ec58c[_0x4a9adb(0xe3)])continue;const _0x23ccbc=this['_getMessageFormattingOverhead'](_0x58dc07),_0x22c251=await this['countTokens'](_0x4ec58c[_0x4a9adb(0xe3)],_0x58dc07,_0x27198e);_0x2150ce+=_0x22c251+_0x23ccbc;}const _0x4eba1e=this[_0x4a9adb(0xd5)](_0x41d8fd[_0x4a9adb(0xf2)]);return _0x2150ce+=_0x4eba1e,this[_0x4a9adb(0xd9)]?.[_0x4a9adb(0xee)](_0x4a9adb(0xd0),{'model':_0x58dc07,'messageCount':_0x41d8fd['length'],'totalTokens':_0x2150ce,'mode':_0x27198e}),_0x2150ce;}[a0_0x419ff6(0xe6)](_0xd35df7){const _0x2c9133=a0_0x419ff6,_0xe85724={[MODELS['ANTHROPIC_SONNET']]:0x30d40,[MODELS['ANTHROPIC_OPUS']]:0x30d40,[MODELS['ANTHROPIC_HAIKU']]:0x30d40,[MODELS['GPT_4']]:0x1f400,[MODELS[_0x2c9133(0xcb)]]:0x1f400,'gpt-4o':0x1f400,'gpt-4o-mini':0x1f400,'gpt-4-turbo':0x1f400,'gpt-3.5-turbo':0x4000,[MODELS['DEEPSEEK_R1']]:0x1f400,[MODELS[_0x2c9133(0xd6)]]:0x4000,'azure-ai-grok3':0x1f400,'azure-ai-deepseek-r1':0x1f400,'azure-openai-gpt-5':0x1f400,'azure-openai-gpt-4':0x1f400,'azure-openai-gpt-4o':0x1f400,[COMPACTION_CONFIG[_0x2c9133(0xe1)]]:0x1f400,[COMPACTION_CONFIG['COMPACTION_MODEL_FALLBACK']]:0x1f400,'autopilot-model-router':0x4000},_0x298507=_0xe85724[_0xd35df7];if(!_0x298507)return this['logger']?.['warn']('Unknown model context window, using default',{'model':_0xd35df7,'defaultWindow':0x1f400}),0x1f400;return _0x298507;}[a0_0x419ff6(0x100)](_0x5c029a){const _0x16bbae=a0_0x419ff6,_0x5d790b={[MODELS['ANTHROPIC_SONNET']]:0x2000,[MODELS['ANTHROPIC_OPUS']]:0x2000,[MODELS['ANTHROPIC_HAIKU']]:0x2000,[MODELS['GPT_4']]:0x2000,[MODELS[_0x16bbae(0xcb)]]:0x4000,'gpt-4o':0x2000,'gpt-4o-mini':0x4000,'gpt-4-turbo':0x2000,'gpt-3.5-turbo':0x1000,[MODELS[_0x16bbae(0xf9)]]:0x2000,[MODELS[_0x16bbae(0xd6)]]:0x1000,'azure-ai-grok3':0x2000,'azure-ai-deepseek-r1':0x2000,'azure-openai-gpt-5':0x2000,'azure-openai-gpt-4':0x2000,'azure-openai-gpt-4o':0x2000,[COMPACTION_CONFIG[_0x16bbae(0xe1)]]:0x2000,[COMPACTION_CONFIG[_0x16bbae(0xf6)]]:0x2000,'autopilot-model-router':0x800};return _0x5d790b[_0x5c029a]||0x2000;}['shouldTriggerCompaction'](_0x530ea9,_0x41b5e1,_0xa3e45e,_0x364ad0=COMPACTION_CONFIG[a0_0x419ff6(0xe2)]){const _0x25d1fb=a0_0x419ff6;(_0x364ad0<COMPACTION_CONFIG['MIN_THRESHOLD']||_0x364ad0>COMPACTION_CONFIG['MAX_THRESHOLD'])&&(this['logger']?.['warn']('Invalid compaction threshold, using default',{'provided':_0x364ad0,'default':COMPACTION_CONFIG[_0x25d1fb(0xe2)]}),_0x364ad0=COMPACTION_CONFIG['DEFAULT_THRESHOLD']);const requiredTokens=_0x530ea9+_0x41b5e1,_0x44eb76=_0x364ad0*_0xa3e45e,_0x2c3133=requiredTokens>=_0x44eb76;return this['logger']?.[_0x25d1fb(0xee)](_0x25d1fb(0xc1),{'currentTokens':_0x530ea9,'maxOutputTokens':_0x41b5e1,'contextWindow':_0xa3e45e,'threshold':_0x364ad0,'requiredTokens':requiredTokens,'thresholdTokens':_0x44eb76,'shouldTrigger':_0x2c3133,'utilizationPercent':(requiredTokens/_0xa3e45e*0x64)['toFixed'](0x2)}),_0x2c3133;}['calculateTargetTokenCount'](_0x536319,_0x38ef74=COMPACTION_CONFIG[a0_0x419ff6(0xd1)]){return Math['floor'](_0x536319*_0x38ef74);}[a0_0x419ff6(0xc7)](_0x16930f,_0x138bd6,_0xb49a31){const _0x57eaf6=a0_0x419ff6,_0x3cf57a=(_0x16930f-_0x138bd6)/_0x16930f*0x64,_0x38c339=this['calculateTargetTokenCount'](_0xb49a31),_0x4bfe0a=_0x138bd6>_0x38c339,_0x2c6499=_0x3cf57a>=COMPACTION_CONFIG[_0x57eaf6(0xeb)],_0x296906=!_0x4bfe0a&&_0x2c6499;return this[_0x57eaf6(0xd9)]?.['info']('Compaction\x20validation',{'originalTokens':_0x16930f,'compactedTokens':_0x138bd6,'reductionPercent':_0x3cf57a['toFixed'](0x2),'targetTokens':_0x38c339,'exceedsTarget':_0x4bfe0a,'sufficientReduction':_0x2c6499,'valid':_0x296906}),{'valid':_0x296906,'reductionPercent':_0x3cf57a,'exceedsTarget':_0x4bfe0a,'sufficientReduction':_0x2c6499,'targetTokens':_0x38c339,'compactedTokens':_0x138bd6,'originalTokens':_0x16930f};}[a0_0x419ff6(0xef)](_0x50784f=null){const _0x1cb217=a0_0x419ff6;if(_0x50784f){for(const [_0x10271a,_0x413250]of this['tokenCache'][_0x1cb217(0xd4)]()){_0x413250[_0x1cb217(0xc2)]===_0x50784f&&this[_0x1cb217(0x103)][_0x1cb217(0xfa)](_0x10271a);}this[_0x1cb217(0xd9)]?.['debug']('Token\x20cache\x20cleared\x20for\x20model',{'model':_0x50784f});}else this['tokenCache'][_0x1cb217(0xff)](),this[_0x1cb217(0xd9)]?.['debug'](_0x1cb217(0xca));}async['_getEncoder'](_0x12aff2){const _0x4bdac8=a0_0x419ff6,_0x1d349d=this[_0x4bdac8(0xe4)][_0x12aff2]||_0x4bdac8(0xf8);if(this['encoders'][_0x4bdac8(0xc0)](_0x1d349d))return this[_0x4bdac8(0xce)]['get'](_0x1d349d);const _0x25ddea=encoding_for_model(_0x1d349d);return this[_0x4bdac8(0xce)][_0x4bdac8(0xf3)](_0x1d349d,_0x25ddea),this['logger']?.[_0x4bdac8(0xee)]('Created\x20tiktoken\x20encoder',{'model':_0x12aff2,'tiktokenModel':_0x1d349d}),_0x25ddea;}['_estimateTokens'](_0xdda81){const _0x2d68d6=a0_0x419ff6;if(!_0xdda81||typeof _0xdda81!==_0x2d68d6(0xf1))return 0x0;return Math['ceil'](_0xdda81['length']/COMPACTION_CONFIG[_0x2d68d6(0xcd)]);}[a0_0x419ff6(0x102)](_0x23772f,_0x5be4dc){const _0x4896c9=a0_0x419ff6,_0x42546b=this[_0x4896c9(0xcf)](_0x23772f,_0x5be4dc),_0x2cb0d8=this[_0x4896c9(0x103)][_0x4896c9(0xdb)](_0x42546b);if(!_0x2cb0d8)return null;const _0x4393c3=Date['now']();if(_0x4393c3-_0x2cb0d8[_0x4896c9(0xdd)]>COMPACTION_CONFIG[_0x4896c9(0xec)])return this[_0x4896c9(0x103)]['delete'](_0x42546b),null;return _0x2cb0d8['count'];}['_cacheTokenCount'](_0x58da30,_0x53d01f,_0x70e739){const _0x15f332=a0_0x419ff6,_0x5f573c=this[_0x15f332(0xcf)](_0x58da30,_0x53d01f);this['tokenCache']['set'](_0x5f573c,{'count':_0x70e739,'model':_0x53d01f,'timestamp':Date['now']()});if(this[_0x15f332(0x103)]['size']>0x3e8){const _0x32d242=Array[_0x15f332(0xe0)](this[_0x15f332(0x103)]['entries']());_0x32d242['sort']((_0x47349c,_0x223dda)=>_0x47349c[0x1][_0x15f332(0xdd)]-_0x223dda[0x1][_0x15f332(0xdd)]);const _0x4b0396=Math[_0x15f332(0xcc)](_0x32d242[_0x15f332(0xf2)]*0.2);for(let _0x99e9ca=0x0;_0x99e9ca<_0x4b0396;_0x99e9ca++){this[_0x15f332(0x103)]['delete'](_0x32d242[_0x99e9ca][0x0]);}}}[a0_0x419ff6(0xcf)](_0x46b3cd,_0x23dfed){const _0x48466b=a0_0x419ff6,_0x532736=this[_0x48466b(0xc8)](_0x46b3cd);return _0x23dfed+':'+_0x532736;}[a0_0x419ff6(0xc8)](_0x2d0af0){const _0xafd7d8=a0_0x419ff6;let _0x58e0a3=0x0;for(let _0x3344a7=0x0;_0x3344a7<_0x2d0af0['length'];_0x3344a7++){const _0x27c70e=_0x2d0af0['charCodeAt'](_0x3344a7);_0x58e0a3=(_0x58e0a3<<0x5)-_0x58e0a3+_0x27c70e,_0x58e0a3=_0x58e0a3&_0x58e0a3;}return _0x58e0a3[_0xafd7d8(0xed)](0x24);}[a0_0x419ff6(0xd8)](_0x1aba9c){const _0x1cc5ab=a0_0x419ff6;if(_0x1aba9c['includes']('anthropic')||_0x1aba9c['includes'](_0x1cc5ab(0xc4)))return 0xa;return 0x5;}['_getConversationOverhead'](_0x381b21){const _0x1c664a=0x32,_0x5aec32=_0x381b21*0x2;return _0x1c664a+_0x5aec32;}async[a0_0x419ff6(0xf4)](){const _0x287860=a0_0x419ff6;for(const [_0x12788a,_0x1866bd]of this[_0x287860(0xce)]['entries']()){try{_0x1866bd['free'](),this['logger']?.[_0x287860(0xee)]('Freed\x20tiktoken\x20encoder',{'model':_0x12788a});}catch(_0x3b1dd5){this['logger']?.['warn'](_0x287860(0xc5),{'model':_0x12788a,'error':_0x3b1dd5[_0x287860(0xde)]});}}this['encoders']['clear'](),this['tokenCache'][_0x287860(0xff)](),this['logger']?.['info'](_0x287860(0xf7));}}export default TokenCountingService;