codereview-aia 0.2.1 → 0.2.3

package/dist/analysis/tokens/TokenAnalyzer.js

@@ -1,502 +0,0 @@
-"use strict";
-/**
- * @fileoverview Token analysis service for pre-review token counting and estimation.
- *
- * This module provides fast, provider-agnostic token counting and analysis functionality
- * to estimate token usage and costs before performing actual reviews.
- */
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TokenAnalyzer = void 0;
-const tokenizers_1 = require("../../tokenizers");
-const logger_1 = __importDefault(require("../../utils/logger"));
-/**
- * Service for analyzing token usage in files
- */
-class TokenAnalyzer {
-    static DEFAULT_PROMPT_OVERHEAD = 1500;
-    static DEFAULT_CONTEXT_MAINTENANCE_FACTOR = 0.08; // Reduced to 8% for better efficiency
-    static DEFAULT_SAFETY_MARGIN_FACTOR = 0.1; // Use 90% of context window by default
-    static DEFAULT_CONTEXT_WINDOW = 100000; // Default fallback
-    /**
-     * Context window mapping for supported models.
-     * Leave values as null if the exact window is unknown so they can be filled in later.
-     */
-    static MODEL_CONTEXT_WINDOWS = {
-        'openai/gpt-5.1-codex': 400000,
-        'gpt-5.1-codex': 400000,
-        'anthropic/claude-haiku-4.5': 200000,
-        'claude-haiku-4.5': 200000,
-        'moonshotai/kimi-k2-thinking': 262144,
-        'kimi-k2-thinking': 262144,
-        'x-ai/grok-4-fast': 2000000,
-        'grok-4-fast': 2000000,
-    };
-    /**
-     * Get the context window size for a model
-     * @param modelName Name of the model
-     * @returns Context window size in tokens
-     */
-    static getContextWindowSize(modelName) {
-        logger_1.default.debug(`getContextWindowSize: modelName=${modelName}`);
-        const configured = TokenAnalyzer.lookupConfiguredContextWindow(modelName);
-        if (typeof configured === 'number') {
-            logger_1.default.info(`Using configured context window for ${modelName}: ${configured.toLocaleString()} tokens`);
-            return configured;
-        }
-        if (configured === null) {
-            logger_1.default.warn(`Context window size for ${modelName} has not been set. Update TokenAnalyzer.MODEL_CONTEXT_WINDOWS to provide the exact token limit.`);
-        }
-        else {
-            logger_1.default.warn(`No matching context window size found for model: ${modelName}`);
-        }
-        logger_1.default.warn(`Using default context window size: ${TokenAnalyzer.DEFAULT_CONTEXT_WINDOW.toLocaleString()} tokens`);
-        return TokenAnalyzer.DEFAULT_CONTEXT_WINDOW;
-    }
-    static lookupConfiguredContextWindow(modelName) {
-        const normalized = modelName.toLowerCase();
-        if (normalized in TokenAnalyzer.MODEL_CONTEXT_WINDOWS) {
-            return TokenAnalyzer.MODEL_CONTEXT_WINDOWS[normalized];
-        }
-        if (normalized.includes(':')) {
-            const [, baseName] = normalized.split(':');
-            if (baseName && baseName in TokenAnalyzer.MODEL_CONTEXT_WINDOWS) {
-                return TokenAnalyzer.MODEL_CONTEXT_WINDOWS[baseName];
-            }
-        }
-        return undefined;
-    }
-    /**
-     * Analyze token usage for a set of files
-     * @param files Files to analyze
-     * @param options Analysis options
-     * @returns Token analysis result
-     */
-    static analyzeFiles(files, options) {
-        logger_1.default.info('Analyzing token usage for files...');
-        logger_1.default.debug(`TokenAnalyzer: modelName=${options.modelName}`);
-        const contextWindowSize = TokenAnalyzer.getContextWindowSize(options.modelName);
-        const promptOverhead = options.additionalPromptOverhead || TokenAnalyzer.DEFAULT_PROMPT_OVERHEAD;
-        const contextMaintenanceFactor = options.contextMaintenanceFactor || TokenAnalyzer.DEFAULT_CONTEXT_MAINTENANCE_FACTOR;
-        const safetyMarginFactor = options.safetyMarginFactor || TokenAnalyzer.DEFAULT_SAFETY_MARGIN_FACTOR;
-        // Calculate effective context window size with safety margin
-        const effectiveContextWindowSize = Math.floor(contextWindowSize * (1 - safetyMarginFactor));
-        logger_1.default.info(`Using effective context window size: ${effectiveContextWindowSize.toLocaleString()} tokens (${Math.round((1 - safetyMarginFactor) * 100)}% of ${contextWindowSize.toLocaleString()} tokens)`);
-        // Analyze each file
-        const fileAnalyses = files.map((file) => {
-            const content = file.content;
-            const tokenCount = (0, tokenizers_1.countTokens)(content, options.modelName);
-            const sizeInBytes = content.length;
-            const tokensPerByte = sizeInBytes > 0 ? tokenCount / sizeInBytes : 0;
-            return {
-                path: file.path,
-                relativePath: file.relativePath,
-                tokenCount,
-                sizeInBytes,
-                tokensPerByte,
-            };
-        });
-        // Calculate totals
-        const totalTokens = fileAnalyses.reduce((sum, file) => sum + file.tokenCount, 0);
-        const totalSizeInBytes = fileAnalyses.reduce((sum, file) => sum + file.sizeInBytes, 0);
-        const averageTokensPerByte = totalSizeInBytes > 0 ? totalTokens / totalSizeInBytes : 0;
-        // Estimate total tokens with overhead
-        const estimatedTotalTokens = totalTokens + promptOverhead;
-        // Determine if chunking is needed
-        const exceedsContextWindow = estimatedTotalTokens > effectiveContextWindowSize;
-        logger_1.default.info(`Token analysis summary:`);
-        logger_1.default.info(`- Total files: ${files.length}`);
-        logger_1.default.info(`- Total tokens: ${totalTokens.toLocaleString()}`);
-        logger_1.default.info(`- Prompt overhead: ${promptOverhead.toLocaleString()}`);
-        logger_1.default.info(`- Estimated total tokens: ${estimatedTotalTokens.toLocaleString()}`);
-        logger_1.default.info(`- Context window size: ${contextWindowSize.toLocaleString()}`);
-        logger_1.default.info(`- Effective context size (with safety margin): ${effectiveContextWindowSize.toLocaleString()}`);
-        logger_1.default.info(`- Context utilization: ${((estimatedTotalTokens / effectiveContextWindowSize) * 100).toFixed(2)}%`);
-        // Calculate recommended chunks if needed
-        const chunkingRecommendation = TokenAnalyzer.generateChunkingRecommendation(fileAnalyses, estimatedTotalTokens, effectiveContextWindowSize, contextMaintenanceFactor, options.forceSinglePass, options.batchTokenLimit);
-        // Log chunking decision
-        if (chunkingRecommendation.chunkingRecommended) {
-            logger_1.default.info(`Multi-pass review recommended: ${chunkingRecommendation.reason}`);
-            logger_1.default.info(`Estimated passes needed: ${chunkingRecommendation.recommendedChunks.length}`);
-        }
-        else {
-            logger_1.default.info(`Single-pass review recommended: ${chunkingRecommendation.reason}`);
-        }
-        return {
-            files: fileAnalyses,
-            totalTokens,
-            totalSizeInBytes,
-            averageTokensPerByte,
-            fileCount: files.length,
-            promptOverheadTokens: promptOverhead,
-            estimatedTotalTokens,
-            contextWindowSize,
-            exceedsContextWindow,
-            estimatedPassesNeeded: chunkingRecommendation.recommendedChunks.length,
-            chunkingRecommendation,
-        };
-    }
-    /**
-     * Generate a chunking recommendation for files that exceed context window
-     * @param fileAnalyses Array of file token analyses
-     * @param estimatedTotalTokens Total tokens including overhead
-     * @param contextWindowSize Maximum context window size
-     * @param contextMaintenanceFactor Context maintenance overhead factor
-     * @param forceSinglePass Force single pass mode regardless of token count
-     * @param batchTokenLimit Force maximum tokens per batch (for testing)
-     * @returns Chunking recommendation
-     */
-    static generateChunkingRecommendation(fileAnalyses, estimatedTotalTokens, contextWindowSize, contextMaintenanceFactor, forceSinglePass, batchTokenLimit) {
-        // If forceSinglePass is true, skip chunking regardless of token count
-        if (forceSinglePass) {
-            logger_1.default.debug(`Forcing single-pass review mode as requested (forceSinglePass=true)`);
-            return {
-                chunkingRecommended: false,
-                recommendedChunks: [
-                    {
-                        files: fileAnalyses.map((f) => f.path),
-                        estimatedTokenCount: estimatedTotalTokens,
-                        priority: 1,
-                    },
-                ],
-                reason: 'Single-pass mode forced by configuration',
-            };
-        }
-        // If batchTokenLimit is provided, use it to force smaller batches
-        let effectiveContextLimit = contextWindowSize;
-        if (batchTokenLimit && batchTokenLimit > 0) {
-            effectiveContextLimit = Math.min(batchTokenLimit, contextWindowSize);
-            logger_1.default.info(`Using batch token limit: ${batchTokenLimit.toLocaleString()} tokens (forcing smaller batches for testing)`);
-            if (batchTokenLimit < contextWindowSize) {
-                logger_1.default.info(`This will force chunking even if content would fit in the model's context window`);
-            }
-        }
-        // If content fits within context window, no chunking needed
-        if (estimatedTotalTokens <= effectiveContextLimit) {
-            logger_1.default.debug(`Content fits within effective limit (${estimatedTotalTokens.toLocaleString()} <= ${effectiveContextLimit.toLocaleString()} tokens)`);
-            return {
-                chunkingRecommended: false,
-                recommendedChunks: [
-                    {
-                        files: fileAnalyses.map((f) => f.path),
-                        estimatedTokenCount: estimatedTotalTokens,
-                        priority: 1,
-                    },
-                ],
-                reason: batchTokenLimit
-                    ? 'Content fits within batch token limit'
-                    : 'Content fits within model context window',
-            };
-        }
-        logger_1.default.debug(`Content exceeds effective limit (${estimatedTotalTokens.toLocaleString()} > ${effectiveContextLimit.toLocaleString()} tokens)`);
-        logger_1.default.debug(`Generating chunking recommendation with context maintenance factor: ${contextMaintenanceFactor}`);
-        // Calculate effective context window size accounting for context maintenance
-        const effectiveContextSize = Math.floor(effectiveContextLimit * (1 - contextMaintenanceFactor));
-        logger_1.default.debug(`Effective context size for chunking: ${effectiveContextSize.toLocaleString()} tokens (${Math.round((1 - contextMaintenanceFactor) * 100)}% of ${effectiveContextLimit.toLocaleString()} tokens)`);
-        // Use optimized bin-packing algorithm for better chunk distribution
-        const chunks = TokenAnalyzer.optimizedBinPacking(fileAnalyses, effectiveContextSize, effectiveContextLimit);
-        logger_1.default.info(`Created ${chunks.length} optimized chunks for multi-pass review`);
-        let reason = `Content exceeds effective limit (${estimatedTotalTokens.toLocaleString()} > ${effectiveContextLimit.toLocaleString()} tokens)`;
-        if (batchTokenLimit && batchTokenLimit < contextWindowSize) {
-            reason = `Batch token limit forcing smaller batches (limit: ${batchTokenLimit.toLocaleString()} tokens)`;
-        }
-        return {
-            chunkingRecommended: true,
-            recommendedChunks: chunks,
-            reason,
-        };
-    }
-    /**
-     * Optimized bin-packing algorithm to minimize the number of chunks
-     * Uses an advanced first-fit decreasing with multi-level optimization
-     * @param fileAnalyses Array of file token analyses
-     * @param maxChunkSize Maximum size for each chunk in tokens
-     * @param contextWindowSize Original context window for logging
-     * @returns Array of optimized file chunks
-     */
-    static optimizedBinPacking(fileAnalyses, maxChunkSize, _contextWindowSize) {
-        // Sort files by token count (largest first) for first-fit decreasing
-        const sortedFiles = [...fileAnalyses].sort((a, b) => b.tokenCount - a.tokenCount);
-        // Calculate target chunk size for optimal distribution
-        const totalTokens = sortedFiles.reduce((sum, f) => sum + f.tokenCount, 0);
-        const minChunksNeeded = Math.ceil(totalTokens / maxChunkSize);
-        const targetChunkSize = Math.floor(totalTokens / minChunksNeeded);
-        logger_1.default.debug(`Bin-packing optimization:`);
-        logger_1.default.debug(`  - Total tokens: ${totalTokens.toLocaleString()}`);
-        logger_1.default.debug(`  - Max chunk size: ${maxChunkSize.toLocaleString()}`);
-        logger_1.default.debug(`  - Min chunks needed: ${minChunksNeeded}`);
-        logger_1.default.debug(`  - Target chunk size: ${targetChunkSize.toLocaleString()}`);
-        // Initialize chunks array
-        const chunks = [];
-        // Track oversized files separately
-        const oversizedFiles = [];
-        const largeFiles = [];
-        const mediumFiles = [];
-        const smallFiles = [];
-        // Categorize files by size for better packing
-        for (const file of sortedFiles) {
-            if (file.tokenCount > maxChunkSize) {
-                oversizedFiles.push(file);
-                logger_1.default.warn(`File "${file.path}" is oversized (${file.tokenCount.toLocaleString()} > ${maxChunkSize.toLocaleString()} tokens)`);
-            }
-            else if (file.tokenCount > maxChunkSize * 0.5) {
-                largeFiles.push(file);
-            }
-            else if (file.tokenCount > maxChunkSize * 0.2) {
-                mediumFiles.push(file);
-            }
-            else {
-                smallFiles.push(file);
-            }
-        }
-        logger_1.default.debug(`File categorization:`);
-        logger_1.default.debug(`  - Oversized: ${oversizedFiles.length}`);
-        logger_1.default.debug(`  - Large (>50% of max): ${largeFiles.length}`);
-        logger_1.default.debug(`  - Medium (20-50% of max): ${mediumFiles.length}`);
-        logger_1.default.debug(`  - Small (<20% of max): ${smallFiles.length}`);
-        // Process oversized files first (split them if possible)
-        for (const file of oversizedFiles) {
-            // For now, put oversized files in their own chunks
-            // TODO: In future, we could split file content
-            chunks.push({
-                files: [file.path],
-                estimatedTokenCount: file.tokenCount,
-                priority: chunks.length + 1,
-            });
-            logger_1.default.debug(`Created dedicated chunk ${chunks.length} for oversized file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
-        }
-        // Process large files - try to pair them optimally
-        for (const file of largeFiles) {
-            let placed = false;
-            // Try to find a chunk with complementary space
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                const remainingSpace = maxChunkSize - chunk.estimatedTokenCount;
-                // Check if this file fits well (within 80% efficiency)
-                if (remainingSpace >= file.tokenCount &&
-                    chunk.estimatedTokenCount + file.tokenCount >= targetChunkSize * 0.8) {
-                    chunk.files.push(file.path);
-                    chunk.estimatedTokenCount += file.tokenCount;
-                    placed = true;
-                    logger_1.default.debug(`Added large file "${file.path}" (${file.tokenCount.toLocaleString()} tokens) to chunk ${i + 1}`);
-                    break;
-                }
-            }
-            if (!placed) {
-                // Create a new chunk for this large file
-                chunks.push({
-                    files: [file.path],
-                    estimatedTokenCount: file.tokenCount,
-                    priority: chunks.length + 1,
-                });
-                logger_1.default.debug(`Created new chunk ${chunks.length} for large file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
-            }
-        }
-        // Process medium files - use first-fit with efficiency threshold
-        for (const file of mediumFiles) {
-            let placed = false;
-            // Find first chunk where this file fits efficiently
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                const remainingSpace = maxChunkSize - chunk.estimatedTokenCount;
-                if (remainingSpace >= file.tokenCount) {
-                    chunk.files.push(file.path);
-                    chunk.estimatedTokenCount += file.tokenCount;
-                    placed = true;
-                    logger_1.default.debug(`Added medium file "${file.path}" (${file.tokenCount.toLocaleString()} tokens) to chunk ${i + 1}`);
-                    break;
-                }
-            }
-            if (!placed) {
-                // Create a new chunk
-                chunks.push({
-                    files: [file.path],
-                    estimatedTokenCount: file.tokenCount,
-                    priority: chunks.length + 1,
-                });
-                logger_1.default.debug(`Created new chunk ${chunks.length} for medium file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
-            }
-        }
-        // Process small files - pack them to fill gaps
-        // Sort small files for better packing (largest first)
-        smallFiles.sort((a, b) => b.tokenCount - a.tokenCount);
-        for (const file of smallFiles) {
-            let placed = false;
-            // Find the fullest chunk that can still fit this file
-            let bestChunkIndex = -1;
-            let bestChunkFullness = 0;
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                const remainingSpace = maxChunkSize - chunk.estimatedTokenCount;
-                const chunkFullness = chunk.estimatedTokenCount / maxChunkSize;
-                if (remainingSpace >= file.tokenCount && chunkFullness > bestChunkFullness) {
-                    bestChunkIndex = i;
-                    bestChunkFullness = chunkFullness;
-                }
-            }
-            if (bestChunkIndex !== -1) {
-                const chunk = chunks[bestChunkIndex];
-                chunk.files.push(file.path);
-                chunk.estimatedTokenCount += file.tokenCount;
-                placed = true;
-                logger_1.default.debug(`Added small file "${file.path}" (${file.tokenCount.toLocaleString()} tokens) to chunk ${bestChunkIndex + 1}`);
-            }
-            if (!placed) {
-                // Create a new chunk only if absolutely necessary
-                chunks.push({
-                    files: [file.path],
-                    estimatedTokenCount: file.tokenCount,
-                    priority: chunks.length + 1,
-                });
-                logger_1.default.debug(`Created new chunk ${chunks.length} for small file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
-            }
-        }
-        // Perform aggressive balancing to minimize chunk count
-        const balancedChunks = TokenAnalyzer.aggressiveBalance(chunks, fileAnalyses, maxChunkSize);
-        // Log chunk statistics
-        const avgTokensPerChunk = Math.round(balancedChunks.reduce((sum, c) => sum + c.estimatedTokenCount, 0) / balancedChunks.length);
-        const maxTokensInChunk = Math.max(...balancedChunks.map((c) => c.estimatedTokenCount));
-        const minTokensInChunk = Math.min(...balancedChunks.map((c) => c.estimatedTokenCount));
-        logger_1.default.info(`Chunk statistics:`);
-        logger_1.default.info(`  - Total chunks: ${balancedChunks.length}`);
-        logger_1.default.info(`  - Average tokens per chunk: ${avgTokensPerChunk.toLocaleString()}`);
-        logger_1.default.info(`  - Max tokens in a chunk: ${maxTokensInChunk.toLocaleString()}`);
-        logger_1.default.info(`  - Min tokens in a chunk: ${minTokensInChunk.toLocaleString()}`);
-        logger_1.default.info(`  - Chunk efficiency: ${((avgTokensPerChunk / maxChunkSize) * 100).toFixed(1)}%`);
-        return balancedChunks;
-    }
-    /**
-     * Aggressive balancing to minimize chunk count and maximize efficiency
-     * @param chunks Initial chunks from bin-packing
-     * @param fileAnalyses Original file analyses for lookup
-     * @param maxChunkSize Maximum size for each chunk
-     * @returns Balanced chunks with minimized count
-     */
-    static aggressiveBalance(chunks, fileAnalyses, maxChunkSize) {
-        // Create a map for quick file lookups
-        const fileMap = new Map();
-        for (const file of fileAnalyses) {
-            fileMap.set(file.path, file);
-        }
-        // First pass: Try to merge small chunks
-        const mergedChunks = [];
-        const sortedForMerging = [...chunks].sort((a, b) => a.estimatedTokenCount - b.estimatedTokenCount);
-        const usedChunks = new Set();
-        for (let i = 0; i < sortedForMerging.length; i++) {
-            if (usedChunks.has(i))
-                continue;
-            const chunk1 = sortedForMerging[i];
-            const mergedChunk = {
-                files: [...chunk1.files],
-                estimatedTokenCount: chunk1.estimatedTokenCount,
-                priority: mergedChunks.length + 1,
-            };
-            usedChunks.add(i);
-            // Try to merge with other small chunks
-            for (let j = i + 1; j < sortedForMerging.length; j++) {
-                if (usedChunks.has(j))
-                    continue;
-                const chunk2 = sortedForMerging[j];
-                const combinedSize = mergedChunk.estimatedTokenCount + chunk2.estimatedTokenCount;
-                // Merge if combined size is still within limits
-                if (combinedSize <= maxChunkSize) {
-                    mergedChunk.files.push(...chunk2.files);
-                    mergedChunk.estimatedTokenCount = combinedSize;
-                    usedChunks.add(j);
-                    logger_1.default.debug(`Merged chunks: ${chunk2.files.length} files (${chunk2.estimatedTokenCount.toLocaleString()} tokens) into chunk with ${mergedChunk.files.length} files`);
-                }
-            }
-            mergedChunks.push(mergedChunk);
-        }
-        logger_1.default.debug(`Chunk merging reduced count from ${chunks.length} to ${mergedChunks.length}`);
-        // Second pass: Balance the merged chunks
-        const sortedChunks = [...mergedChunks].sort((a, b) => a.estimatedTokenCount - b.estimatedTokenCount);
-        // Try to move files to achieve better balance
-        let improved = true;
-        let iterations = 0;
-        const maxIterations = 20; // More iterations for aggressive optimization
-        while (improved && iterations < maxIterations) {
-            improved = false;
-            iterations++;
-            // Find the most and least full chunks
-            sortedChunks.sort((a, b) => a.estimatedTokenCount - b.estimatedTokenCount);
-            for (let i = 0; i < Math.floor(sortedChunks.length / 2); i++) {
-                const smallChunk = sortedChunks[i];
-                const largeChunk = sortedChunks[sortedChunks.length - 1 - i];
-                // Calculate variance threshold based on chunk count
-                const varianceThreshold = Math.max(500, maxChunkSize * 0.05); // 5% of max or 500 tokens
-                // Skip if chunks are already well balanced
-                if (largeChunk.estimatedTokenCount - smallChunk.estimatedTokenCount < varianceThreshold) {
-                    continue;
-                }
-                // Try to find optimal file to move
-                let bestFile = null;
-                let bestImprovement = 0;
-                for (const filePath of largeChunk.files) {
-                    const file = fileMap.get(filePath);
-                    if (!file)
-                        continue;
-                    const newSmallSize = smallChunk.estimatedTokenCount + file.tokenCount;
-                    const newLargeSize = largeChunk.estimatedTokenCount - file.tokenCount;
-                    // Check if moving this file would improve balance
-                    if (newSmallSize <= maxChunkSize && newLargeSize > 0) {
-                        const currentDiff = largeChunk.estimatedTokenCount - smallChunk.estimatedTokenCount;
-                        const newDiff = Math.abs(newLargeSize - newSmallSize);
-                        const improvement = currentDiff - newDiff;
-                        if (improvement > bestImprovement) {
-                            bestFile = filePath;
-                            bestImprovement = improvement;
-                        }
-                    }
-                }
-                // Move the best file if found
-                if (bestFile && bestImprovement > 100) {
-                    const file = fileMap.get(bestFile);
-                    largeChunk.files = largeChunk.files.filter((f) => f !== bestFile);
-                    smallChunk.files.push(bestFile);
-                    // Update token counts
-                    largeChunk.estimatedTokenCount -= file.tokenCount;
-                    smallChunk.estimatedTokenCount += file.tokenCount;
-                    logger_1.default.debug(`Balanced: Moved file "${bestFile}" (${file.tokenCount.toLocaleString()} tokens) - improvement: ${bestImprovement.toLocaleString()} tokens`);
-                    improved = true;
-                }
-            }
-        }
-        // Final pass: Remove any empty chunks
-        const finalChunks = sortedChunks.filter((chunk) => chunk.files.length > 0);
-        // Re-assign priorities based on token count (largest first for processing)
-        finalChunks.sort((a, b) => b.estimatedTokenCount - a.estimatedTokenCount);
-        finalChunks.forEach((chunk, index) => {
-            chunk.priority = index + 1;
-        });
-        if (iterations === maxIterations) {
-            logger_1.default.debug(`Aggressive balancing stopped after ${maxIterations} iterations`);
-        }
-        else {
-            logger_1.default.debug(`Aggressive balancing completed in ${iterations} iterations`);
-        }
-        return finalChunks;
-    }
-    /**
-     * Analyze a single file for token usage
-     * @param file File to analyze
-     * @param options Analysis options
-     * @returns Token analysis for the file
-     */
-    static analyzeFile(file, options) {
-        const content = file.content;
-        const tokenCount = (0, tokenizers_1.countTokens)(content, options.modelName);
-        const sizeInBytes = content.length;
-        return {
-            path: file.path,
-            relativePath: file.relativePath,
-            tokenCount,
-            sizeInBytes,
-            tokensPerByte: sizeInBytes > 0 ? tokenCount / sizeInBytes : 0,
-        };
-    }
-}
-exports.TokenAnalyzer = TokenAnalyzer;
-//# sourceMappingURL=TokenAnalyzer.js.map

package/dist/analysis/tokens/TokenAnalyzer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"TokenAnalyzer.js","sourceRoot":"","sources":["../../../src/analysis/tokens/TokenAnalyzer.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;AAEH,iDAA+C;AAE/C,gEAAwC;AA4FxC;;GAEG;AACH,MAAa,aAAa;IAChB,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC;IACtC,MAAM,CAAC,kCAAkC,GAAG,IAAI,CAAC,CAAC,sCAAsC;IACxF,MAAM,CAAC,4BAA4B,GAAG,GAAG,CAAC,CAAC,uCAAuC;IAClF,MAAM,CAAC,sBAAsB,GAAG,MAAM,CAAC,CAAC,mBAAmB;IACnE;;;OAGG;IACK,MAAM,CAAC,qBAAqB,GAAkC;QACpE,sBAAsB,EAAE,MAAM;QAC9B,eAAe,EAAE,MAAM;QACvB,4BAA4B,EAAE,MAAM;QACpC,kBAAkB,EAAE,MAAM;QAC1B,6BAA6B,EAAE,MAAM;QACrC,kBAAkB,EAAE,MAAM;QAC1B,kBAAkB,EAAE,OAAO;QAC3B,aAAa,EAAE,OAAO;KACvB,CAAC;IAEF;;;;OAIG;IACK,MAAM,CAAC,oBAAoB,CAAC,SAAiB;QACnD,gBAAM,CAAC,KAAK,CAAC,mCAAmC,SAAS,EAAE,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,aAAa,CAAC,6BAA6B,CAAC,SAAS,CAAC,CAAC;QAE1E,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACnC,gBAAM,CAAC,IAAI,CAAC,uCAAuC,SAAS,KAAK,UAAU,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YACvG,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,gBAAM,CAAC,IAAI,CACT,2BAA2B,SAAS,iGAAiG,CACtI,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,oDAAoD,SAAS,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,gBAAM,CAAC,IAAI,CACT,sCAAsC,aAAa,CAAC,sBAAsB,CAAC,cAAc,EAAE,SAAS,CACrG,CAAC;QACF,OAAO,aAAa,CAAC,sBAAsB,CAAC;IAC9C,CAAC;IAEO,MAAM,CAAC,6BAA6B,CAAC,SAAiB;QAC5D,MAAM,UAAU,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,UAAU,IAAI,aAAa,CAAC,qBAAqB,EAAE,CAAC;YACtD,OAAO,aAAa,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,EAAE,QAAQ,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,QAAQ,IAAI,QAAQ,IAAI,aAAa,CAAC,qBAAqB,EAAE,CAAC;gBAChE,OAAO,aAAa,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,YAAY,CACxB,KAAiB,EACjB,OAA6B;QAE7B,gBAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QAClD,gBAAM,CAAC,KAAK,CAAC,4BAA4B,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAE9D,MAAM,iBAAiB,GAAG,aAAa,CAAC,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChF,MAAM,cAAc,GAClB,OAAO,CAAC,wBAAwB,IAAI,aAAa,CAAC,uBAAuB,CAAC;QAC5E,MAAM,wBAAwB,GAC5B,OAAO,CAAC,wBAAwB,IAAI,aAAa,CAAC,kCAAkC,CAAC;QACvF,MAAM,kBAAkB,GACtB,OAAO,CAAC,kBAAkB,IAAI,aAAa,CAAC,4BAA4B,CAAC;QAE3E,6DAA6D;QAC7D,MAAM,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC;QAC5F,gBAAM,CAAC,IAAI,CACT,wCAAwC,0BAA0B,CAAC,cAAc,EAAE,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,GAAG,GAAG,CAAC,QAAQ,iBAAiB,CAAC,cAAc,EAAE,UAAU,CAC9L,CAAC;QAEF,oBAAoB;QACpB,MAAM,YAAY,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAA,wBAAW,EAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YACnC,MAAM,aAAa,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAErE,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,UAAU;gBACV,WAAW;gBACX,aAAa;aACd,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,mBAAmB;QACnB,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACjF,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACvF,MAAM,oBAAoB,GAAG,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvF,sCAAsC;QACtC,MAAM,oBAAoB,GAAG,WAAW,GAAG,cAAc,CAAC;QAE1D,kCAAkC;QAClC,MAAM,oBAAoB,GAAG,oBAAoB,GAAG,0BAA0B,CAAC;QAE/E,gBAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACvC,gBAAM,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,gBAAM,CAAC,IAAI,CAAC,mBAAmB,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/D,gBAAM,CAAC,IAAI,CAAC,sBAAsB,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACrE,gBAAM,CAAC,IAAI,CAAC,6BAA6B,oBAAoB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAClF,gBAAM,CAAC,IAAI,CAAC,0BAA0B,iBAAiB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC5E,gBAAM,CAAC,IAAI,CACT,kDAAkD,0BAA0B,CAAC,cAAc,EAAE,EAAE,CAChG,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,0BAA0B,CAAC,CAAC,oBAAoB,GAAG,0BAA0B,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACpG,CAAC;QAEF,yCAAyC;QACzC,MAAM,sBAAsB,GAAG,aAAa,CAAC,8BAA8B,CACzE,YAAY,EACZ,oBAAoB,EACpB,0BAA0B,EAC1B,wBAAwB,EACxB,OAAO,CAAC,eAAe,EACvB,OAAO,CAAC,eAAe,CACxB,CAAC;QAEF,wBAAwB;QACxB,IAAI,sBAAsB,CAAC,mBAAmB,EAAE,CAAC;YAC/C,gBAAM,CAAC,IAAI,CAAC,kCAAkC,sBAAsB,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/E,gBAAM,CAAC,IAAI,CAAC,4BAA4B,sBAAsB,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,mCAAmC,sBAAsB,CAAC,MAAM,EAAE,CAAC,CAAC;QAClF,CAAC;QAED,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,WAAW;YACX,gBAAgB;YAChB,oBAAoB;YACpB,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,oBAAoB,EAAE,cAAc;YACpC,oBAAoB;YACpB,iBAAiB;YACjB,oBAAoB;YACpB,qBAAqB,EAAE,sBAAsB,CAAC,iBAAiB,CAAC,MAAM;YACtE,sBAAsB;SACvB,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACK,MAAM,CAAC,8BAA8B,CAC3C,YAAiC,EACjC,oBAA4B,EAC5B,iBAAyB,EACzB,wBAAgC,EAChC,eAAyB,EACzB,eAAwB;QAExB,sEAAsE;QACtE,IAAI,eAAe,EAAE,CAAC;YACpB,gBAAM,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACpF,OAAO;gBACL,mBAAmB,EAAE,KAAK;gBAC1B,iBAAiB,EAAE;oBACjB;wBACE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;wBACtC,mBAAmB,EAAE,oBAAoB;wBACzC,QAAQ,EAAE,CAAC;qBACZ;iBACF;gBACD,MAAM,EAAE,0CAA0C;aACnD,CAAC;QACJ,CAAC;QAED,kEAAkE;QAClE,IAAI,qBAAqB,GAAG,iBAAiB,CAAC;QAC9C,IAAI,eAAe,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YAC3C,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;YACrE,gBAAM,CAAC,IAAI,CACT,4BAA4B,eAAe,CAAC,cAAc,EAAE,+CAA+C,CAC5G,CAAC;YACF,IAAI,eAAe,GAAG,iBAAiB,EAAE,CAAC;gBACxC,gBAAM,CAAC,IAAI,CACT,kFAAkF,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,IAAI,oBAAoB,IAAI,qBAAqB,EAAE,CAAC;YAClD,gBAAM,CAAC,KAAK,CACV,wCAAwC,oBAAoB,CAAC,cAAc,EAAE,OAAO,qBAAqB,CAAC,cAAc,EAAE,UAAU,CACrI,CAAC;YACF,OAAO;gBACL,mBAAmB,EAAE,KAAK;gBAC1B,iBAAiB,EAAE;oBACjB;wBACE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;wBACtC,mBAAmB,EAAE,oBAAoB;wBACzC,QAAQ,EAAE,CAAC;qBACZ;iBACF;gBACD,MAAM,EAAE,eAAe;oBACrB,CAAC,CAAC,uCAAuC;oBACzC,CAAC,CAAC,0CAA0C;aAC/C,CAAC;QACJ,CAAC;QAED,gBAAM,CAAC,KAAK,CACV,oCAAoC,oBAAoB,CAAC,cAAc,EAAE,MAAM,qBAAqB,CAAC,cAAc,EAAE,UAAU,CAChI,CAAC;QACF,gBAAM,CAAC,KAAK,CACV,uEAAuE,wBAAwB,EAAE,CAClG,CAAC;QAEF,6EAA6E;QAC7E,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,CAAC,CAAC,GAAG,wBAAwB,CAAC,CAAC,CAAC;QAEhG,gBAAM,CAAC,KAAK,CACV,wCAAwC,oBAAoB,CAAC,cAAc,EAAE,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,wBAAwB,CAAC,GAAG,GAAG,CAAC,QAAQ,qBAAqB,CAAC,cAAc,EAAE,UAAU,CAClM,CAAC;QAEF,oEAAoE;QACpE,MAAM,MAAM,GAAG,aAAa,CAAC,mBAAmB,CAC9C,YAAY,EACZ,oBAAoB,EACpB,qBAAqB,CACtB,CAAC;QAEF,gBAAM,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,MAAM,yCAAyC,CAAC,CAAC;QAE/E,IAAI,MAAM,GAAG,oCAAoC,oBAAoB,CAAC,cAAc,EAAE,MAAM,qBAAqB,CAAC,cAAc,EAAE,UAAU,CAAC;QAC7I,IAAI,eAAe,IAAI,eAAe,GAAG,iBAAiB,EAAE,CAAC;YAC3D,MAAM,GAAG,qDAAqD,eAAe,CAAC,cAAc,EAAE,UAAU,CAAC;QAC3G,CAAC;QAED,OAAO;YACL,mBAAmB,EAAE,IAAI;YACzB,iBAAiB,EAAE,MAAM;YACzB,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACK,MAAM,CAAC,mBAAmB,CAChC,YAAiC,EACjC,YAAoB,EACpB,kBAA0B;QAE1B,qEAAqE;QACrE,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAElF,uDAAuD;QACvD,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,eAAe,CAAC,CAAC;QAElE,gBAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC1C,gBAAM,CAAC,KAAK,CAAC,qBAAqB,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAClE,gBAAM,CAAC,KAAK,CAAC,uBAAuB,YAAY,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACrE,gBAAM,CAAC,KAAK,CAAC,0BAA0B,eAAe,EAAE,CAAC,CAAC;QAC1D,gBAAM,CAAC,KAAK,CAAC,0BAA0B,eAAe,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAE3E,0BAA0B;QAC1B,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,mCAAmC;QACnC,MAAM,cAAc,GAAwB,EAAE,CAAC;QAC/C,MAAM,UAAU,GAAwB,EAAE,CAAC;QAC3C,MAAM,WAAW,GAAwB,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAwB,EAAE,CAAC;QAE3C,8CAA8C;QAC9C,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,UAAU,GAAG,YAAY,EAAE,CAAC;gBACnC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,gBAAM,CAAC,IAAI,CACT,SAAS,IAAI,CAAC,IAAI,mBAAmB,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,MAAM,YAAY,CAAC,cAAc,EAAE,UAAU,CACnH,CAAC;YACJ,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC;gBAChD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC;gBAChD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,gBAAM,CAAC,KAAK,CAAC,kBAAkB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QACxD,gBAAM,CAAC,KAAK,CAAC,4BAA4B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,gBAAM,CAAC,KAAK,CAAC,+BAA+B,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAClE,gBAAM,CAAC,KAAK,CAAC,4BAA4B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9D,yDAAyD;QACzD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,mDAAmD;YACnD,+CAA+C;YAC/C,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;gBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;aAC5B,CAAC,CAAC;YACH,gBAAM,CAAC,KAAK,CACV,2BAA2B,MAAM,CAAC,MAAM,wBAAwB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAC1H,CAAC;QACJ,CAAC;QAED,mDAAmD;QACnD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,MAAM,GAAG,KAAK,CAAC;YAEnB,+CAA+C;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,cAAc,GAAG,YAAY,GAAG,KAAK,CAAC,mBAAmB,CAAC;gBAEhE,uDAAuD;gBACvD,IACE,cAAc,IAAI,IAAI,CAAC,UAAU;oBACjC,KAAK,CAAC,mBAAmB,GAAG,IAAI,CAAC,UAAU,IAAI,eAAe,GAAG,GAAG,EACpE,CAAC;oBACD,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5B,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAC7C,MAAM,GAAG,IAAI,CAAC;oBACd,gBAAM,CAAC,KAAK,CACV,qBAAqB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,qBAAqB,CAAC,GAAG,CAAC,EAAE,CACjG,CAAC;oBACF,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;oBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBACH,gBAAM,CAAC,KAAK,CACV,qBAAqB,MAAM,CAAC,MAAM,oBAAoB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAChH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,iEAAiE;QACjE,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,MAAM,GAAG,KAAK,CAAC;YAEnB,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,cAAc,GAAG,YAAY,GAAG,KAAK,CAAC,mBAAmB,CAAC;gBAEhE,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACtC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5B,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAC7C,MAAM,GAAG,IAAI,CAAC;oBACd,gBAAM,CAAC,KAAK,CACV,sBAAsB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAClG,CAAC;oBACF,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,qBAAqB;gBACrB,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;oBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBACH,gBAAM,CAAC,KAAK,CACV,qBAAqB,MAAM,CAAC,MAAM,qBAAqB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CACjH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,sDAAsD;QACtD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAEvD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,MAAM,GAAG,KAAK,CAAC;YAEnB,sDAAsD;YACtD,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC;YACxB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,cAAc,GAAG,YAAY,GAAG,KAAK,CAAC,mBAAmB,CAAC;gBAChE,MAAM,aAAa,GAAG,KAAK,CAAC,mBAAmB,GAAG,YAAY,CAAC;gBAE/D,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,IAAI,aAAa,GAAG,iBAAiB,EAAE,CAAC;oBAC3E,cAAc,GAAG,CAAC,CAAC;oBACnB,iBAAiB,GAAG,aAAa,CAAC;gBACpC,CAAC;YACH,CAAC;YAED,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;gBACrC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;gBAC7C,MAAM,GAAG,IAAI,CAAC;gBACd,gBAAM,CAAC,KAAK,CACV,qBAAqB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,qBAAqB,cAAc,GAAG,CAAC,EAAE,CAC9G,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kDAAkD;gBAClD,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;oBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBACH,gBAAM,CAAC,KAAK,CACV,qBAAqB,MAAM,CAAC,MAAM,oBAAoB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAChH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,uDAAuD;QACvD,MAAM,cAAc,GAAG,aAAa,CAAC,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;QAE3F,uBAAuB;QACvB,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAClC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM,CAC1F,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACvF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAEvF,gBAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACjC,gBAAM,CAAC,IAAI,CAAC,qBAAqB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1D,gBAAM,CAAC,IAAI,CAAC,iCAAiC,iBAAiB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACnF,gBAAM,CAAC,IAAI,CAAC,8BAA8B,gBAAgB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/E,gBAAM,CAAC,IAAI,CAAC,8BAA8B,gBAAgB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/E,gBAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,iBAAiB,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE/F,OAAO,cAAc,CAAC;IACxB,CAAC;IAED;;;;;;OAMG;IACK,MAAM,CAAC,iBAAiB,CAC9B,MAAmB,EACnB,YAAiC,EACjC,YAAoB;QAEpB,sCAAsC;QACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;QACrD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC/B,CAAC;QAED,wCAAwC;QACxC,MAAM,YAAY,GAAgB,EAAE,CAAC;QACrC,MAAM,gBAAgB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CACvC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CACxD,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEhC,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,WAAW,GAAc;gBAC7B,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;gBACxB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,QAAQ,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC;aAClC,CAAC;YACF,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAElB,uCAAuC;YACvC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrD,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAEhC,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,YAAY,GAAG,WAAW,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAC;gBAElF,gDAAgD;gBAChD,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;oBACjC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;oBACxC,WAAW,CAAC,mBAAmB,GAAG,YAAY,CAAC;oBAC/C,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBAClB,gBAAM,CAAC,KAAK,CACV,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,4BAA4B,WAAW,CAAC,KAAK,CAAC,MAAM,QAAQ,CACxJ,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,oCAAoC,MAAM,CAAC,MAAM,OAAO,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QAE5F,yCAAyC;QACzC,MAAM,YAAY,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CACxD,CAAC;QAEF,8CAA8C;QAC9C,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,aAAa,GAAG,EAAE,CAAC,CAAC,8CAA8C;QAExE,OAAO,QAAQ,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;YAC9C,QAAQ,GAAG,KAAK,CAAC;YACjB,UAAU,EAAE,CAAC;YAEb,sCAAsC;YACtC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC;YAE3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7D,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,UAAU,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBAE7D,oDAAoD;gBACpD,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,0BAA0B;gBAExF,2CAA2C;gBAC3C,IAAI,UAAU,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,GAAG,iBAAiB,EAAE,CAAC;oBACxF,SAAS;gBACX,CAAC;gBAED,mCAAmC;gBACnC,IAAI,QAAQ,GAAkB,IAAI,CAAC;gBACnC,IAAI,eAAe,GAAG,CAAC,CAAC;gBAExB,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;oBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBACnC,IAAI,CAAC,IAAI;wBAAE,SAAS;oBAEpB,MAAM,YAAY,GAAG,UAAU,CAAC,mBAAmB,GAAG,IAAI,CAAC,UAAU,CAAC;oBACtE,MAAM,YAAY,GAAG,UAAU,CAAC,mBAAmB,GAAG,IAAI,CAAC,UAAU,CAAC;oBAEtE,kDAAkD;oBAClD,IAAI,YAAY,IAAI,YAAY,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;wBACrD,MAAM,WAAW,GAAG,UAAU,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;wBACpF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC;wBACtD,MAAM,WAAW,GAAG,WAAW,GAAG,OAAO,CAAC;wBAE1C,IAAI,WAAW,GAAG,eAAe,EAAE,CAAC;4BAClC,QAAQ,GAAG,QAAQ,CAAC;4BACpB,eAAe,GAAG,WAAW,CAAC;wBAChC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,8BAA8B;gBAC9B,IAAI,QAAQ,IAAI,eAAe,GAAG,GAAG,EAAE,CAAC;oBACtC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;oBACpC,UAAU,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC;oBAClE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAEhC,sBAAsB;oBACtB,UAAU,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAClD,UAAU,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAElD,gBAAM,CAAC,KAAK,CACV,yBAAyB,QAAQ,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,2BAA2B,eAAe,CAAC,cAAc,EAAE,SAAS,CAC5I,CAAC;oBAEF,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE3E,2EAA2E;QAC3E,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC1E,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACnC,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,IAAI,UAAU,KAAK,aAAa,EAAE,CAAC;YACjC,gBAAM,CAAC,KAAK,CAAC,sCAAsC,aAAa,aAAa,CAAC,CAAC;QACjF,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,KAAK,CAAC,qCAAqC,UAAU,aAAa,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,WAAW,CAAC,IAAc,EAAE,OAA6B;QACrE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAA,wBAAW,EAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,UAAU;YACV,WAAW;YACX,aAAa,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;SAC9D,CAAC;IACJ,CAAC;;AAjoBH,sCAkoBC"}

package/dist/analysis/tokens/index.d.ts

@@ -1,7 +0,0 @@
-/**
- * @fileoverview Token analysis module exports.
- *
- * This module exports all token analysis related functions and types.
- */
-export * from './TokenAnalysisFormatter';
-export * from './TokenAnalyzer';

package/dist/analysis/tokens/index.js

@@ -1,24 +0,0 @@
-"use strict";
-/**
- * @fileoverview Token analysis module exports.
- *
- * This module exports all token analysis related functions and types.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./TokenAnalysisFormatter"), exports);
-__exportStar(require("./TokenAnalyzer"), exports);
-//# sourceMappingURL=index.js.map

package/dist/analysis/tokens/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/analysis/tokens/index.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;;;;;;AAEH,2DAAyC;AACzC,kDAAgC"}