codereview-aia 0.1.3 → 0.2.0

package/dist/analysis/tokens/TokenAnalyzer.js

@@ -0,0 +1,502 @@
+"use strict";
+/**
+ * @fileoverview Token analysis service for pre-review token counting and estimation.
+ *
+ * This module provides fast, provider-agnostic token counting and analysis functionality
+ * to estimate token usage and costs before performing actual reviews.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TokenAnalyzer = void 0;
+const tokenizers_1 = require("../../tokenizers");
+const logger_1 = __importDefault(require("../../utils/logger"));
+/**
+ * Service for analyzing token usage in files
+ */
+class TokenAnalyzer {
+    static DEFAULT_PROMPT_OVERHEAD = 1500;
+    static DEFAULT_CONTEXT_MAINTENANCE_FACTOR = 0.08; // Reduced to 8% for better efficiency
+    static DEFAULT_SAFETY_MARGIN_FACTOR = 0.1; // Use 90% of context window by default
+    static DEFAULT_CONTEXT_WINDOW = 100000; // Default fallback
+    /**
+     * Context window mapping for supported models.
+     * Leave values as null if the exact window is unknown so they can be filled in later.
+     */
+    static MODEL_CONTEXT_WINDOWS = {
+        'openai/gpt-5.1-codex': 400000,
+        'gpt-5.1-codex': 400000,
+        'anthropic/claude-haiku-4.5': 200000,
+        'claude-haiku-4.5': 200000,
+        'moonshotai/kimi-k2-thinking': 262144,
+        'kimi-k2-thinking': 262144,
+        'x-ai/grok-4-fast': 2000000,
+        'grok-4-fast': 2000000,
+    };
+    /**
+     * Get the context window size for a model
+     * @param modelName Name of the model
+     * @returns Context window size in tokens
+     */
+    static getContextWindowSize(modelName) {
+        logger_1.default.debug(`getContextWindowSize: modelName=${modelName}`);
+        const configured = TokenAnalyzer.lookupConfiguredContextWindow(modelName);
+        if (typeof configured === 'number') {
+            logger_1.default.info(`Using configured context window for ${modelName}: ${configured.toLocaleString()} tokens`);
+            return configured;
+        }
+        if (configured === null) {
+            logger_1.default.warn(`Context window size for ${modelName} has not been set. Update TokenAnalyzer.MODEL_CONTEXT_WINDOWS to provide the exact token limit.`);
+        }
+        else {
+            logger_1.default.warn(`No matching context window size found for model: ${modelName}`);
+        }
+        logger_1.default.warn(`Using default context window size: ${TokenAnalyzer.DEFAULT_CONTEXT_WINDOW.toLocaleString()} tokens`);
+        return TokenAnalyzer.DEFAULT_CONTEXT_WINDOW;
+    }
+    static lookupConfiguredContextWindow(modelName) {
+        const normalized = modelName.toLowerCase();
+        if (normalized in TokenAnalyzer.MODEL_CONTEXT_WINDOWS) {
+            return TokenAnalyzer.MODEL_CONTEXT_WINDOWS[normalized];
+        }
+        if (normalized.includes(':')) {
+            const [, baseName] = normalized.split(':');
+            if (baseName && baseName in TokenAnalyzer.MODEL_CONTEXT_WINDOWS) {
+                return TokenAnalyzer.MODEL_CONTEXT_WINDOWS[baseName];
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Analyze token usage for a set of files
+     * @param files Files to analyze
+     * @param options Analysis options
+     * @returns Token analysis result
+     */
+    static analyzeFiles(files, options) {
+        logger_1.default.info('Analyzing token usage for files...');
+        logger_1.default.debug(`TokenAnalyzer: modelName=${options.modelName}`);
+        const contextWindowSize = TokenAnalyzer.getContextWindowSize(options.modelName);
+        const promptOverhead = options.additionalPromptOverhead || TokenAnalyzer.DEFAULT_PROMPT_OVERHEAD;
+        const contextMaintenanceFactor = options.contextMaintenanceFactor || TokenAnalyzer.DEFAULT_CONTEXT_MAINTENANCE_FACTOR;
+        const safetyMarginFactor = options.safetyMarginFactor || TokenAnalyzer.DEFAULT_SAFETY_MARGIN_FACTOR;
+        // Calculate effective context window size with safety margin
+        const effectiveContextWindowSize = Math.floor(contextWindowSize * (1 - safetyMarginFactor));
+        logger_1.default.info(`Using effective context window size: ${effectiveContextWindowSize.toLocaleString()} tokens (${Math.round((1 - safetyMarginFactor) * 100)}% of ${contextWindowSize.toLocaleString()} tokens)`);
+        // Analyze each file
+        const fileAnalyses = files.map((file) => {
+            const content = file.content;
+            const tokenCount = (0, tokenizers_1.countTokens)(content, options.modelName);
+            const sizeInBytes = content.length;
+            const tokensPerByte = sizeInBytes > 0 ? tokenCount / sizeInBytes : 0;
+            return {
+                path: file.path,
+                relativePath: file.relativePath,
+                tokenCount,
+                sizeInBytes,
+                tokensPerByte,
+            };
+        });
+        // Calculate totals
+        const totalTokens = fileAnalyses.reduce((sum, file) => sum + file.tokenCount, 0);
+        const totalSizeInBytes = fileAnalyses.reduce((sum, file) => sum + file.sizeInBytes, 0);
+        const averageTokensPerByte = totalSizeInBytes > 0 ? totalTokens / totalSizeInBytes : 0;
+        // Estimate total tokens with overhead
+        const estimatedTotalTokens = totalTokens + promptOverhead;
+        // Determine if chunking is needed
+        const exceedsContextWindow = estimatedTotalTokens > effectiveContextWindowSize;
+        logger_1.default.info(`Token analysis summary:`);
+        logger_1.default.info(`- Total files: ${files.length}`);
+        logger_1.default.info(`- Total tokens: ${totalTokens.toLocaleString()}`);
+        logger_1.default.info(`- Prompt overhead: ${promptOverhead.toLocaleString()}`);
+        logger_1.default.info(`- Estimated total tokens: ${estimatedTotalTokens.toLocaleString()}`);
+        logger_1.default.info(`- Context window size: ${contextWindowSize.toLocaleString()}`);
+        logger_1.default.info(`- Effective context size (with safety margin): ${effectiveContextWindowSize.toLocaleString()}`);
+        logger_1.default.info(`- Context utilization: ${((estimatedTotalTokens / effectiveContextWindowSize) * 100).toFixed(2)}%`);
+        // Calculate recommended chunks if needed
+        const chunkingRecommendation = TokenAnalyzer.generateChunkingRecommendation(fileAnalyses, estimatedTotalTokens, effectiveContextWindowSize, contextMaintenanceFactor, options.forceSinglePass, options.batchTokenLimit);
+        // Log chunking decision
+        if (chunkingRecommendation.chunkingRecommended) {
+            logger_1.default.info(`Multi-pass review recommended: ${chunkingRecommendation.reason}`);
+            logger_1.default.info(`Estimated passes needed: ${chunkingRecommendation.recommendedChunks.length}`);
+        }
+        else {
+            logger_1.default.info(`Single-pass review recommended: ${chunkingRecommendation.reason}`);
+        }
+        return {
+            files: fileAnalyses,
+            totalTokens,
+            totalSizeInBytes,
+            averageTokensPerByte,
+            fileCount: files.length,
+            promptOverheadTokens: promptOverhead,
+            estimatedTotalTokens,
+            contextWindowSize,
+            exceedsContextWindow,
+            estimatedPassesNeeded: chunkingRecommendation.recommendedChunks.length,
+            chunkingRecommendation,
+        };
+    }
+    /**
+     * Generate a chunking recommendation for files that exceed context window
+     * @param fileAnalyses Array of file token analyses
+     * @param estimatedTotalTokens Total tokens including overhead
+     * @param contextWindowSize Maximum context window size
+     * @param contextMaintenanceFactor Context maintenance overhead factor
+     * @param forceSinglePass Force single pass mode regardless of token count
+     * @param batchTokenLimit Force maximum tokens per batch (for testing)
+     * @returns Chunking recommendation
+     */
+    static generateChunkingRecommendation(fileAnalyses, estimatedTotalTokens, contextWindowSize, contextMaintenanceFactor, forceSinglePass, batchTokenLimit) {
+        // If forceSinglePass is true, skip chunking regardless of token count
+        if (forceSinglePass) {
+            logger_1.default.debug(`Forcing single-pass review mode as requested (forceSinglePass=true)`);
+            return {
+                chunkingRecommended: false,
+                recommendedChunks: [
+                    {
+                        files: fileAnalyses.map((f) => f.path),
+                        estimatedTokenCount: estimatedTotalTokens,
+                        priority: 1,
+                    },
+                ],
+                reason: 'Single-pass mode forced by configuration',
+            };
+        }
+        // If batchTokenLimit is provided, use it to force smaller batches
+        let effectiveContextLimit = contextWindowSize;
+        if (batchTokenLimit && batchTokenLimit > 0) {
+            effectiveContextLimit = Math.min(batchTokenLimit, contextWindowSize);
+            logger_1.default.info(`Using batch token limit: ${batchTokenLimit.toLocaleString()} tokens (forcing smaller batches for testing)`);
+            if (batchTokenLimit < contextWindowSize) {
+                logger_1.default.info(`This will force chunking even if content would fit in the model's context window`);
+            }
+        }
+        // If content fits within context window, no chunking needed
+        if (estimatedTotalTokens <= effectiveContextLimit) {
+            logger_1.default.debug(`Content fits within effective limit (${estimatedTotalTokens.toLocaleString()} <= ${effectiveContextLimit.toLocaleString()} tokens)`);
+            return {
+                chunkingRecommended: false,
+                recommendedChunks: [
+                    {
+                        files: fileAnalyses.map((f) => f.path),
+                        estimatedTokenCount: estimatedTotalTokens,
+                        priority: 1,
+                    },
+                ],
+                reason: batchTokenLimit
+                    ? 'Content fits within batch token limit'
+                    : 'Content fits within model context window',
+            };
+        }
+        logger_1.default.debug(`Content exceeds effective limit (${estimatedTotalTokens.toLocaleString()} > ${effectiveContextLimit.toLocaleString()} tokens)`);
+        logger_1.default.debug(`Generating chunking recommendation with context maintenance factor: ${contextMaintenanceFactor}`);
+        // Calculate effective context window size accounting for context maintenance
+        const effectiveContextSize = Math.floor(effectiveContextLimit * (1 - contextMaintenanceFactor));
+        logger_1.default.debug(`Effective context size for chunking: ${effectiveContextSize.toLocaleString()} tokens (${Math.round((1 - contextMaintenanceFactor) * 100)}% of ${effectiveContextLimit.toLocaleString()} tokens)`);
+        // Use optimized bin-packing algorithm for better chunk distribution
+        const chunks = TokenAnalyzer.optimizedBinPacking(fileAnalyses, effectiveContextSize, effectiveContextLimit);
+        logger_1.default.info(`Created ${chunks.length} optimized chunks for multi-pass review`);
+        let reason = `Content exceeds effective limit (${estimatedTotalTokens.toLocaleString()} > ${effectiveContextLimit.toLocaleString()} tokens)`;
+        if (batchTokenLimit && batchTokenLimit < contextWindowSize) {
+            reason = `Batch token limit forcing smaller batches (limit: ${batchTokenLimit.toLocaleString()} tokens)`;
+        }
+        return {
+            chunkingRecommended: true,
+            recommendedChunks: chunks,
+            reason,
+        };
+    }
+    /**
+     * Optimized bin-packing algorithm to minimize the number of chunks
+     * Uses an advanced first-fit decreasing with multi-level optimization
+     * @param fileAnalyses Array of file token analyses
+     * @param maxChunkSize Maximum size for each chunk in tokens
+     * @param contextWindowSize Original context window for logging
+     * @returns Array of optimized file chunks
+     */
+    static optimizedBinPacking(fileAnalyses, maxChunkSize, _contextWindowSize) {
+        // Sort files by token count (largest first) for first-fit decreasing
+        const sortedFiles = [...fileAnalyses].sort((a, b) => b.tokenCount - a.tokenCount);
+        // Calculate target chunk size for optimal distribution
+        const totalTokens = sortedFiles.reduce((sum, f) => sum + f.tokenCount, 0);
+        const minChunksNeeded = Math.ceil(totalTokens / maxChunkSize);
+        const targetChunkSize = Math.floor(totalTokens / minChunksNeeded);
+        logger_1.default.debug(`Bin-packing optimization:`);
+        logger_1.default.debug(`  - Total tokens: ${totalTokens.toLocaleString()}`);
+        logger_1.default.debug(`  - Max chunk size: ${maxChunkSize.toLocaleString()}`);
+        logger_1.default.debug(`  - Min chunks needed: ${minChunksNeeded}`);
+        logger_1.default.debug(`  - Target chunk size: ${targetChunkSize.toLocaleString()}`);
+        // Initialize chunks array
+        const chunks = [];
+        // Track oversized files separately
+        const oversizedFiles = [];
+        const largeFiles = [];
+        const mediumFiles = [];
+        const smallFiles = [];
+        // Categorize files by size for better packing
+        for (const file of sortedFiles) {
+            if (file.tokenCount > maxChunkSize) {
+                oversizedFiles.push(file);
+                logger_1.default.warn(`File "${file.path}" is oversized (${file.tokenCount.toLocaleString()} > ${maxChunkSize.toLocaleString()} tokens)`);
+            }
+            else if (file.tokenCount > maxChunkSize * 0.5) {
+                largeFiles.push(file);
+            }
+            else if (file.tokenCount > maxChunkSize * 0.2) {
+                mediumFiles.push(file);
+            }
+            else {
+                smallFiles.push(file);
+            }
+        }
+        logger_1.default.debug(`File categorization:`);
+        logger_1.default.debug(`  - Oversized: ${oversizedFiles.length}`);
+        logger_1.default.debug(`  - Large (>50% of max): ${largeFiles.length}`);
+        logger_1.default.debug(`  - Medium (20-50% of max): ${mediumFiles.length}`);
+        logger_1.default.debug(`  - Small (<20% of max): ${smallFiles.length}`);
+        // Process oversized files first (split them if possible)
+        for (const file of oversizedFiles) {
+            // For now, put oversized files in their own chunks
+            // TODO: In future, we could split file content
+            chunks.push({
+                files: [file.path],
+                estimatedTokenCount: file.tokenCount,
+                priority: chunks.length + 1,
+            });
+            logger_1.default.debug(`Created dedicated chunk ${chunks.length} for oversized file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
+        }
+        // Process large files - try to pair them optimally
+        for (const file of largeFiles) {
+            let placed = false;
+            // Try to find a chunk with complementary space
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                const remainingSpace = maxChunkSize - chunk.estimatedTokenCount;
+                // Check if this file fits well (within 80% efficiency)
+                if (remainingSpace >= file.tokenCount &&
+                    chunk.estimatedTokenCount + file.tokenCount >= targetChunkSize * 0.8) {
+                    chunk.files.push(file.path);
+                    chunk.estimatedTokenCount += file.tokenCount;
+                    placed = true;
+                    logger_1.default.debug(`Added large file "${file.path}" (${file.tokenCount.toLocaleString()} tokens) to chunk ${i + 1}`);
+                    break;
+                }
+            }
+            if (!placed) {
+                // Create a new chunk for this large file
+                chunks.push({
+                    files: [file.path],
+                    estimatedTokenCount: file.tokenCount,
+                    priority: chunks.length + 1,
+                });
+                logger_1.default.debug(`Created new chunk ${chunks.length} for large file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
+            }
+        }
+        // Process medium files - use first-fit with efficiency threshold
+        for (const file of mediumFiles) {
+            let placed = false;
+            // Find first chunk where this file fits efficiently
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                const remainingSpace = maxChunkSize - chunk.estimatedTokenCount;
+                if (remainingSpace >= file.tokenCount) {
+                    chunk.files.push(file.path);
+                    chunk.estimatedTokenCount += file.tokenCount;
+                    placed = true;
+                    logger_1.default.debug(`Added medium file "${file.path}" (${file.tokenCount.toLocaleString()} tokens) to chunk ${i + 1}`);
+                    break;
+                }
+            }
+            if (!placed) {
+                // Create a new chunk
+                chunks.push({
+                    files: [file.path],
+                    estimatedTokenCount: file.tokenCount,
+                    priority: chunks.length + 1,
+                });
+                logger_1.default.debug(`Created new chunk ${chunks.length} for medium file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
+            }
+        }
+        // Process small files - pack them to fill gaps
+        // Sort small files for better packing (largest first)
+        smallFiles.sort((a, b) => b.tokenCount - a.tokenCount);
+        for (const file of smallFiles) {
+            let placed = false;
+            // Find the fullest chunk that can still fit this file
+            let bestChunkIndex = -1;
+            let bestChunkFullness = 0;
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                const remainingSpace = maxChunkSize - chunk.estimatedTokenCount;
+                const chunkFullness = chunk.estimatedTokenCount / maxChunkSize;
+                if (remainingSpace >= file.tokenCount && chunkFullness > bestChunkFullness) {
+                    bestChunkIndex = i;
+                    bestChunkFullness = chunkFullness;
+                }
+            }
+            if (bestChunkIndex !== -1) {
+                const chunk = chunks[bestChunkIndex];
+                chunk.files.push(file.path);
+                chunk.estimatedTokenCount += file.tokenCount;
+                placed = true;
+                logger_1.default.debug(`Added small file "${file.path}" (${file.tokenCount.toLocaleString()} tokens) to chunk ${bestChunkIndex + 1}`);
+            }
+            if (!placed) {
+                // Create a new chunk only if absolutely necessary
+                chunks.push({
+                    files: [file.path],
+                    estimatedTokenCount: file.tokenCount,
+                    priority: chunks.length + 1,
+                });
+                logger_1.default.debug(`Created new chunk ${chunks.length} for small file "${file.path}" (${file.tokenCount.toLocaleString()} tokens)`);
+            }
+        }
+        // Perform aggressive balancing to minimize chunk count
+        const balancedChunks = TokenAnalyzer.aggressiveBalance(chunks, fileAnalyses, maxChunkSize);
+        // Log chunk statistics
+        const avgTokensPerChunk = Math.round(balancedChunks.reduce((sum, c) => sum + c.estimatedTokenCount, 0) / balancedChunks.length);
+        const maxTokensInChunk = Math.max(...balancedChunks.map((c) => c.estimatedTokenCount));
+        const minTokensInChunk = Math.min(...balancedChunks.map((c) => c.estimatedTokenCount));
+        logger_1.default.info(`Chunk statistics:`);
+        logger_1.default.info(`  - Total chunks: ${balancedChunks.length}`);
+        logger_1.default.info(`  - Average tokens per chunk: ${avgTokensPerChunk.toLocaleString()}`);
+        logger_1.default.info(`  - Max tokens in a chunk: ${maxTokensInChunk.toLocaleString()}`);
+        logger_1.default.info(`  - Min tokens in a chunk: ${minTokensInChunk.toLocaleString()}`);
+        logger_1.default.info(`  - Chunk efficiency: ${((avgTokensPerChunk / maxChunkSize) * 100).toFixed(1)}%`);
+        return balancedChunks;
+    }
+    /**
+     * Aggressive balancing to minimize chunk count and maximize efficiency
+     * @param chunks Initial chunks from bin-packing
+     * @param fileAnalyses Original file analyses for lookup
+     * @param maxChunkSize Maximum size for each chunk
+     * @returns Balanced chunks with minimized count
+     */
+    static aggressiveBalance(chunks, fileAnalyses, maxChunkSize) {
+        // Create a map for quick file lookups
+        const fileMap = new Map();
+        for (const file of fileAnalyses) {
+            fileMap.set(file.path, file);
+        }
+        // First pass: Try to merge small chunks
+        const mergedChunks = [];
+        const sortedForMerging = [...chunks].sort((a, b) => a.estimatedTokenCount - b.estimatedTokenCount);
+        const usedChunks = new Set();
+        for (let i = 0; i < sortedForMerging.length; i++) {
+            if (usedChunks.has(i))
+                continue;
+            const chunk1 = sortedForMerging[i];
+            const mergedChunk = {
+                files: [...chunk1.files],
+                estimatedTokenCount: chunk1.estimatedTokenCount,
+                priority: mergedChunks.length + 1,
+            };
+            usedChunks.add(i);
+            // Try to merge with other small chunks
+            for (let j = i + 1; j < sortedForMerging.length; j++) {
+                if (usedChunks.has(j))
+                    continue;
+                const chunk2 = sortedForMerging[j];
+                const combinedSize = mergedChunk.estimatedTokenCount + chunk2.estimatedTokenCount;
+                // Merge if combined size is still within limits
+                if (combinedSize <= maxChunkSize) {
+                    mergedChunk.files.push(...chunk2.files);
+                    mergedChunk.estimatedTokenCount = combinedSize;
+                    usedChunks.add(j);
+                    logger_1.default.debug(`Merged chunks: ${chunk2.files.length} files (${chunk2.estimatedTokenCount.toLocaleString()} tokens) into chunk with ${mergedChunk.files.length} files`);
+                }
+            }
+            mergedChunks.push(mergedChunk);
+        }
+        logger_1.default.debug(`Chunk merging reduced count from ${chunks.length} to ${mergedChunks.length}`);
+        // Second pass: Balance the merged chunks
+        const sortedChunks = [...mergedChunks].sort((a, b) => a.estimatedTokenCount - b.estimatedTokenCount);
+        // Try to move files to achieve better balance
+        let improved = true;
+        let iterations = 0;
+        const maxIterations = 20; // More iterations for aggressive optimization
+        while (improved && iterations < maxIterations) {
+            improved = false;
+            iterations++;
+            // Find the most and least full chunks
+            sortedChunks.sort((a, b) => a.estimatedTokenCount - b.estimatedTokenCount);
+            for (let i = 0; i < Math.floor(sortedChunks.length / 2); i++) {
+                const smallChunk = sortedChunks[i];
+                const largeChunk = sortedChunks[sortedChunks.length - 1 - i];
+                // Calculate variance threshold based on chunk count
+                const varianceThreshold = Math.max(500, maxChunkSize * 0.05); // 5% of max or 500 tokens
+                // Skip if chunks are already well balanced
+                if (largeChunk.estimatedTokenCount - smallChunk.estimatedTokenCount < varianceThreshold) {
+                    continue;
+                }
+                // Try to find optimal file to move
+                let bestFile = null;
+                let bestImprovement = 0;
+                for (const filePath of largeChunk.files) {
+                    const file = fileMap.get(filePath);
+                    if (!file)
+                        continue;
+                    const newSmallSize = smallChunk.estimatedTokenCount + file.tokenCount;
+                    const newLargeSize = largeChunk.estimatedTokenCount - file.tokenCount;
+                    // Check if moving this file would improve balance
+                    if (newSmallSize <= maxChunkSize && newLargeSize > 0) {
+                        const currentDiff = largeChunk.estimatedTokenCount - smallChunk.estimatedTokenCount;
+                        const newDiff = Math.abs(newLargeSize - newSmallSize);
+                        const improvement = currentDiff - newDiff;
+                        if (improvement > bestImprovement) {
+                            bestFile = filePath;
+                            bestImprovement = improvement;
+                        }
+                    }
+                }
+                // Move the best file if found
+                if (bestFile && bestImprovement > 100) {
+                    const file = fileMap.get(bestFile);
+                    largeChunk.files = largeChunk.files.filter((f) => f !== bestFile);
+                    smallChunk.files.push(bestFile);
+                    // Update token counts
+                    largeChunk.estimatedTokenCount -= file.tokenCount;
+                    smallChunk.estimatedTokenCount += file.tokenCount;
+                    logger_1.default.debug(`Balanced: Moved file "${bestFile}" (${file.tokenCount.toLocaleString()} tokens) - improvement: ${bestImprovement.toLocaleString()} tokens`);
+                    improved = true;
+                }
+            }
+        }
+        // Final pass: Remove any empty chunks
+        const finalChunks = sortedChunks.filter((chunk) => chunk.files.length > 0);
+        // Re-assign priorities based on token count (largest first for processing)
+        finalChunks.sort((a, b) => b.estimatedTokenCount - a.estimatedTokenCount);
+        finalChunks.forEach((chunk, index) => {
+            chunk.priority = index + 1;
+        });
+        if (iterations === maxIterations) {
+            logger_1.default.debug(`Aggressive balancing stopped after ${maxIterations} iterations`);
+        }
+        else {
+            logger_1.default.debug(`Aggressive balancing completed in ${iterations} iterations`);
+        }
+        return finalChunks;
+    }
+    /**
+     * Analyze a single file for token usage
+     * @param file File to analyze
+     * @param options Analysis options
+     * @returns Token analysis for the file
+     */
+    static analyzeFile(file, options) {
+        const content = file.content;
+        const tokenCount = (0, tokenizers_1.countTokens)(content, options.modelName);
+        const sizeInBytes = content.length;
+        return {
+            path: file.path,
+            relativePath: file.relativePath,
+            tokenCount,
+            sizeInBytes,
+            tokensPerByte: sizeInBytes > 0 ? tokenCount / sizeInBytes : 0,
+        };
+    }
+}
+exports.TokenAnalyzer = TokenAnalyzer;
+//# sourceMappingURL=TokenAnalyzer.js.map

package/dist/analysis/tokens/TokenAnalyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenAnalyzer.js","sourceRoot":"","sources":["../../../src/analysis/tokens/TokenAnalyzer.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;AAEH,iDAA+C;AAE/C,gEAAwC;AA4FxC;;GAEG;AACH,MAAa,aAAa;IAChB,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC;IACtC,MAAM,CAAC,kCAAkC,GAAG,IAAI,CAAC,CAAC,sCAAsC;IACxF,MAAM,CAAC,4BAA4B,GAAG,GAAG,CAAC,CAAC,uCAAuC;IAClF,MAAM,CAAC,sBAAsB,GAAG,MAAM,CAAC,CAAC,mBAAmB;IACnE;;;OAGG;IACK,MAAM,CAAC,qBAAqB,GAAkC;QACpE,sBAAsB,EAAE,MAAM;QAC9B,eAAe,EAAE,MAAM;QACvB,4BAA4B,EAAE,MAAM;QACpC,kBAAkB,EAAE,MAAM;QAC1B,6BAA6B,EAAE,MAAM;QACrC,kBAAkB,EAAE,MAAM;QAC1B,kBAAkB,EAAE,OAAO;QAC3B,aAAa,EAAE,OAAO;KACvB,CAAC;IAEF;;;;OAIG;IACK,MAAM,CAAC,oBAAoB,CAAC,SAAiB;QACnD,gBAAM,CAAC,KAAK,CAAC,mCAAmC,SAAS,EAAE,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,aAAa,CAAC,6BAA6B,CAAC,SAAS,CAAC,CAAC;QAE1E,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACnC,gBAAM,CAAC,IAAI,CAAC,uCAAuC,SAAS,KAAK,UAAU,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YACvG,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,gBAAM,CAAC,IAAI,CACT,2BAA2B,SAAS,iGAAiG,CACtI,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,oDAAoD,SAAS,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,gBAAM,CAAC,IAAI,CACT,sCAAsC,aAAa,CAAC,sBAAsB,CAAC,cAAc,EAAE,SAAS,CACrG,CAAC;QACF,OAAO,aAAa,CAAC,sBAAsB,CAAC;IAC9C,CAAC;IAEO,MAAM,CAAC,6BAA6B,CAAC,SAAiB;QAC5D,MAAM,UAAU,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,UAAU,IAAI,aAAa,CAAC,qBAAqB,EAAE,CAAC;YACtD,OAAO,aAAa,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,EAAE,QAAQ,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,QAAQ,IAAI,QAAQ,IAAI,aAAa,CAAC,qBAAqB,EAAE,CAAC;gBAChE,OAAO,aAAa,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,YAAY,CACxB,KAAiB,EACjB,OAA6B;QAE7B,gBAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QAClD,gBAAM,CAAC,KAAK,CAAC,4BAA4B,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAE9D,MAAM,iBAAiB,GAAG,aAAa,CAAC,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChF,MAAM,cAAc,GAClB,OAAO,CAAC,wBAAwB,IAAI,aAAa,CAAC,uBAAuB,CAAC;QAC5E,MAAM,wBAAwB,GAC5B,OAAO,CAAC,wBAAwB,IAAI,aAAa,CAAC,kCAAkC,CAAC;QACvF,MAAM,kBAAkB,GACtB,OAAO,CAAC,kBAAkB,IAAI,aAAa,CAAC,4BAA4B,CAAC;QAE3E,6DAA6D;QAC7D,MAAM,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC;QAC5F,gBAAM,CAAC,IAAI,CACT,wCAAwC,0BAA0B,CAAC,cAAc,EAAE,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,GAAG,GAAG,CAAC,QAAQ,iBAAiB,CAAC,cAAc,EAAE,UAAU,CAC9L,CAAC;QAEF,oBAAoB;QACpB,MAAM,YAAY,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAA,wBAAW,EAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YACnC,MAAM,aAAa,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAErE,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,UAAU;gBACV,WAAW;gBACX,aAAa;aACd,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,mBAAmB;QACnB,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACjF,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACvF,MAAM,oBAAoB,GAAG,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvF,sCAAsC;QACtC,MAAM,oBAAoB,GAAG,WAAW,GAAG,cAAc,CAAC;QAE1D,kCAAkC;QAClC,MAAM,oBAAoB,GAAG,oBAAoB,GAAG,0BAA0B,CAAC;QAE/E,gBAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACvC,gBAAM,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,gBAAM,CAAC,IAAI,CAAC,mBAAmB,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/D,gBAAM,CAAC,IAAI,CAAC,sBAAsB,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACrE,gBAAM,CAAC,IAAI,CAAC,6BAA6B,oBAAoB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAClF,gBAAM,CAAC,IAAI,CAAC,0BAA0B,iBAAiB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC5E,gBAAM,CAAC,IAAI,CACT,kDAAkD,0BAA0B,CAAC,cAAc,EAAE,EAAE,CAChG,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,0BAA0B,CAAC,CAAC,oBAAoB,GAAG,0BAA0B,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACpG,CAAC;QAEF,yCAAyC;QACzC,MAAM,sBAAsB,GAAG,aAAa,CAAC,8BAA8B,CACzE,YAAY,EACZ,oBAAoB,EACpB,0BAA0B,EAC1B,wBAAwB,EACxB,OAAO,CAAC,eAAe,EACvB,OAAO,CAAC,eAAe,CACxB,CAAC;QAEF,wBAAwB;QACxB,IAAI,sBAAsB,CAAC,mBAAmB,EAAE,CAAC;YAC/C,gBAAM,CAAC,IAAI,CAAC,kCAAkC,sBAAsB,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/E,gBAAM,CAAC,IAAI,CAAC,4BAA4B,sBAAsB,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,mCAAmC,sBAAsB,CAAC,MAAM,EAAE,CAAC,CAAC;QAClF,CAAC;QAED,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,WAAW;YACX,gBAAgB;YAChB,oBAAoB;YACpB,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,oBAAoB,EAAE,cAAc;YACpC,oBAAoB;YACpB,iBAAiB;YACjB,oBAAoB;YACpB,qBAAqB,EAAE,sBAAsB,CAAC,iBAAiB,CAAC,MAAM;YACtE,sBAAsB;SACvB,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACK,MAAM,CAAC,8BAA8B,CAC3C,YAAiC,EACjC,oBAA4B,EAC5B,iBAAyB,EACzB,wBAAgC,EAChC,eAAyB,EACzB,eAAwB;QAExB,sEAAsE;QACtE,IAAI,eAAe,EAAE,CAAC;YACpB,gBAAM,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACpF,OAAO;gBACL,mBAAmB,EAAE,KAAK;gBAC1B,iBAAiB,EAAE;oBACjB;wBACE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;wBACtC,mBAAmB,EAAE,oBAAoB;wBACzC,QAAQ,EAAE,CAAC;qBACZ;iBACF;gBACD,MAAM,EAAE,0CAA0C;aACnD,CAAC;QACJ,CAAC;QAED,kEAAkE;QAClE,IAAI,qBAAqB,GAAG,iBAAiB,CAAC;QAC9C,IAAI,eAAe,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YAC3C,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;YACrE,gBAAM,CAAC,IAAI,CACT,4BAA4B,eAAe,CAAC,cAAc,EAAE,+CAA+C,CAC5G,CAAC;YACF,IAAI,eAAe,GAAG,iBAAiB,EAAE,CAAC;gBACxC,gBAAM,CAAC,IAAI,CACT,kFAAkF,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,IAAI,oBAAoB,IAAI,qBAAqB,EAAE,CAAC;YAClD,gBAAM,CAAC,KAAK,CACV,wCAAwC,oBAAoB,CAAC,cAAc,EAAE,OAAO,qBAAqB,CAAC,cAAc,EAAE,UAAU,CACrI,CAAC;YACF,OAAO;gBACL,mBAAmB,EAAE,KAAK;gBAC1B,iBAAiB,EAAE;oBACjB;wBACE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;wBACtC,mBAAmB,EAAE,oBAAoB;wBACzC,QAAQ,EAAE,CAAC;qBACZ;iBACF;gBACD,MAAM,EAAE,eAAe;oBACrB,CAAC,CAAC,uCAAuC;oBACzC,CAAC,CAAC,0CAA0C;aAC/C,CAAC;QACJ,CAAC;QAED,gBAAM,CAAC,KAAK,CACV,oCAAoC,oBAAoB,CAAC,cAAc,EAAE,MAAM,qBAAqB,CAAC,cAAc,EAAE,UAAU,CAChI,CAAC;QACF,gBAAM,CAAC,KAAK,CACV,uEAAuE,wBAAwB,EAAE,CAClG,CAAC;QAEF,6EAA6E;QAC7E,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,CAAC,CAAC,GAAG,wBAAwB,CAAC,CAAC,CAAC;QAEhG,gBAAM,CAAC,KAAK,CACV,wCAAwC,oBAAoB,CAAC,cAAc,EAAE,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,wBAAwB,CAAC,GAAG,GAAG,CAAC,QAAQ,qBAAqB,CAAC,cAAc,EAAE,UAAU,CAClM,CAAC;QAEF,oEAAoE;QACpE,MAAM,MAAM,GAAG,aAAa,CAAC,mBAAmB,CAC9C,YAAY,EACZ,oBAAoB,EACpB,qBAAqB,CACtB,CAAC;QAEF,gBAAM,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,MAAM,yCAAyC,CAAC,CAAC;QAE/E,IAAI,MAAM,GAAG,oCAAoC,oBAAoB,CAAC,cAAc,EAAE,MAAM,qBAAqB,CAAC,cAAc,EAAE,UAAU,CAAC;QAC7I,IAAI,eAAe,IAAI,eAAe,GAAG,iBAAiB,EAAE,CAAC;YAC3D,MAAM,GAAG,qDAAqD,eAAe,CAAC,cAAc,EAAE,UAAU,CAAC;QAC3G,CAAC;QAED,OAAO;YACL,mBAAmB,EAAE,IAAI;YACzB,iBAAiB,EAAE,MAAM;YACzB,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACK,MAAM,CAAC,mBAAmB,CAChC,YAAiC,EACjC,YAAoB,EACpB,kBAA0B;QAE1B,qEAAqE;QACrE,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAElF,uDAAuD;QACvD,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,eAAe,CAAC,CAAC;QAElE,gBAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC1C,gBAAM,CAAC,KAAK,CAAC,qBAAqB,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAClE,gBAAM,CAAC,KAAK,CAAC,uBAAuB,YAAY,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACrE,gBAAM,CAAC,KAAK,CAAC,0BAA0B,eAAe,EAAE,CAAC,CAAC;QAC1D,gBAAM,CAAC,KAAK,CAAC,0BAA0B,eAAe,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAE3E,0BAA0B;QAC1B,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,mCAAmC;QACnC,MAAM,cAAc,GAAwB,EAAE,CAAC;QAC/C,MAAM,UAAU,GAAwB,EAAE,CAAC;QAC3C,MAAM,WAAW,GAAwB,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAwB,EAAE,CAAC;QAE3C,8CAA8C;QAC9C,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,UAAU,GAAG,YAAY,EAAE,CAAC;gBACnC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,gBAAM,CAAC,IAAI,CACT,SAAS,IAAI,CAAC,IAAI,mBAAmB,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,MAAM,YAAY,CAAC,cAAc,EAAE,UAAU,CACnH,CAAC;YACJ,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC;gBAChD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC;gBAChD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,gBAAM,CAAC,KAAK,CAAC,kBAAkB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QACxD,gBAAM,CAAC,KAAK,CAAC,4BAA4B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,gBAAM,CAAC,KAAK,CAAC,+BAA+B,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAClE,gBAAM,CAAC,KAAK,CAAC,4BAA4B,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9D,yDAAyD;QACzD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,mDAAmD;YACnD,+CAA+C;YAC/C,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;gBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;aAC5B,CAAC,CAAC;YACH,gBAAM,CAAC,KAAK,CACV,2BAA2B,MAAM,CAAC,MAAM,wBAAwB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAC1H,CAAC;QACJ,CAAC;QAED,mDAAmD;QACnD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,MAAM,GAAG,KAAK,CAAC;YAEnB,+CAA+C;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,cAAc,GAAG,YAAY,GAAG,KAAK,CAAC,mBAAmB,CAAC;gBAEhE,uDAAuD;gBACvD,IACE,cAAc,IAAI,IAAI,CAAC,UAAU;oBACjC,KAAK,CAAC,mBAAmB,GAAG,IAAI,CAAC,UAAU,IAAI,eAAe,GAAG,GAAG,EACpE,CAAC;oBACD,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5B,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAC7C,MAAM,GAAG,IAAI,CAAC;oBACd,gBAAM,CAAC,KAAK,CACV,qBAAqB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,qBAAqB,CAAC,GAAG,CAAC,EAAE,CACjG,CAAC;oBACF,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;oBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBACH,gBAAM,CAAC,KAAK,CACV,qBAAqB,MAAM,CAAC,MAAM,oBAAoB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAChH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,iEAAiE;QACjE,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,MAAM,GAAG,KAAK,CAAC;YAEnB,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,cAAc,GAAG,YAAY,GAAG,KAAK,CAAC,mBAAmB,CAAC;gBAEhE,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACtC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5B,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAC7C,MAAM,GAAG,IAAI,CAAC;oBACd,gBAAM,CAAC,KAAK,CACV,sBAAsB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAClG,CAAC;oBACF,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,qBAAqB;gBACrB,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;oBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBACH,gBAAM,CAAC,KAAK,CACV,qBAAqB,MAAM,CAAC,MAAM,qBAAqB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CACjH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,sDAAsD;QACtD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAEvD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,MAAM,GAAG,KAAK,CAAC;YAEnB,sDAAsD;YACtD,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC;YACxB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,cAAc,GAAG,YAAY,GAAG,KAAK,CAAC,mBAAmB,CAAC;gBAChE,MAAM,aAAa,GAAG,KAAK,CAAC,mBAAmB,GAAG,YAAY,CAAC;gBAE/D,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,IAAI,aAAa,GAAG,iBAAiB,EAAE,CAAC;oBAC3E,cAAc,GAAG,CAAC,CAAC;oBACnB,iBAAiB,GAAG,aAAa,CAAC;gBACpC,CAAC;YACH,CAAC;YAED,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;gBACrC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;gBAC7C,MAAM,GAAG,IAAI,CAAC;gBACd,gBAAM,CAAC,KAAK,CACV,qBAAqB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,qBAAqB,cAAc,GAAG,CAAC,EAAE,CAC9G,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kDAAkD;gBAClD,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,mBAAmB,EAAE,IAAI,CAAC,UAAU;oBACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBACH,gBAAM,CAAC,KAAK,CACV,qBAAqB,MAAM,CAAC,MAAM,oBAAoB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAChH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,uDAAuD;QACvD,MAAM,cAAc,GAAG,aAAa,CAAC,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;QAE3F,uBAAuB;QACvB,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAClC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM,CAC1F,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACvF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAEvF,gBAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACjC,gBAAM,CAAC,IAAI,CAAC,qBAAqB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1D,gBAAM,CAAC,IAAI,CAAC,iCAAiC,iBAAiB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACnF,gBAAM,CAAC,IAAI,CAAC,8BAA8B,gBAAgB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/E,gBAAM,CAAC,IAAI,CAAC,8BAA8B,gBAAgB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/E,gBAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,iBAAiB,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE/F,OAAO,cAAc,CAAC;IACxB,CAAC;IAED;;;;;;OAMG;IACK,MAAM,CAAC,iBAAiB,CAC9B,MAAmB,EACnB,YAAiC,EACjC,YAAoB;QAEpB,sCAAsC;QACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;QACrD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC/B,CAAC;QAED,wCAAwC;QACxC,MAAM,YAAY,GAAgB,EAAE,CAAC;QACrC,MAAM,gBAAgB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CACvC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CACxD,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEhC,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,WAAW,GAAc;gBAC7B,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;gBACxB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,QAAQ,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC;aAClC,CAAC;YACF,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAElB,uCAAuC;YACvC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrD,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAEhC,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,YAAY,GAAG,WAAW,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAC;gBAElF,gDAAgD;gBAChD,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;oBACjC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;oBACxC,WAAW,CAAC,mBAAmB,GAAG,YAAY,CAAC;oBAC/C,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBAClB,gBAAM,CAAC,KAAK,CACV,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,4BAA4B,WAAW,CAAC,KAAK,CAAC,MAAM,QAAQ,CACxJ,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,oCAAoC,MAAM,CAAC,MAAM,OAAO,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QAE5F,yCAAyC;QACzC,MAAM,YAAY,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CACxD,CAAC;QAEF,8CAA8C;QAC9C,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,aAAa,GAAG,EAAE,CAAC,CAAC,8CAA8C;QAExE,OAAO,QAAQ,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;YAC9C,QAAQ,GAAG,KAAK,CAAC;YACjB,UAAU,EAAE,CAAC;YAEb,sCAAsC;YACtC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC;YAE3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7D,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,UAAU,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBAE7D,oDAAoD;gBACpD,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,0BAA0B;gBAExF,2CAA2C;gBAC3C,IAAI,UAAU,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,GAAG,iBAAiB,EAAE,CAAC;oBACxF,SAAS;gBACX,CAAC;gBAED,mCAAmC;gBACnC,IAAI,QAAQ,GAAkB,IAAI,CAAC;gBACnC,IAAI,eAAe,GAAG,CAAC,CAAC;gBAExB,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;oBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBACnC,IAAI,CAAC,IAAI;wBAAE,SAAS;oBAEpB,MAAM,YAAY,GAAG,UAAU,CAAC,mBAAmB,GAAG,IAAI,CAAC,UAAU,CAAC;oBACtE,MAAM,YAAY,GAAG,UAAU,CAAC,mBAAmB,GAAG,IAAI,CAAC,UAAU,CAAC;oBAEtE,kDAAkD;oBAClD,IAAI,YAAY,IAAI,YAAY,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;wBACrD,MAAM,WAAW,GAAG,UAAU,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;wBACpF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC;wBACtD,MAAM,WAAW,GAAG,WAAW,GAAG,OAAO,CAAC;wBAE1C,IAAI,WAAW,GAAG,eAAe,EAAE,CAAC;4BAClC,QAAQ,GAAG,QAAQ,CAAC;4BACpB,eAAe,GAAG,WAAW,CAAC;wBAChC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,8BAA8B;gBAC9B,IAAI,QAAQ,IAAI,eAAe,GAAG,GAAG,EAAE,CAAC;oBACtC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;oBACpC,UAAU,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC;oBAClE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAEhC,sBAAsB;oBACtB,UAAU,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAClD,UAAU,CAAC,mBAAmB,IAAI,IAAI,CAAC,UAAU,CAAC;oBAElD,gBAAM,CAAC,KAAK,CACV,yBAAyB,QAAQ,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,2BAA2B,eAAe,CAAC,cAAc,EAAE,SAAS,CAC5I,CAAC;oBAEF,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE3E,2EAA2E;QAC3E,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC1E,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACnC,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,IAAI,UAAU,KAAK,aAAa,EAAE,CAAC;YACjC,gBAAM,CAAC,KAAK,CAAC,sCAAsC,aAAa,aAAa,CAAC,CAAC;QACjF,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,KAAK,CAAC,qCAAqC,UAAU,aAAa,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,WAAW,CAAC,IAAc,EAAE,OAA6B;QACrE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAA,wBAAW,EAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,UAAU;YACV,WAAW;YACX,aAAa,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;SAC9D,CAAC;IACJ,CAAC;;AAjoBH,sCAkoBC"}

package/{src/analysis/tokens/index.ts → dist/analysis/tokens/index.d.ts}

@@ -3,6 +3,5 @@
  *
  * This module exports all token analysis related functions and types.
  */
-
 export * from './TokenAnalysisFormatter';
 export * from './TokenAnalyzer';

package/dist/analysis/tokens/index.js

@@ -0,0 +1,24 @@
+"use strict";
+/**
+ * @fileoverview Token analysis module exports.
+ *
+ * This module exports all token analysis related functions and types.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./TokenAnalysisFormatter"), exports);
+__exportStar(require("./TokenAnalyzer"), exports);
+//# sourceMappingURL=index.js.map

package/dist/analysis/tokens/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/analysis/tokens/index.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;;;;;;AAEH,2DAAyC;AACzC,kDAAgC"}