codereview-aia 0.1.3 → 0.2.0

package/{src/analysis/tokens/TokenAnalysisFormatter.ts → dist/analysis/tokens/TokenAnalysisFormatter.js}

@@ -1,36 +1,36 @@
+"use strict";
 /**
  * @fileoverview Formatter for token analysis results.
  *
  * This module provides utilities for formatting token analysis results into
  * human-readable strings and other formats for display or logging.
  */
-
-import type { FileTokenAnalysis, TokenAnalysisResult } from './TokenAnalyzer';
-
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatTokenAnalysis = formatTokenAnalysis;
+exports.formatTokenAnalysisAsJson = formatTokenAnalysisAsJson;
+exports.formatFileTokenAnalysis = formatFileTokenAnalysis;
 /**
  * Format a number with commas as thousands separators
  * @param num Number to format
  * @returns Formatted number string
  */
-function formatNumber(num: number): string {
-  return num.toLocaleString();
+function formatNumber(num) {
+    return num.toLocaleString();
 }
-
 /**
  * Format a file size in bytes to a human-readable string
  * @param bytes Size in bytes
  * @returns Formatted size string (e.g., "1.23 KB")
  */
-function formatFileSize(bytes: number): string {
-  if (bytes < 1024) {
-    return `${bytes} B`;
-  }
-  if (bytes < 1024 * 1024) {
-    return `${(bytes / 1024).toFixed(2)} KB`;
-  }
-  return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+function formatFileSize(bytes) {
+    if (bytes < 1024) {
+        return `${bytes} B`;
+    }
+    if (bytes < 1024 * 1024) {
+        return `${(bytes / 1024).toFixed(2)} KB`;
+    }
+    return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
 }
-
 /**
  * Format token analysis result as a human-readable string
  * @param analysis Token analysis result
@@ -38,34 +38,29 @@ function formatFileSize(bytes: number): string {
  * @param includedFiles Whether to include detailed file listing
  * @returns Formatted string representation of the analysis
  */
-export function formatTokenAnalysis(
-  analysis: TokenAnalysisResult,
-  modelName: string,
-  includeFiles = false,
-): string {
-  // Extract provider and model from both colon and slash formats
-  // Handle formats: "provider:model", "provider/model", or just "model"
-  let provider: string | undefined;
-  let model: string;
-
-  if (modelName.includes(':')) {
-    // Traditional format: "provider:model"
-    [provider, model] = modelName.split(':', 2);
-  } else if (modelName.includes('/')) {
-    // OpenRouter format: "provider/model"
-    [provider, model] = modelName.split('/', 2);
-  } else {
-    // Just model name
-    provider = undefined;
-    model = modelName;
-  }
-
-  const displayModel = model || modelName;
-  const displayProvider = provider
-    ? `${provider.charAt(0).toUpperCase() + provider.slice(1)}`
-    : 'Unknown';
-
-  let output = `
+function formatTokenAnalysis(analysis, modelName, includeFiles = false) {
+    // Extract provider and model from both colon and slash formats
+    // Handle formats: "provider:model", "provider/model", or just "model"
+    let provider;
+    let model;
+    if (modelName.includes(':')) {
+        // Traditional format: "provider:model"
+        [provider, model] = modelName.split(':', 2);
+    }
+    else if (modelName.includes('/')) {
+        // OpenRouter format: "provider/model"
+        [provider, model] = modelName.split('/', 2);
+    }
+    else {
+        // Just model name
+        provider = undefined;
+        model = modelName;
+    }
+    const displayModel = model || modelName;
+    const displayProvider = provider
+        ? `${provider.charAt(0).toUpperCase() + provider.slice(1)}`
+        : 'Unknown';
+    let output = `
 === Token Analysis Report ===
 
 Provider: ${displayProvider}
@@ -82,73 +77,67 @@ Context Utilization:
   ${((analysis.estimatedTotalTokens / analysis.contextWindowSize) * 100).toFixed(2)}% of context window used
 
 `;
-
-  // Add chunking information if recommended
-  if (analysis.chunkingRecommendation.chunkingRecommended) {
-    output += `
+    // Add chunking information if recommended
+    if (analysis.chunkingRecommendation.chunkingRecommended) {
+        output += `
 Multi-Pass Analysis:
   Chunking Required: Yes
   Reason: ${analysis.chunkingRecommendation.reason}
   Estimated Passes: ${formatNumber(analysis.estimatedPassesNeeded)}
 `;
-
-    // Add chunk details
-    analysis.chunkingRecommendation.recommendedChunks.forEach((chunk, index) => {
-      output += `
+        // Add chunk details
+        analysis.chunkingRecommendation.recommendedChunks.forEach((chunk, index) => {
+            output += `
   Chunk ${index + 1}:
     Files: ${formatNumber(chunk.files.length)}
     Estimated Tokens: ${formatNumber(chunk.estimatedTokenCount)}
     Priority: ${chunk.priority}
 `;
-    });
-  } else {
-    output += `
+        });
+    }
+    else {
+        output += `
 Multi-Pass Analysis:
   Chunking Required: No
   Reason: ${analysis.chunkingRecommendation.reason}
 `;
-  }
-
-  // Add file details if requested
-  if (includeFiles) {
-    output += `
+    }
+    // Add file details if requested
+    if (includeFiles) {
+        output += `
 File Details:
 `;
-
-    // Sort files by token count (largest first)
-    const sortedFiles = [...analysis.files].sort((a, b) => b.tokenCount - a.tokenCount);
-
-    sortedFiles.forEach((file) => {
-      output += `  ${file.relativePath}:
+        // Sort files by token count (largest first)
+        const sortedFiles = [...analysis.files].sort((a, b) => b.tokenCount - a.tokenCount);
+        sortedFiles.forEach((file) => {
+            output += `  ${file.relativePath}:
     Tokens: ${formatNumber(file.tokenCount)}
     Size: ${formatFileSize(file.sizeInBytes)}
     Tokens/Byte: ${file.tokensPerByte.toFixed(2)}
 `;
-    });
-  }
-
-  return output;
+        });
+    }
+    return output;
 }
-
 /**
  * Format token analysis result as JSON
  * @param analysis Token analysis result
  * @returns JSON string representation of the analysis
  */
-export function formatTokenAnalysisAsJson(analysis: TokenAnalysisResult): string {
-  return JSON.stringify(analysis, null, 2);
+function formatTokenAnalysisAsJson(analysis) {
+    return JSON.stringify(analysis, null, 2);
 }
-
 /**
  * Format a single file token analysis as a string
  * @param fileAnalysis File token analysis
  * @returns Formatted string representation of the file analysis
  */
-export function formatFileTokenAnalysis(fileAnalysis: FileTokenAnalysis): string {
-  return `
+function formatFileTokenAnalysis(fileAnalysis) {
+    return `
 File: ${fileAnalysis.relativePath}
 Tokens: ${formatNumber(fileAnalysis.tokenCount)}
 Size: ${formatFileSize(fileAnalysis.sizeInBytes)}
 Tokens/Byte: ${fileAnalysis.tokensPerByte.toFixed(2)}
 `;
 }
+//# sourceMappingURL=TokenAnalysisFormatter.js.map

package/dist/analysis/tokens/TokenAnalysisFormatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenAnalysisFormatter.js","sourceRoot":"","sources":["../../../src/analysis/tokens/TokenAnalysisFormatter.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AAmCH,kDA0FC;AAOD,8DAEC;AAOD,0DAOC;AAhJD;;;;GAIG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG,CAAC,cAAc,EAAE,CAAC;AAC9B,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;QACjB,OAAO,GAAG,KAAK,IAAI,CAAC;IACtB,CAAC;IACD,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AACpD,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,mBAAmB,CACjC,QAA6B,EAC7B,SAAiB,EACjB,YAAY,GAAG,KAAK;IAEpB,+DAA+D;IAC/D,sEAAsE;IACtE,IAAI,QAA4B,CAAC;IACjC,IAAI,KAAa,CAAC;IAElB,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,uCAAuC;QACvC,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,sCAAsC;QACtC,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAC9C,CAAC;SAAM,CAAC;QACN,kBAAkB;QAClB,QAAQ,GAAG,SAAS,CAAC;QACrB,KAAK,GAAG,SAAS,CAAC;IACpB,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,IAAI,SAAS,CAAC;IACxC,MAAM,eAAe,GAAG,QAAQ;QAC9B,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QAC3D,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,MAAM,GAAG;;;YAGH,eAAe;SAClB,YAAY;SACZ,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,cAAc,CAAC,QAAQ,CAAC,gBAAgB,CAAC;;;oBAGnE,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC;qBACjC,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC;4BACpC,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC;yBAC9C,YAAY,CAAC,QAAQ,CAAC,iBAAiB,CAAC;;;IAG7D,CAAC,CAAC,QAAQ,CAAC,oBAAoB,GAAG,QAAQ,CAAC,iBAAiB,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;;CAElF,CAAC;IAEA,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,sBAAsB,CAAC,mBAAmB,EAAE,CAAC;QACxD,MAAM,IAAI;;;YAGF,QAAQ,CAAC,sBAAsB,CAAC,MAAM;sBAC5B,YAAY,CAAC,QAAQ,CAAC,qBAAqB,CAAC;CACjE,CAAC;QAEE,oBAAoB;QACpB,QAAQ,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzE,MAAM,IAAI;UACN,KAAK,GAAG,CAAC;aACN,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;wBACrB,YAAY,CAAC,KAAK,CAAC,mBAAmB,CAAC;gBAC/C,KAAK,CAAC,QAAQ;CAC7B,CAAC;QACE,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,IAAI;;;YAGF,QAAQ,CAAC,sBAAsB,CAAC,MAAM;CACjD,CAAC;IACA,CAAC;IAED,gCAAgC;IAChC,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,IAAI;;CAEb,CAAC;QAEE,4CAA4C;QAC5C,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAEpF,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,IAAI,KAAK,IAAI,CAAC,YAAY;cACxB,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;YAC/B,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC;mBACzB,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;CAC/C,CAAC;QACE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,QAA6B;IACrE,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,SAAgB,uBAAuB,CAAC,YAA+B;IACrE,OAAO;QACD,YAAY,CAAC,YAAY;UACvB,YAAY,CAAC,YAAY,CAAC,UAAU,CAAC;QACvC,cAAc,CAAC,YAAY,CAAC,WAAW,CAAC;eACjC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;CACnD,CAAC;AACF,CAAC"}

package/dist/analysis/tokens/TokenAnalyzer.d.ts

@@ -0,0 +1,155 @@
+/**
+ * @fileoverview Token analysis service for pre-review token counting and estimation.
+ *
+ * This module provides fast, provider-agnostic token counting and analysis functionality
+ * to estimate token usage and costs before performing actual reviews.
+ */
+import type { FileInfo } from '../../types/review';
+/**
+ * Result of token analysis for a single file
+ */
+export interface FileTokenAnalysis {
+    /** Path to the file */
+    path: string;
+    /** Relative path to the file */
+    relativePath: string | undefined;
+    /** Number of tokens in the file */
+    tokenCount: number;
+    /** Size of file in bytes */
+    sizeInBytes: number;
+    /** Tokens per byte ratio (used for optimization analysis) */
+    tokensPerByte: number;
+}
+/**
+ * Result of token analysis for a set of files
+ */
+export interface TokenAnalysisResult {
+    /** Analysis of individual files */
+    files: FileTokenAnalysis[];
+    /** Total number of tokens across all files */
+    totalTokens: number;
+    /** Total size of all files in bytes */
+    totalSizeInBytes: number;
+    /** Average tokens per byte across all files */
+    averageTokensPerByte: number;
+    /** Total number of files analyzed */
+    fileCount: number;
+    /** Token overhead for prompts, instructions, etc. */
+    promptOverheadTokens: number;
+    /** Estimated total token count including overhead */
+    estimatedTotalTokens: number;
+    /** Maximum context window size for the model */
+    contextWindowSize: number;
+    /** Whether the content exceeds the context window */
+    exceedsContextWindow: boolean;
+    /** Number of passes needed for multi-pass review */
+    estimatedPassesNeeded: number;
+    /** Chunking strategy recommendation */
+    chunkingRecommendation: ChunkingRecommendation;
+}
+/**
+ * Recommendation for chunking strategy
+ */
+export interface ChunkingRecommendation {
+    /** Whether chunking is recommended */
+    chunkingRecommended: boolean;
+    /** Approximate file chunks for multi-pass processing */
+    recommendedChunks: FileChunk[];
+    /** Reason for chunking recommendation */
+    reason: string;
+}
+/**
+ * A chunk of files for multi-pass processing
+ */
+export interface FileChunk {
+    /** Files in this chunk */
+    files: string[];
+    /** Estimated token count for this chunk */
+    estimatedTokenCount: number;
+    /** Priority of this chunk (higher = more important) */
+    priority: number;
+}
+/**
+ * Options for token analysis
+ */
+export interface TokenAnalysisOptions {
+    /** Type of review being performed */
+    reviewType: string;
+    /** Name of the model being used */
+    modelName: string;
+    /** Whether to optimize for speed (less accurate) or precision */
+    optimizeForSpeed?: boolean;
+    /** Additional prompt overhead to consider */
+    additionalPromptOverhead?: number;
+    /** Context maintenance factor for multi-pass reviews (0-1) */
+    contextMaintenanceFactor?: number;
+    /** Safety margin factor for context window (0-1) */
+    safetyMarginFactor?: number;
+    /** Force single pass mode regardless of token count */
+    forceSinglePass?: boolean;
+    /** Force maximum tokens per batch (for testing consolidation) */
+    batchTokenLimit?: number;
+}
+/**
+ * Service for analyzing token usage in files
+ */
+export declare class TokenAnalyzer {
+    private static DEFAULT_PROMPT_OVERHEAD;
+    private static DEFAULT_CONTEXT_MAINTENANCE_FACTOR;
+    private static DEFAULT_SAFETY_MARGIN_FACTOR;
+    private static DEFAULT_CONTEXT_WINDOW;
+    /**
+     * Context window mapping for supported models.
+     * Leave values as null if the exact window is unknown so they can be filled in later.
+     */
+    private static MODEL_CONTEXT_WINDOWS;
+    /**
+     * Get the context window size for a model
+     * @param modelName Name of the model
+     * @returns Context window size in tokens
+     */
+    private static getContextWindowSize;
+    private static lookupConfiguredContextWindow;
+    /**
+     * Analyze token usage for a set of files
+     * @param files Files to analyze
+     * @param options Analysis options
+     * @returns Token analysis result
+     */
+    static analyzeFiles(files: FileInfo[], options: TokenAnalysisOptions): TokenAnalysisResult;
+    /**
+     * Generate a chunking recommendation for files that exceed context window
+     * @param fileAnalyses Array of file token analyses
+     * @param estimatedTotalTokens Total tokens including overhead
+     * @param contextWindowSize Maximum context window size
+     * @param contextMaintenanceFactor Context maintenance overhead factor
+     * @param forceSinglePass Force single pass mode regardless of token count
+     * @param batchTokenLimit Force maximum tokens per batch (for testing)
+     * @returns Chunking recommendation
+     */
+    private static generateChunkingRecommendation;
+    /**
+     * Optimized bin-packing algorithm to minimize the number of chunks
+     * Uses an advanced first-fit decreasing with multi-level optimization
+     * @param fileAnalyses Array of file token analyses
+     * @param maxChunkSize Maximum size for each chunk in tokens
+     * @param contextWindowSize Original context window for logging
+     * @returns Array of optimized file chunks
+     */
+    private static optimizedBinPacking;
+    /**
+     * Aggressive balancing to minimize chunk count and maximize efficiency
+     * @param chunks Initial chunks from bin-packing
+     * @param fileAnalyses Original file analyses for lookup
+     * @param maxChunkSize Maximum size for each chunk
+     * @returns Balanced chunks with minimized count
+     */
+    private static aggressiveBalance;
+    /**
+     * Analyze a single file for token usage
+     * @param file File to analyze
+     * @param options Analysis options
+     * @returns Token analysis for the file
+     */
+    static analyzeFile(file: FileInfo, options: TokenAnalysisOptions): FileTokenAnalysis;
+}