@git.zone/tsdoc 1.5.2 → 1.6.1

package/ts/context/enhanced-context.ts

@@ -1,7 +1,10 @@
 import * as plugins from '../plugins.js';
-import type { ContextMode, IContextResult, IFileInfo, TaskType } from './types.js';
+import type { ContextMode, IContextResult, IFileInfo, TaskType, IFileMetadata } from './types.js';
 import { ContextTrimmer } from './context-trimmer.js';
 import { ConfigManager } from './config-manager.js';
+import { LazyFileLoader } from './lazy-file-loader.js';
+import { ContextCache } from './context-cache.js';
+import { ContextAnalyzer } from './context-analyzer.js';
 
 /**
  * Enhanced ProjectContext that supports context optimization strategies
@@ -10,6 +13,9 @@ export class EnhancedContext {
   private projectDir: string;
   private trimmer: ContextTrimmer;
   private configManager: ConfigManager;
+  private lazyLoader: LazyFileLoader;
+  private cache: ContextCache;
+  private analyzer: ContextAnalyzer;
   private contextMode: ContextMode = 'trimmed';
   private tokenBudget: number = 190000; // Default for o4-mini
   private contextResult: IContextResult = {
@@ -29,6 +35,13 @@ export class EnhancedContext {
     this.projectDir = projectDirArg;
     this.configManager = ConfigManager.getInstance();
     this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig());
+    this.lazyLoader = new LazyFileLoader(projectDirArg);
+    this.cache = new ContextCache(projectDirArg, this.configManager.getCacheConfig());
+    this.analyzer = new ContextAnalyzer(
+      projectDirArg,
+      this.configManager.getPrioritizationWeights(),
+      this.configManager.getTierConfig()
+    );
   }
   
   /**
@@ -38,6 +51,7 @@ export class EnhancedContext {
     await this.configManager.initialize(this.projectDir);
     this.tokenBudget = this.configManager.getMaxTokens();
     this.trimmer.updateConfig(this.configManager.getTrimConfig());
+    await this.cache.init();
   }
   
   /**
@@ -55,75 +69,17 @@ export class EnhancedContext {
   public setTokenBudget(maxTokens: number): void {
     this.tokenBudget = maxTokens;
   }
-  
-  /**
-   * Gather files from the project
-   * @param includePaths Optional paths to include
-   * @param excludePaths Optional paths to exclude
-   */
-  public async gatherFiles(includePaths?: string[], excludePaths?: string[]): Promise<Record<string, plugins.smartfile.SmartFile | plugins.smartfile.SmartFile[]>> {
-    const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'package.json'),
-      this.projectDir,
-    );
-    
-    const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'readme.md'),
-      this.projectDir,
-    );
 
-    const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'readme.hints.md'),
-      this.projectDir,
-    );
-    
-    const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'npmextra.json'),
-      this.projectDir,
-    );
-    
-    // Use provided include paths or default to all TypeScript files
-    const includeGlobs = includePaths?.map(path => `${path}/**/*.ts`) || ['ts*/**/*.ts'];
-    
-    // Get TypeScript files
-    const smartfilesModPromises = includeGlobs.map(glob => 
-      plugins.smartfile.fs.fileTreeToObject(this.projectDir, glob)
-    );
-    
-    const smartfilesModArrays = await Promise.all(smartfilesModPromises);
-    
-    // Flatten the arrays
-    const smartfilesMod: plugins.smartfile.SmartFile[] = [];
-    smartfilesModArrays.forEach(array => {
-      smartfilesMod.push(...array);
-    });
-    
-    // Get test files if not excluded
-    let smartfilesTest: plugins.smartfile.SmartFile[] = [];
-    if (!excludePaths?.includes('test/')) {
-      smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
-        this.projectDir,
-        'test/**/*.ts',
-      );
-    }
-    
-    return {
-      smartfilePackageJSON,
-      smartfilesReadme,
-      smartfilesReadmeHints,
-      smartfilesNpmextraJSON,
-      smartfilesMod,
-      smartfilesTest,
-    };
-  }
-  
   /**
-   * Convert files to context string
-   * @param files The files to convert
-   * @param mode The context mode to use
+   * Convert files to context with smart analysis and prioritization
+   * @param metadata - File metadata to analyze
+   * @param taskType - Task type for context-aware prioritization
+   * @param mode - Context mode to use
+   * @returns Context string
    */
-  public async convertFilesToContext(
-    files: plugins.smartfile.SmartFile[],
+  public async convertFilesToContextWithAnalysis(
+    metadata: IFileMetadata[],
+    taskType: TaskType,
     mode: ContextMode = this.contextMode
   ): Promise<string> {
     // Reset context result
@@ -135,140 +91,173 @@ export class EnhancedContext {
       excludedFiles: [],
       tokenSavings: 0
     };
-    
+
+    // Analyze files for smart prioritization
+    const analysis = await this.analyzer.analyze(metadata, taskType, []);
+
+    // Sort files by importance score (highest first)
+    const sortedAnalysis = [...analysis.files].sort(
+      (a, b) => b.importanceScore - a.importanceScore
+    );
+
+    // Filter out excluded tier
+    const relevantFiles = sortedAnalysis.filter(f => f.tier !== 'excluded');
+
     let totalTokenCount = 0;
     let totalOriginalTokens = 0;
-    
-    // Sort files by importance (for now just a simple alphabetical sort)
-    // Later this could be enhanced with more sophisticated prioritization
-    const sortedFiles = [...files].sort((a, b) => a.relative.localeCompare(b.relative));
-    
     const processedFiles: string[] = [];
-    
-    for (const smartfile of sortedFiles) {
-      // Calculate original token count
-      const originalContent = smartfile.contents.toString();
-      const originalTokenCount = this.countTokens(originalContent);
-      totalOriginalTokens += originalTokenCount;
-      
-      // Apply trimming based on mode
-      let processedContent = originalContent;
-      
-      if (mode !== 'full') {
-        processedContent = this.trimmer.trimFile(
-          smartfile.relative,
-          originalContent,
-          mode
-        );
-      }
-      
-      // Calculate new token count
-      const processedTokenCount = this.countTokens(processedContent);
-      
-      // Check if we have budget for this file
-      if (totalTokenCount + processedTokenCount > this.tokenBudget) {
-        // We don't have budget for this file
-        this.contextResult.excludedFiles.push({
-          path: smartfile.path,
-          contents: originalContent,
-          relativePath: smartfile.relative,
-          tokenCount: originalTokenCount
-        });
-        continue;
-      }
-      
-      // Format the file for context
-      const formattedContent = `
-====== START OF FILE ${smartfile.relative} ======
-  
+
+    // Load files with cache support
+    for (const fileAnalysis of relevantFiles) {
+      try {
+        // Check cache first
+        let contents: string;
+        let originalTokenCount: number;
+
+        const cached = await this.cache.get(fileAnalysis.path);
+        if (cached) {
+          contents = cached.contents;
+          originalTokenCount = cached.tokenCount;
+        } else {
+          // Load file
+          const fileData = await plugins.smartfile.fs.toStringSync(fileAnalysis.path);
+          contents = fileData;
+          originalTokenCount = this.countTokens(contents);
+
+          // Cache it
+          await this.cache.set({
+            path: fileAnalysis.path,
+            contents,
+            tokenCount: originalTokenCount,
+            mtime: Date.now(),
+            cachedAt: Date.now()
+          });
+        }
+
+        totalOriginalTokens += originalTokenCount;
+
+        // Apply tier-based trimming
+        let processedContent = contents;
+        let trimLevel: 'none' | 'light' | 'aggressive' = 'light';
+
+        if (fileAnalysis.tier === 'essential') {
+          trimLevel = 'none';
+        } else if (fileAnalysis.tier === 'important') {
+          trimLevel = 'light';
+        } else if (fileAnalysis.tier === 'optional') {
+          trimLevel = 'aggressive';
+        }
+
+        // Apply trimming based on mode and tier
+        if (mode !== 'full' && trimLevel !== 'none') {
+          const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
+          processedContent = this.trimmer.trimFileWithLevel(
+            relativePath,
+            contents,
+            trimLevel
+          );
+        }
+
+        // Calculate token count
+        const processedTokenCount = this.countTokens(processedContent);
+
+        // Check token budget
+        if (totalTokenCount + processedTokenCount > this.tokenBudget) {
+          // We don't have budget for this file
+          const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
+          this.contextResult.excludedFiles.push({
+            path: fileAnalysis.path,
+            contents,
+            relativePath,
+            tokenCount: originalTokenCount,
+            importanceScore: fileAnalysis.importanceScore
+          });
+          continue;
+        }
+
+        // Format the file for context
+        const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
+        const formattedContent = `
+====== START OF FILE ${relativePath} ======
+
 ${processedContent}
-  
-====== END OF FILE ${smartfile.relative} ======
-      `;
-      
-      processedFiles.push(formattedContent);
-      totalTokenCount += processedTokenCount;
-      
-      // Track file in appropriate list
-      const fileInfo: IFileInfo = {
-        path: smartfile.path,
-        contents: processedContent,
-        relativePath: smartfile.relative,
-        tokenCount: processedTokenCount
-      };
-      
-      if (mode === 'full' || processedContent === originalContent) {
-        this.contextResult.includedFiles.push(fileInfo);
-      } else {
-        this.contextResult.trimmedFiles.push(fileInfo);
-        this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
+
+====== END OF FILE ${relativePath} ======
+        `;
+
+        processedFiles.push(formattedContent);
+        totalTokenCount += processedTokenCount;
+
+        // Track file in appropriate list
+        const fileInfo: IFileInfo = {
+          path: fileAnalysis.path,
+          contents: processedContent,
+          relativePath,
+          tokenCount: processedTokenCount,
+          importanceScore: fileAnalysis.importanceScore
+        };
+
+        if (trimLevel === 'none' || processedContent === contents) {
+          this.contextResult.includedFiles.push(fileInfo);
+        } else {
+          this.contextResult.trimmedFiles.push(fileInfo);
+          this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
+        }
+      } catch (error) {
+        console.warn(`Failed to process file ${fileAnalysis.path}:`, error.message);
       }
     }
-    
+
     // Join all processed files
     const context = processedFiles.join('\n');
-    
+
     // Update context result
     this.contextResult.context = context;
     this.contextResult.tokenCount = totalTokenCount;
-    
+
     return context;
   }
-  
+
   /**
-   * Build context for the project
-   * @param taskType Optional task type for task-specific context
+   * Build context for the project using smart analysis
+   * @param taskType Task type for context-aware prioritization (defaults to 'description')
    */
   public async buildContext(taskType?: TaskType): Promise<IContextResult> {
     // Initialize if needed
     if (this.tokenBudget === 0) {
       await this.initialize();
     }
-    
-    // Get task-specific configuration if a task type is provided
-    if (taskType) {
-      const taskConfig = this.configManager.getTaskConfig(taskType);
-      if (taskConfig.mode) {
-        this.setContextMode(taskConfig.mode);
-      }
-    }
-    
-    // Gather files
-    const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
-    const files = await this.gatherFiles(
-      taskConfig?.includePaths,
-      taskConfig?.excludePaths
-    );
-    
-    // Convert files to context
-    // Create an array of all files to process
-    const allFiles: plugins.smartfile.SmartFile[] = [];
-    
-    // Add individual files
-    if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
-    if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
-    if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
-    if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
-    
-    // Add arrays of files
-    if (files.smartfilesMod) {
-      if (Array.isArray(files.smartfilesMod)) {
-        allFiles.push(...files.smartfilesMod);
-      } else {
-        allFiles.push(files.smartfilesMod);
-      }
-    }
-    
-    if (files.smartfilesTest) {
-      if (Array.isArray(files.smartfilesTest)) {
-        allFiles.push(...files.smartfilesTest);
-      } else {
-        allFiles.push(files.smartfilesTest);
-      }
+
+    // Smart context building always requires a task type for optimal prioritization
+    // Default to 'description' if not provided
+    const effectiveTaskType = taskType || 'description';
+
+    // Get task-specific configuration
+    const taskConfig = this.configManager.getTaskConfig(effectiveTaskType);
+    if (taskConfig.mode) {
+      this.setContextMode(taskConfig.mode);
     }
-    
-    const context = await this.convertFilesToContext(allFiles);
-    
+
+    // Build globs for scanning
+    const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
+      'ts/**/*.ts',
+      'ts*/**/*.ts'
+    ];
+
+    // Add config files
+    const configGlobs = [
+      'package.json',
+      'readme.md',
+      'readme.hints.md',
+      'npmextra.json'
+    ];
+
+    // Scan files for metadata (fast, doesn't load contents)
+    const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
+
+    // Use smart analyzer to build context with intelligent prioritization
+    await this.convertFilesToContextWithAnalysis(metadata, effectiveTaskType, this.contextMode);
+
     return this.contextResult;
   }
   

package/ts/context/index.ts

@@ -2,14 +2,27 @@ import { EnhancedContext } from './enhanced-context.js';
 import { TaskContextFactory } from './task-context-factory.js';
 import { ConfigManager } from './config-manager.js';
 import { ContextTrimmer } from './context-trimmer.js';
-import type { 
-  ContextMode, 
-  IContextConfig, 
+import { LazyFileLoader } from './lazy-file-loader.js';
+import { ContextCache } from './context-cache.js';
+import { ContextAnalyzer } from './context-analyzer.js';
+import type {
+  ContextMode,
+  IContextConfig,
   IContextResult,
   IFileInfo,
   ITrimConfig,
   ITaskConfig,
-  TaskType
+  TaskType,
+  ICacheConfig,
+  IAnalyzerConfig,
+  IPrioritizationWeights,
+  ITierConfig,
+  ITierSettings,
+  IFileMetadata,
+  ICacheEntry,
+  IFileDependencies,
+  IFileAnalysis,
+  IAnalysisResult
 } from './types.js';
 
 export {
@@ -18,6 +31,9 @@ export {
   TaskContextFactory,
   ConfigManager,
   ContextTrimmer,
+  LazyFileLoader,
+  ContextCache,
+  ContextAnalyzer,
 };
 
 // Types
@@ -28,5 +44,15 @@ export type {
   IFileInfo,
   ITrimConfig,
   ITaskConfig,
-  TaskType
+  TaskType,
+  ICacheConfig,
+  IAnalyzerConfig,
+  IPrioritizationWeights,
+  ITierConfig,
+  ITierSettings,
+  IFileMetadata,
+  ICacheEntry,
+  IFileDependencies,
+  IFileAnalysis,
+  IAnalysisResult
 };

package/ts/context/lazy-file-loader.ts

@@ -0,0 +1,191 @@
+import * as plugins from '../plugins.js';
+import * as fs from 'fs';
+import type { IFileMetadata, IFileInfo } from './types.js';
+
+/**
+ * LazyFileLoader handles efficient file loading by:
+ * - Scanning files for metadata without loading contents
+ * - Providing fast file size and token estimates
+ * - Loading contents only when requested
+ * - Parallel loading of selected files
+ */
+export class LazyFileLoader {
+  private projectRoot: string;
+  private metadataCache: Map<string, IFileMetadata> = new Map();
+
+  /**
+   * Creates a new LazyFileLoader
+   * @param projectRoot - Root directory of the project
+   */
+  constructor(projectRoot: string) {
+    this.projectRoot = projectRoot;
+  }
+
+  /**
+   * Scans files in given globs and creates metadata without loading contents
+   * @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts'])
+   * @returns Array of file metadata
+   */
+  public async scanFiles(globs: string[]): Promise<IFileMetadata[]> {
+    const metadata: IFileMetadata[] = [];
+
+    for (const globPattern of globs) {
+      try {
+        const smartFiles = await plugins.smartfile.fs.fileTreeToObject(this.projectRoot, globPattern);
+        const fileArray = Array.isArray(smartFiles) ? smartFiles : [smartFiles];
+
+        for (const smartFile of fileArray) {
+          try {
+            const meta = await this.getMetadata(smartFile.path);
+            metadata.push(meta);
+          } catch (error) {
+            // Skip files that can't be read
+            console.warn(`Failed to get metadata for ${smartFile.path}:`, error.message);
+          }
+        }
+      } catch (error) {
+        // Skip patterns that don't match any files
+        console.warn(`No files found for pattern ${globPattern}`);
+      }
+    }
+
+    return metadata;
+  }
+
+  /**
+   * Gets metadata for a single file without loading contents
+   * @param filePath - Absolute path to the file
+   * @returns File metadata
+   */
+  public async getMetadata(filePath: string): Promise<IFileMetadata> {
+    // Check cache first
+    if (this.metadataCache.has(filePath)) {
+      const cached = this.metadataCache.get(filePath)!;
+      const currentStats = await fs.promises.stat(filePath);
+
+      // Return cached if file hasn't changed
+      if (cached.mtime === Math.floor(currentStats.mtimeMs)) {
+        return cached;
+      }
+    }
+
+    // Get file stats
+    const stats = await fs.promises.stat(filePath);
+    const relativePath = plugins.path.relative(this.projectRoot, filePath);
+
+    // Estimate tokens: rough estimate of ~4 characters per token
+    // This is faster than reading and tokenizing the entire file
+    const estimatedTokens = Math.ceil(stats.size / 4);
+
+    const metadata: IFileMetadata = {
+      path: filePath,
+      relativePath,
+      size: stats.size,
+      mtime: Math.floor(stats.mtimeMs),
+      estimatedTokens,
+    };
+
+    // Cache the metadata
+    this.metadataCache.set(filePath, metadata);
+
+    return metadata;
+  }
+
+  /**
+   * Loads file contents for selected files in parallel
+   * @param metadata - Array of file metadata to load
+   * @param tokenizer - Function to calculate accurate token count
+   * @returns Array of complete file info with contents
+   */
+  public async loadFiles(
+    metadata: IFileMetadata[],
+    tokenizer: (content: string) => number
+  ): Promise<IFileInfo[]> {
+    // Load files in parallel
+    const loadPromises = metadata.map(async (meta) => {
+      try {
+        const contents = await plugins.smartfile.fs.toStringSync(meta.path);
+        const tokenCount = tokenizer(contents);
+
+        const fileInfo: IFileInfo = {
+          path: meta.path,
+          relativePath: meta.relativePath,
+          contents,
+          tokenCount,
+          importanceScore: meta.importanceScore,
+        };
+
+        return fileInfo;
+      } catch (error) {
+        console.warn(`Failed to load file ${meta.path}:`, error.message);
+        return null;
+      }
+    });
+
+    // Wait for all loads to complete and filter out failures
+    const results = await Promise.all(loadPromises);
+    return results.filter((r): r is IFileInfo => r !== null);
+  }
+
+  /**
+   * Loads a single file with contents
+   * @param filePath - Absolute path to the file
+   * @param tokenizer - Function to calculate accurate token count
+   * @returns Complete file info with contents
+   */
+  public async loadFile(
+    filePath: string,
+    tokenizer: (content: string) => number
+  ): Promise<IFileInfo> {
+    const meta = await this.getMetadata(filePath);
+    const contents = await plugins.smartfile.fs.toStringSync(filePath);
+    const tokenCount = tokenizer(contents);
+    const relativePath = plugins.path.relative(this.projectRoot, filePath);
+
+    return {
+      path: filePath,
+      relativePath,
+      contents,
+      tokenCount,
+      importanceScore: meta.importanceScore,
+    };
+  }
+
+  /**
+   * Updates importance scores for metadata entries
+   * @param scores - Map of file paths to importance scores
+   */
+  public updateImportanceScores(scores: Map<string, number>): void {
+    for (const [path, score] of scores) {
+      const meta = this.metadataCache.get(path);
+      if (meta) {
+        meta.importanceScore = score;
+      }
+    }
+  }
+
+  /**
+   * Clears the metadata cache
+   */
+  public clearCache(): void {
+    this.metadataCache.clear();
+  }
+
+  /**
+   * Gets total estimated tokens for all cached metadata
+   */
+  public getTotalEstimatedTokens(): number {
+    let total = 0;
+    for (const meta of this.metadataCache.values()) {
+      total += meta.estimatedTokens;
+    }
+    return total;
+  }
+
+  /**
+   * Gets cached metadata entries
+   */
+  public getCachedMetadata(): IFileMetadata[] {
+    return Array.from(this.metadataCache.values());
+  }
+}