@git.zone/tsdoc 1.5.1 → 1.6.0

package/ts/context/context-cache.ts

@@ -0,0 +1,285 @@
+import * as plugins from '../plugins.js';
+import * as fs from 'fs';
+import type { ICacheEntry, ICacheConfig } from './types.js';
+
+/**
+ * ContextCache provides persistent caching of file contents and token counts
+ * with automatic invalidation on file changes
+ */
+export class ContextCache {
+  private cacheDir: string;
+  private cache: Map<string, ICacheEntry> = new Map();
+  private config: Required<ICacheConfig>;
+  private cacheIndexPath: string;
+
+  /**
+   * Creates a new ContextCache
+   * @param projectRoot - Root directory of the project
+   * @param config - Cache configuration
+   */
+  constructor(projectRoot: string, config: Partial<ICacheConfig> = {}) {
+    this.config = {
+      enabled: config.enabled ?? true,
+      ttl: config.ttl ?? 3600, // 1 hour default
+      maxSize: config.maxSize ?? 100, // 100MB default
+      directory: config.directory ?? plugins.path.join(projectRoot, '.nogit', 'context-cache'),
+    };
+
+    this.cacheDir = this.config.directory;
+    this.cacheIndexPath = plugins.path.join(this.cacheDir, 'index.json');
+  }
+
+  /**
+   * Initializes the cache by loading from disk
+   */
+  public async init(): Promise<void> {
+    if (!this.config.enabled) {
+      return;
+    }
+
+    // Ensure cache directory exists
+    await plugins.smartfile.fs.ensureDir(this.cacheDir);
+
+    // Load cache index if it exists
+    try {
+      const indexExists = await plugins.smartfile.fs.fileExists(this.cacheIndexPath);
+      if (indexExists) {
+        const indexContent = await plugins.smartfile.fs.toStringSync(this.cacheIndexPath);
+        const indexData = JSON.parse(indexContent) as ICacheEntry[];
+        if (Array.isArray(indexData)) {
+          for (const entry of indexData) {
+            this.cache.set(entry.path, entry);
+          }
+        }
+      }
+    } catch (error) {
+      console.warn('Failed to load cache index:', error.message);
+      // Start with empty cache if loading fails
+    }
+
+    // Clean up expired and invalid entries
+    await this.cleanup();
+  }
+
+  /**
+   * Gets a cached entry if it's still valid
+   * @param filePath - Absolute path to the file
+   * @returns Cache entry if valid, null otherwise
+   */
+  public async get(filePath: string): Promise<ICacheEntry | null> {
+    if (!this.config.enabled) {
+      return null;
+    }
+
+    const entry = this.cache.get(filePath);
+    if (!entry) {
+      return null;
+    }
+
+    // Check if entry is expired
+    const now = Date.now();
+    if (now - entry.cachedAt > this.config.ttl * 1000) {
+      this.cache.delete(filePath);
+      return null;
+    }
+
+    // Check if file has been modified
+    try {
+      const stats = await fs.promises.stat(filePath);
+      const currentMtime = Math.floor(stats.mtimeMs);
+
+      if (currentMtime !== entry.mtime) {
+        // File has changed, invalidate cache
+        this.cache.delete(filePath);
+        return null;
+      }
+
+      return entry;
+    } catch (error) {
+      // File doesn't exist anymore
+      this.cache.delete(filePath);
+      return null;
+    }
+  }
+
+  /**
+   * Stores a cache entry
+   * @param entry - Cache entry to store
+   */
+  public async set(entry: ICacheEntry): Promise<void> {
+    if (!this.config.enabled) {
+      return;
+    }
+
+    this.cache.set(entry.path, entry);
+
+    // Check cache size and evict old entries if needed
+    await this.enforceMaxSize();
+
+    // Persist to disk (async, don't await)
+    this.persist().catch((error) => {
+      console.warn('Failed to persist cache:', error.message);
+    });
+  }
+
+  /**
+   * Stores multiple cache entries
+   * @param entries - Array of cache entries
+   */
+  public async setMany(entries: ICacheEntry[]): Promise<void> {
+    if (!this.config.enabled) {
+      return;
+    }
+
+    for (const entry of entries) {
+      this.cache.set(entry.path, entry);
+    }
+
+    await this.enforceMaxSize();
+    await this.persist();
+  }
+
+  /**
+   * Checks if a file is cached and valid
+   * @param filePath - Absolute path to the file
+   * @returns True if cached and valid
+   */
+  public async has(filePath: string): Promise<boolean> {
+    const entry = await this.get(filePath);
+    return entry !== null;
+  }
+
+  /**
+   * Gets cache statistics
+   */
+  public getStats(): {
+    entries: number;
+    totalSize: number;
+    oldestEntry: number | null;
+    newestEntry: number | null;
+  } {
+    let totalSize = 0;
+    let oldestEntry: number | null = null;
+    let newestEntry: number | null = null;
+
+    for (const entry of this.cache.values()) {
+      totalSize += entry.contents.length;
+
+      if (oldestEntry === null || entry.cachedAt < oldestEntry) {
+        oldestEntry = entry.cachedAt;
+      }
+
+      if (newestEntry === null || entry.cachedAt > newestEntry) {
+        newestEntry = entry.cachedAt;
+      }
+    }
+
+    return {
+      entries: this.cache.size,
+      totalSize,
+      oldestEntry,
+      newestEntry,
+    };
+  }
+
+  /**
+   * Clears all cache entries
+   */
+  public async clear(): Promise<void> {
+    this.cache.clear();
+    await this.persist();
+  }
+
+  /**
+   * Clears specific cache entries
+   * @param filePaths - Array of file paths to clear
+   */
+  public async clearPaths(filePaths: string[]): Promise<void> {
+    for (const path of filePaths) {
+      this.cache.delete(path);
+    }
+    await this.persist();
+  }
+
+  /**
+   * Cleans up expired and invalid cache entries
+   */
+  private async cleanup(): Promise<void> {
+    const now = Date.now();
+    const toDelete: string[] = [];
+
+    for (const [path, entry] of this.cache.entries()) {
+      // Check expiration
+      if (now - entry.cachedAt > this.config.ttl * 1000) {
+        toDelete.push(path);
+        continue;
+      }
+
+      // Check if file still exists and hasn't changed
+      try {
+        const stats = await fs.promises.stat(path);
+        const currentMtime = Math.floor(stats.mtimeMs);
+
+        if (currentMtime !== entry.mtime) {
+          toDelete.push(path);
+        }
+      } catch (error) {
+        // File doesn't exist
+        toDelete.push(path);
+      }
+    }
+
+    for (const path of toDelete) {
+      this.cache.delete(path);
+    }
+
+    if (toDelete.length > 0) {
+      await this.persist();
+    }
+  }
+
+  /**
+   * Enforces maximum cache size by evicting oldest entries
+   */
+  private async enforceMaxSize(): Promise<void> {
+    const stats = this.getStats();
+    const maxSizeBytes = this.config.maxSize * 1024 * 1024; // Convert MB to bytes
+
+    if (stats.totalSize <= maxSizeBytes) {
+      return;
+    }
+
+    // Sort entries by age (oldest first)
+    const entries = Array.from(this.cache.entries()).sort(
+      (a, b) => a[1].cachedAt - b[1].cachedAt
+    );
+
+    // Remove oldest entries until we're under the limit
+    let currentSize = stats.totalSize;
+    for (const [path, entry] of entries) {
+      if (currentSize <= maxSizeBytes) {
+        break;
+      }
+
+      currentSize -= entry.contents.length;
+      this.cache.delete(path);
+    }
+  }
+
+  /**
+   * Persists cache index to disk
+   */
+  private async persist(): Promise<void> {
+    if (!this.config.enabled) {
+      return;
+    }
+
+    try {
+      const entries = Array.from(this.cache.values());
+      const content = JSON.stringify(entries, null, 2);
+      await plugins.smartfile.memory.toFs(content, this.cacheIndexPath);
+    } catch (error) {
+      console.warn('Failed to persist cache index:', error.message);
+    }
+  }
+}

package/ts/context/context-trimmer.ts

@@ -243,4 +243,68 @@ export class ContextTrimmer {
       ...config
     };
   }
+
+  /**
+   * Trim a file based on its importance tier
+   * @param filePath The path to the file
+   * @param content The file's contents
+   * @param level The trimming level to apply ('none', 'light', 'aggressive')
+   * @returns The trimmed file contents
+   */
+  public trimFileWithLevel(
+    filePath: string,
+    content: string,
+    level: 'none' | 'light' | 'aggressive'
+  ): string {
+    // No trimming for essential files
+    if (level === 'none') {
+      return content;
+    }
+
+    // Create a temporary config based on level
+    const originalConfig = { ...this.config };
+
+    try {
+      if (level === 'light') {
+        // Light trimming: preserve signatures, remove only complex implementations
+        this.config = {
+          ...this.config,
+          removeImplementations: false,
+          preserveInterfaces: true,
+          preserveTypeDefs: true,
+          preserveJSDoc: true,
+          maxFunctionLines: 10,
+          removeComments: false,
+          removeBlankLines: true
+        };
+      } else if (level === 'aggressive') {
+        // Aggressive trimming: remove all implementations, keep only signatures
+        this.config = {
+          ...this.config,
+          removeImplementations: true,
+          preserveInterfaces: true,
+          preserveTypeDefs: true,
+          preserveJSDoc: true,
+          maxFunctionLines: 3,
+          removeComments: true,
+          removeBlankLines: true
+        };
+      }
+
+      // Process based on file type
+      let result = content;
+      if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) {
+        result = this.trimTypeScriptFile(content);
+      } else if (filePath.endsWith('.md')) {
+        result = this.trimMarkdownFile(content);
+      } else if (filePath.endsWith('.json')) {
+        result = this.trimJsonFile(content);
+      }
+
+      return result;
+    } finally {
+      // Restore original config
+      this.config = originalConfig;
+    }
+  }
 }

package/ts/context/enhanced-context.ts

@@ -1,7 +1,10 @@
 import * as plugins from '../plugins.js';
-import type { ContextMode, IContextResult, IFileInfo, TaskType } from './types.js';
+import type { ContextMode, IContextResult, IFileInfo, TaskType, IFileMetadata } from './types.js';
 import { ContextTrimmer } from './context-trimmer.js';
 import { ConfigManager } from './config-manager.js';
+import { LazyFileLoader } from './lazy-file-loader.js';
+import { ContextCache } from './context-cache.js';
+import { ContextAnalyzer } from './context-analyzer.js';
 
 /**
  * Enhanced ProjectContext that supports context optimization strategies
@@ -10,6 +13,9 @@ export class EnhancedContext {
   private projectDir: string;
   private trimmer: ContextTrimmer;
   private configManager: ConfigManager;
+  private lazyLoader: LazyFileLoader;
+  private cache: ContextCache;
+  private analyzer: ContextAnalyzer;
   private contextMode: ContextMode = 'trimmed';
   private tokenBudget: number = 190000; // Default for o4-mini
   private contextResult: IContextResult = {
@@ -29,6 +35,13 @@ export class EnhancedContext {
     this.projectDir = projectDirArg;
     this.configManager = ConfigManager.getInstance();
     this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig());
+    this.lazyLoader = new LazyFileLoader(projectDirArg);
+    this.cache = new ContextCache(projectDirArg, this.configManager.getCacheConfig());
+    this.analyzer = new ContextAnalyzer(
+      projectDirArg,
+      this.configManager.getPrioritizationWeights(),
+      this.configManager.getTierConfig()
+    );
   }
   
   /**
@@ -38,6 +51,7 @@ export class EnhancedContext {
     await this.configManager.initialize(this.projectDir);
     this.tokenBudget = this.configManager.getMaxTokens();
     this.trimmer.updateConfig(this.configManager.getTrimConfig());
+    await this.cache.init();
   }
   
   /**
@@ -138,13 +152,28 @@ export class EnhancedContext {
     
     let totalTokenCount = 0;
     let totalOriginalTokens = 0;
-    
-    // Sort files by importance (for now just a simple alphabetical sort)
-    // Later this could be enhanced with more sophisticated prioritization
-    const sortedFiles = [...files].sort((a, b) => a.relative.localeCompare(b.relative));
-    
+
+    // Convert SmartFile objects to IFileMetadata for analysis
+    const metadata: IFileMetadata[] = files.map(sf => ({
+      path: sf.path,
+      relativePath: sf.relative,
+      size: sf.contents.toString().length,
+      mtime: Date.now(), // SmartFile doesn't expose mtime, use current time
+      estimatedTokens: this.countTokens(sf.contents.toString()),
+      importanceScore: 0
+    }));
+
+    // Analyze files using ContextAnalyzer to get smart prioritization
+    // (Note: This requires task type which we'll pass from buildContext)
+    // For now, sort files by estimated tokens (smaller files first for better efficiency)
+    const sortedFiles = [...files].sort((a, b) => {
+      const aTokens = this.countTokens(a.contents.toString());
+      const bTokens = this.countTokens(b.contents.toString());
+      return aTokens - bTokens;
+    });
+
     const processedFiles: string[] = [];
-    
+
     for (const smartfile of sortedFiles) {
       // Calculate original token count
       const originalContent = smartfile.contents.toString();
@@ -215,6 +244,154 @@ ${processedContent}
     return context;
   }
   
+  /**
+   * Convert files to context with smart analysis and prioritization
+   * @param metadata - File metadata to analyze
+   * @param taskType - Task type for context-aware prioritization
+   * @param mode - Context mode to use
+   * @returns Context string
+   */
+  public async convertFilesToContextWithAnalysis(
+    metadata: IFileMetadata[],
+    taskType: TaskType,
+    mode: ContextMode = this.contextMode
+  ): Promise<string> {
+    // Reset context result
+    this.contextResult = {
+      context: '',
+      tokenCount: 0,
+      includedFiles: [],
+      trimmedFiles: [],
+      excludedFiles: [],
+      tokenSavings: 0
+    };
+
+    // Analyze files for smart prioritization
+    const analysis = await this.analyzer.analyze(metadata, taskType, []);
+
+    // Sort files by importance score (highest first)
+    const sortedAnalysis = [...analysis.files].sort(
+      (a, b) => b.importanceScore - a.importanceScore
+    );
+
+    // Filter out excluded tier
+    const relevantFiles = sortedAnalysis.filter(f => f.tier !== 'excluded');
+
+    let totalTokenCount = 0;
+    let totalOriginalTokens = 0;
+    const processedFiles: string[] = [];
+
+    // Load files with cache support
+    for (const fileAnalysis of relevantFiles) {
+      try {
+        // Check cache first
+        let contents: string;
+        let originalTokenCount: number;
+
+        const cached = await this.cache.get(fileAnalysis.path);
+        if (cached) {
+          contents = cached.contents;
+          originalTokenCount = cached.tokenCount;
+        } else {
+          // Load file
+          const fileData = await plugins.smartfile.fs.toStringSync(fileAnalysis.path);
+          contents = fileData;
+          originalTokenCount = this.countTokens(contents);
+
+          // Cache it
+          await this.cache.set({
+            path: fileAnalysis.path,
+            contents,
+            tokenCount: originalTokenCount,
+            mtime: Date.now(),
+            cachedAt: Date.now()
+          });
+        }
+
+        totalOriginalTokens += originalTokenCount;
+
+        // Apply tier-based trimming
+        let processedContent = contents;
+        let trimLevel: 'none' | 'light' | 'aggressive' = 'light';
+
+        if (fileAnalysis.tier === 'essential') {
+          trimLevel = 'none';
+        } else if (fileAnalysis.tier === 'important') {
+          trimLevel = 'light';
+        } else if (fileAnalysis.tier === 'optional') {
+          trimLevel = 'aggressive';
+        }
+
+        // Apply trimming based on mode and tier
+        if (mode !== 'full' && trimLevel !== 'none') {
+          const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
+          processedContent = this.trimmer.trimFileWithLevel(
+            relativePath,
+            contents,
+            trimLevel
+          );
+        }
+
+        // Calculate token count
+        const processedTokenCount = this.countTokens(processedContent);
+
+        // Check token budget
+        if (totalTokenCount + processedTokenCount > this.tokenBudget) {
+          // We don't have budget for this file
+          const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
+          this.contextResult.excludedFiles.push({
+            path: fileAnalysis.path,
+            contents,
+            relativePath,
+            tokenCount: originalTokenCount,
+            importanceScore: fileAnalysis.importanceScore
+          });
+          continue;
+        }
+
+        // Format the file for context
+        const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
+        const formattedContent = `
+====== START OF FILE ${relativePath} ======
+
+${processedContent}
+
+====== END OF FILE ${relativePath} ======
+        `;
+
+        processedFiles.push(formattedContent);
+        totalTokenCount += processedTokenCount;
+
+        // Track file in appropriate list
+        const fileInfo: IFileInfo = {
+          path: fileAnalysis.path,
+          contents: processedContent,
+          relativePath,
+          tokenCount: processedTokenCount,
+          importanceScore: fileAnalysis.importanceScore
+        };
+
+        if (trimLevel === 'none' || processedContent === contents) {
+          this.contextResult.includedFiles.push(fileInfo);
+        } else {
+          this.contextResult.trimmedFiles.push(fileInfo);
+          this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
+        }
+      } catch (error) {
+        console.warn(`Failed to process file ${fileAnalysis.path}:`, error.message);
+      }
+    }
+
+    // Join all processed files
+    const context = processedFiles.join('\n');
+
+    // Update context result
+    this.contextResult.context = context;
+    this.contextResult.tokenCount = totalTokenCount;
+
+    return context;
+  }
+
   /**
    * Build context for the project
    * @param taskType Optional task type for task-specific context
@@ -233,42 +410,71 @@ ${processedContent}
       }
     }
     
-    // Gather files
-    const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
-    const files = await this.gatherFiles(
-      taskConfig?.includePaths,
-      taskConfig?.excludePaths
-    );
-    
-    // Convert files to context
-    // Create an array of all files to process
-    const allFiles: plugins.smartfile.SmartFile[] = [];
-    
-    // Add individual files
-    if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
-    if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
-    if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
-    if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
-    
-    // Add arrays of files
-    if (files.smartfilesMod) {
-      if (Array.isArray(files.smartfilesMod)) {
-        allFiles.push(...files.smartfilesMod);
-      } else {
-        allFiles.push(files.smartfilesMod);
+    // Check if analyzer is enabled in config
+    const analyzerConfig = this.configManager.getAnalyzerConfig();
+    const useAnalyzer = analyzerConfig.enabled && taskType;
+
+    if (useAnalyzer) {
+      // Use new smart context building with lazy loading and analysis
+      const taskConfig = this.configManager.getTaskConfig(taskType!);
+
+      // Build globs for scanning
+      const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
+        'ts/**/*.ts',
+        'ts*/**/*.ts'
+      ];
+
+      // Add config files
+      const configGlobs = [
+        'package.json',
+        'readme.md',
+        'readme.hints.md',
+        'npmextra.json'
+      ];
+
+      // Scan files for metadata (fast, doesn't load contents)
+      const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
+
+      // Use analyzer to build context with smart prioritization
+      await this.convertFilesToContextWithAnalysis(metadata, taskType!, this.contextMode);
+    } else {
+      // Fall back to old method for backward compatibility
+      const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
+      const files = await this.gatherFiles(
+        taskConfig?.includePaths,
+        taskConfig?.excludePaths
+      );
+
+      // Convert files to context
+      // Create an array of all files to process
+      const allFiles: plugins.smartfile.SmartFile[] = [];
+
+      // Add individual files
+      if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
+      if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
+      if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
+      if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
+
+      // Add arrays of files
+      if (files.smartfilesMod) {
+        if (Array.isArray(files.smartfilesMod)) {
+          allFiles.push(...files.smartfilesMod);
+        } else {
+          allFiles.push(files.smartfilesMod);
+        }
       }
-    }
-    
-    if (files.smartfilesTest) {
-      if (Array.isArray(files.smartfilesTest)) {
-        allFiles.push(...files.smartfilesTest);
-      } else {
-        allFiles.push(files.smartfilesTest);
+
+      if (files.smartfilesTest) {
+        if (Array.isArray(files.smartfilesTest)) {
+          allFiles.push(...files.smartfilesTest);
+        } else {
+          allFiles.push(files.smartfilesTest);
+        }
       }
+
+      await this.convertFilesToContext(allFiles);
     }
-    
-    const context = await this.convertFilesToContext(allFiles);
-    
+
     return this.contextResult;
   }