@git.zone/tsdoc 1.5.1 → 1.6.0

package/ts/context/index.ts

@@ -2,14 +2,27 @@ import { EnhancedContext } from './enhanced-context.js';
 import { TaskContextFactory } from './task-context-factory.js';
 import { ConfigManager } from './config-manager.js';
 import { ContextTrimmer } from './context-trimmer.js';
-import type { 
-  ContextMode, 
-  IContextConfig, 
+import { LazyFileLoader } from './lazy-file-loader.js';
+import { ContextCache } from './context-cache.js';
+import { ContextAnalyzer } from './context-analyzer.js';
+import type {
+  ContextMode,
+  IContextConfig,
   IContextResult,
   IFileInfo,
   ITrimConfig,
   ITaskConfig,
-  TaskType
+  TaskType,
+  ICacheConfig,
+  IAnalyzerConfig,
+  IPrioritizationWeights,
+  ITierConfig,
+  ITierSettings,
+  IFileMetadata,
+  ICacheEntry,
+  IFileDependencies,
+  IFileAnalysis,
+  IAnalysisResult
 } from './types.js';
 
 export {
@@ -18,6 +31,9 @@ export {
   TaskContextFactory,
   ConfigManager,
   ContextTrimmer,
+  LazyFileLoader,
+  ContextCache,
+  ContextAnalyzer,
 };
 
 // Types
@@ -28,5 +44,15 @@ export type {
   IFileInfo,
   ITrimConfig,
   ITaskConfig,
-  TaskType
+  TaskType,
+  ICacheConfig,
+  IAnalyzerConfig,
+  IPrioritizationWeights,
+  ITierConfig,
+  ITierSettings,
+  IFileMetadata,
+  ICacheEntry,
+  IFileDependencies,
+  IFileAnalysis,
+  IAnalysisResult
 };

package/ts/context/lazy-file-loader.ts

@@ -0,0 +1,191 @@
+import * as plugins from '../plugins.js';
+import * as fs from 'fs';
+import type { IFileMetadata, IFileInfo } from './types.js';
+
+/**
+ * LazyFileLoader handles efficient file loading by:
+ * - Scanning files for metadata without loading contents
+ * - Providing fast file size and token estimates
+ * - Loading contents only when requested
+ * - Parallel loading of selected files
+ */
+export class LazyFileLoader {
+  private projectRoot: string;
+  private metadataCache: Map<string, IFileMetadata> = new Map();
+
+  /**
+   * Creates a new LazyFileLoader
+   * @param projectRoot - Root directory of the project
+   */
+  constructor(projectRoot: string) {
+    this.projectRoot = projectRoot;
+  }
+
+  /**
+   * Scans files in given globs and creates metadata without loading contents
+   * @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts'])
+   * @returns Array of file metadata
+   */
+  public async scanFiles(globs: string[]): Promise<IFileMetadata[]> {
+    const metadata: IFileMetadata[] = [];
+
+    for (const globPattern of globs) {
+      try {
+        const smartFiles = await plugins.smartfile.fs.fileTreeToObject(this.projectRoot, globPattern);
+        const fileArray = Array.isArray(smartFiles) ? smartFiles : [smartFiles];
+
+        for (const smartFile of fileArray) {
+          try {
+            const meta = await this.getMetadata(smartFile.path);
+            metadata.push(meta);
+          } catch (error) {
+            // Skip files that can't be read
+            console.warn(`Failed to get metadata for ${smartFile.path}:`, error.message);
+          }
+        }
+      } catch (error) {
+        // Skip patterns that don't match any files
+        console.warn(`No files found for pattern ${globPattern}`);
+      }
+    }
+
+    return metadata;
+  }
+
+  /**
+   * Gets metadata for a single file without loading contents
+   * @param filePath - Absolute path to the file
+   * @returns File metadata
+   */
+  public async getMetadata(filePath: string): Promise<IFileMetadata> {
+    // Check cache first
+    if (this.metadataCache.has(filePath)) {
+      const cached = this.metadataCache.get(filePath)!;
+      const currentStats = await fs.promises.stat(filePath);
+
+      // Return cached if file hasn't changed
+      if (cached.mtime === Math.floor(currentStats.mtimeMs)) {
+        return cached;
+      }
+    }
+
+    // Get file stats
+    const stats = await fs.promises.stat(filePath);
+    const relativePath = plugins.path.relative(this.projectRoot, filePath);
+
+    // Estimate tokens: rough estimate of ~4 characters per token
+    // This is faster than reading and tokenizing the entire file
+    const estimatedTokens = Math.ceil(stats.size / 4);
+
+    const metadata: IFileMetadata = {
+      path: filePath,
+      relativePath,
+      size: stats.size,
+      mtime: Math.floor(stats.mtimeMs),
+      estimatedTokens,
+    };
+
+    // Cache the metadata
+    this.metadataCache.set(filePath, metadata);
+
+    return metadata;
+  }
+
+  /**
+   * Loads file contents for selected files in parallel
+   * @param metadata - Array of file metadata to load
+   * @param tokenizer - Function to calculate accurate token count
+   * @returns Array of complete file info with contents
+   */
+  public async loadFiles(
+    metadata: IFileMetadata[],
+    tokenizer: (content: string) => number
+  ): Promise<IFileInfo[]> {
+    // Load files in parallel
+    const loadPromises = metadata.map(async (meta) => {
+      try {
+        const contents = await plugins.smartfile.fs.toStringSync(meta.path);
+        const tokenCount = tokenizer(contents);
+
+        const fileInfo: IFileInfo = {
+          path: meta.path,
+          relativePath: meta.relativePath,
+          contents,
+          tokenCount,
+          importanceScore: meta.importanceScore,
+        };
+
+        return fileInfo;
+      } catch (error) {
+        console.warn(`Failed to load file ${meta.path}:`, error.message);
+        return null;
+      }
+    });
+
+    // Wait for all loads to complete and filter out failures
+    const results = await Promise.all(loadPromises);
+    return results.filter((r): r is IFileInfo => r !== null);
+  }
+
+  /**
+   * Loads a single file with contents
+   * @param filePath - Absolute path to the file
+   * @param tokenizer - Function to calculate accurate token count
+   * @returns Complete file info with contents
+   */
+  public async loadFile(
+    filePath: string,
+    tokenizer: (content: string) => number
+  ): Promise<IFileInfo> {
+    const meta = await this.getMetadata(filePath);
+    const contents = await plugins.smartfile.fs.toStringSync(filePath);
+    const tokenCount = tokenizer(contents);
+    const relativePath = plugins.path.relative(this.projectRoot, filePath);
+
+    return {
+      path: filePath,
+      relativePath,
+      contents,
+      tokenCount,
+      importanceScore: meta.importanceScore,
+    };
+  }
+
+  /**
+   * Updates importance scores for metadata entries
+   * @param scores - Map of file paths to importance scores
+   */
+  public updateImportanceScores(scores: Map<string, number>): void {
+    for (const [path, score] of scores) {
+      const meta = this.metadataCache.get(path);
+      if (meta) {
+        meta.importanceScore = score;
+      }
+    }
+  }
+
+  /**
+   * Clears the metadata cache
+   */
+  public clearCache(): void {
+    this.metadataCache.clear();
+  }
+
+  /**
+   * Gets total estimated tokens for all cached metadata
+   */
+  public getTotalEstimatedTokens(): number {
+    let total = 0;
+    for (const meta of this.metadataCache.values()) {
+      total += meta.estimatedTokens;
+    }
+    return total;
+  }
+
+  /**
+   * Gets cached metadata entries
+   */
+  public getCachedMetadata(): IFileMetadata[] {
+    return Array.from(this.metadataCache.values());
+  }
+}

package/ts/context/types.ts

@@ -58,6 +58,73 @@ export interface IContextConfig {
   };
   /** Trimming configuration */
   trimming?: ITrimConfig;
+  /** Cache configuration */
+  cache?: ICacheConfig;
+  /** Analyzer configuration */
+  analyzer?: IAnalyzerConfig;
+  /** Prioritization weights */
+  prioritization?: IPrioritizationWeights;
+  /** Tier configuration for adaptive trimming */
+  tiers?: ITierConfig;
+}
+
+/**
+ * Cache configuration
+ */
+export interface ICacheConfig {
+  /** Whether caching is enabled */
+  enabled?: boolean;
+  /** Time-to-live in seconds */
+  ttl?: number;
+  /** Maximum cache size in MB */
+  maxSize?: number;
+  /** Cache directory path */
+  directory?: string;
+}
+
+/**
+ * Analyzer configuration
+ */
+export interface IAnalyzerConfig {
+  /** Whether analyzer is enabled */
+  enabled?: boolean;
+  /** Whether to use AI refinement for selection */
+  useAIRefinement?: boolean;
+  /** AI model to use for refinement */
+  aiModel?: string;
+}
+
+/**
+ * Weights for file prioritization
+ */
+export interface IPrioritizationWeights {
+  /** Weight for dependency centrality */
+  dependencyWeight?: number;
+  /** Weight for task relevance */
+  relevanceWeight?: number;
+  /** Weight for token efficiency */
+  efficiencyWeight?: number;
+  /** Weight for file recency */
+  recencyWeight?: number;
+}
+
+/**
+ * Tier configuration for adaptive trimming
+ */
+export interface ITierConfig {
+  essential?: ITierSettings;
+  important?: ITierSettings;
+  optional?: ITierSettings;
+}
+
+/**
+ * Settings for a single tier
+ */
+export interface ITierSettings {
+  /** Minimum score to qualify for this tier */
+  minScore: number;
+  /** Trimming level to apply */
+  trimLevel: 'none' | 'light' | 'aggressive';
 }
 
 /**
@@ -92,4 +159,90 @@ export interface IContextResult {
   excludedFiles: IFileInfo[];
   /** Token savings from trimming */
   tokenSavings: number;
+}
+
+/**
+ * File metadata without contents (for lazy loading)
+ */
+export interface IFileMetadata {
+  /** The file path */
+  path: string;
+  /** The file's relative path from the project root */
+  relativePath: string;
+  /** File size in bytes */
+  size: number;
+  /** Last modified time (Unix timestamp) */
+  mtime: number;
+  /** Estimated token count (without loading full contents) */
+  estimatedTokens: number;
+  /** The file's importance score */
+  importanceScore?: number;
+}
+
+/**
+ * Cache entry for a file
+ */
+export interface ICacheEntry {
+  /** File path */
+  path: string;
+  /** File contents */
+  contents: string;
+  /** Token count */
+  tokenCount: number;
+  /** Last modified time when cached */
+  mtime: number;
+  /** When this cache entry was created */
+  cachedAt: number;
+}
+
+/**
+ * Dependency information for a file
+ */
+export interface IFileDependencies {
+  /** File path */
+  path: string;
+  /** Files this file imports */
+  imports: string[];
+  /** Files that import this file */
+  importedBy: string[];
+  /** Centrality score (0-1) - how central this file is in the dependency graph */
+  centrality: number;
+}
+
+/**
+ * Analysis result for a file
+ */
+export interface IFileAnalysis {
+  /** File path */
+  path: string;
+  /** Task relevance score (0-1) */
+  relevanceScore: number;
+  /** Dependency centrality score (0-1) */
+  centralityScore: number;
+  /** Token efficiency score (0-1) */
+  efficiencyScore: number;
+  /** Recency score (0-1) */
+  recencyScore: number;
+  /** Combined importance score (0-1) */
+  importanceScore: number;
+  /** Assigned tier */
+  tier: 'essential' | 'important' | 'optional' | 'excluded';
+  /** Reason for the score */
+  reason?: string;
+}
+
+/**
+ * Result of context analysis
+ */
+export interface IAnalysisResult {
+  /** Task type being analyzed */
+  taskType: TaskType;
+  /** Analyzed files with scores */
+  files: IFileAnalysis[];
+  /** Dependency graph */
+  dependencyGraph: Map<string, IFileDependencies>;
+  /** Total files analyzed */
+  totalFiles: number;
+  /** Analysis duration in ms */
+  analysisDuration: number;
 }