@git.zone/tsdoc 1.6.0 → 1.7.0

package/dist_ts/context/types.d.ts

@@ -62,6 +62,8 @@ export interface IContextConfig {
     prioritization?: IPrioritizationWeights;
     /** Tier configuration for adaptive trimming */
     tiers?: ITierConfig;
+    /** Iterative context building configuration */
+    iterative?: IIterativeConfig;
 }
 /**
  * Cache configuration
@@ -78,11 +80,10 @@ export interface ICacheConfig {
 }
 /**
  * Analyzer configuration
+ * Note: Smart analysis is always enabled; this config only controls advanced options
  */
 export interface IAnalyzerConfig {
-    /** Whether analyzer is enabled */
-    enabled?: boolean;
-    /** Whether to use AI refinement for selection */
+    /** Whether to use AI refinement for selection (advanced, disabled by default) */
     useAIRefinement?: boolean;
     /** AI model to use for refinement */
     aiModel?: string;
@@ -230,3 +231,70 @@ export interface IAnalysisResult {
     /** Analysis duration in ms */
     analysisDuration: number;
 }
+/**
+ * Configuration for iterative context building
+ */
+export interface IIterativeConfig {
+    /** Maximum number of iterations allowed */
+    maxIterations?: number;
+    /** Maximum files to request in first iteration */
+    firstPassFileLimit?: number;
+    /** Maximum files to request in subsequent iterations */
+    subsequentPassFileLimit?: number;
+    /** Temperature for AI decision making (0-1) */
+    temperature?: number;
+    /** Model to use for iterative decisions */
+    model?: string;
+}
+/**
+ * AI decision for file selection
+ */
+export interface IFileSelectionDecision {
+    /** AI's reasoning for file selection */
+    reasoning: string;
+    /** File paths to load */
+    filesToLoad: string[];
+    /** Estimated tokens needed */
+    estimatedTokensNeeded?: number;
+}
+/**
+ * AI decision for context sufficiency
+ */
+export interface IContextSufficiencyDecision {
+    /** Whether context is sufficient */
+    sufficient: boolean;
+    /** AI's reasoning */
+    reasoning: string;
+    /** Additional files needed (if not sufficient) */
+    additionalFilesNeeded?: string[];
+}
+/**
+ * State for a single iteration
+ */
+export interface IIterationState {
+    /** Iteration number (1-based) */
+    iteration: number;
+    /** Files loaded in this iteration */
+    filesLoaded: IFileInfo[];
+    /** Tokens used in this iteration */
+    tokensUsed: number;
+    /** Total tokens used so far */
+    totalTokensUsed: number;
+    /** AI decision made in this iteration */
+    decision: IFileSelectionDecision | IContextSufficiencyDecision;
+    /** Duration of this iteration in ms */
+    duration: number;
+}
+/**
+ * Result of iterative context building
+ */
+export interface IIterativeContextResult extends IContextResult {
+    /** Number of iterations performed */
+    iterationCount: number;
+    /** Details of each iteration */
+    iterations: IIterationState[];
+    /** Total API calls made */
+    apiCallCount: number;
+    /** Total duration in ms */
+    totalDuration: number;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@git.zone/tsdoc",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "private": false,
   "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
   "type": "module",

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@git.zone/tsdoc',
-  version: '1.6.0',
+  version: '1.7.0',
   description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
 }

package/ts/cli.ts

@@ -31,18 +31,18 @@ export const run = async () => {
   tsdocCli.addCommand('aidoc').subscribe(async (argvArg) => {
     const aidocInstance = new AiDoc();
     await aidocInstance.start();
-    
+
     // Get context token count if requested
     if (argvArg.tokens || argvArg.showTokens) {
       logger.log('info', `Calculating context token count...`);
       const tokenCount = await aidocInstance.getProjectContextTokenCount(paths.cwd);
       logger.log('ok', `Total context token count: ${tokenCount}`);
-      
+
       if (argvArg.tokensOnly) {
         return; // Exit early if we only want token count
       }
     }
-    
+
     logger.log('info', `Generating new readme...`);
     logger.log('info', `This may take some time...`);
     await aidocInstance.buildReadme(paths.cwd);
@@ -54,67 +54,50 @@ export const run = async () => {
   tsdocCli.addCommand('tokens').subscribe(async (argvArg) => {
     const aidocInstance = new AiDoc();
     await aidocInstance.start();
-    
+
     logger.log('info', `Calculating context token count...`);
-    
-    // Determine context mode based on args
-    let contextMode: context.ContextMode = 'full';
-    if (argvArg.trim || argvArg.trimmed) {
-      contextMode = 'trimmed';
-    } else if (argvArg.summarize || argvArg.summarized) {
-      contextMode = 'summarized';
-    }
-    
+
     // Get task type if specified
     let taskType: context.TaskType | undefined = undefined;
     if (argvArg.task) {
       if (['readme', 'commit', 'description'].includes(argvArg.task)) {
         taskType = argvArg.task as context.TaskType;
       } else {
-        logger.log('warn', `Unknown task type: ${argvArg.task}. Using default context.`);
+        logger.log('warn', `Unknown task type: ${argvArg.task}. Using default (readme).`);
+        taskType = 'readme';
       }
+    } else {
+      // Default to readme if no task specified
+      taskType = 'readme';
     }
-    
-    // Use enhanced context
+
+    // Use iterative context building
     const taskFactory = new context.TaskContextFactory(paths.cwd);
     await taskFactory.initialize();
-    
-    let contextResult: context.IContextResult;
-    
+
+    let contextResult: context.IIterativeContextResult;
+
     if (argvArg.all) {
       // Show stats for all task types
       const stats = await taskFactory.getTokenStats();
-      
+
       logger.log('ok', 'Token statistics by task:');
       for (const [task, data] of Object.entries(stats)) {
         logger.log('info', `\n${task.toUpperCase()}:`);
         logger.log('info', `  Tokens: ${data.tokenCount}`);
         logger.log('info', `  Token savings: ${data.savings}`);
         logger.log('info', `  Files: ${data.includedFiles} included, ${data.trimmedFiles} trimmed, ${data.excludedFiles} excluded`);
-        
+
         // Calculate percentage of model context
         const o4MiniPercentage = (data.tokenCount / 200000 * 100).toFixed(2);
         logger.log('info', `  Context usage: ${o4MiniPercentage}% of o4-mini (200K tokens)`);
       }
-      
+
       return;
     }
-    
-    if (taskType) {
-      // Get context for specific task
-      contextResult = await taskFactory.createContextForTask(taskType);
-    } else {
-      // Get generic context with specified mode
-      const enhancedContext = new context.EnhancedContext(paths.cwd);
-      await enhancedContext.initialize();
-      enhancedContext.setContextMode(contextMode);
-      
-      if (argvArg.maxTokens) {
-        enhancedContext.setTokenBudget(parseInt(argvArg.maxTokens, 10));
-      }
-      
-      contextResult = await enhancedContext.buildContext();
-    }
+
+    // Get context for specific task
+    contextResult = await taskFactory.createContextForTask(taskType);
     
     // Display results
     logger.log('ok', `Total context token count: ${contextResult.tokenCount}`);

package/ts/context/config-manager.ts

@@ -9,7 +9,8 @@ import type {
   ICacheConfig,
   IAnalyzerConfig,
   IPrioritizationWeights,
-  ITierConfig
+  ITierConfig,
+  IIterativeConfig
 } from './types.js';
 
 /**
@@ -85,7 +86,6 @@ export class ConfigManager {
         directory: undefined // Will be set to .nogit/context-cache by ContextCache
       },
       analyzer: {
-        enabled: true,
         useAIRefinement: false, // Disabled by default for now
         aiModel: 'haiku'
       },
@@ -99,6 +99,13 @@ export class ConfigManager {
         essential: { minScore: 0.8, trimLevel: 'none' },
         important: { minScore: 0.5, trimLevel: 'light' },
         optional: { minScore: 0.2, trimLevel: 'aggressive' }
+      },
+      iterative: {
+        maxIterations: 5,
+        firstPassFileLimit: 10,
+        subsequentPassFileLimit: 5,
+        temperature: 0.3,
+        model: 'gpt-4-turbo-preview'
       }
     };
   }
@@ -157,15 +164,15 @@ export class ConfigManager {
    */
   private mergeConfigs(defaultConfig: IContextConfig, userConfig: Partial<IContextConfig>): IContextConfig {
     const result: IContextConfig = { ...defaultConfig };
-    
+
     // Merge top-level properties
     if (userConfig.maxTokens !== undefined) result.maxTokens = userConfig.maxTokens;
     if (userConfig.defaultMode !== undefined) result.defaultMode = userConfig.defaultMode;
-    
+
     // Merge task-specific settings
     if (userConfig.taskSpecificSettings) {
       result.taskSpecificSettings = result.taskSpecificSettings || {};
-      
+
       // For each task type, merge settings
       (['readme', 'commit', 'description'] as TaskType[]).forEach(taskType => {
         if (userConfig.taskSpecificSettings?.[taskType]) {
@@ -176,7 +183,7 @@ export class ConfigManager {
         }
       });
     }
-    
+
     // Merge trimming configuration
     if (userConfig.trimming) {
       result.trimming = {
@@ -217,6 +224,14 @@ export class ConfigManager {
       };
     }
 
+    // Merge iterative configuration
+    if (userConfig.iterative) {
+      result.iterative = {
+        ...result.iterative,
+        ...userConfig.iterative
+      };
+    }
+
     return result;
   }
   
@@ -306,7 +321,7 @@ export class ConfigManager {
    * Get analyzer configuration
    */
   public getAnalyzerConfig(): IAnalyzerConfig {
-    return this.config.analyzer || { enabled: true, useAIRefinement: false, aiModel: 'haiku' };
+    return this.config.analyzer || { useAIRefinement: false, aiModel: 'haiku' };
   }
 
   /**
@@ -332,6 +347,19 @@ export class ConfigManager {
     };
   }
 
+  /**
+   * Get iterative configuration
+   */
+  public getIterativeConfig(): IIterativeConfig {
+    return this.config.iterative || {
+      maxIterations: 5,
+      firstPassFileLimit: 10,
+      subsequentPassFileLimit: 5,
+      temperature: 0.3,
+      model: 'gpt-4-turbo-preview'
+    };
+  }
+
   /**
    * Clear the config cache (force reload on next access)
    */

package/ts/context/context-cache.ts

@@ -1,6 +1,7 @@
 import * as plugins from '../plugins.js';
 import * as fs from 'fs';
 import type { ICacheEntry, ICacheConfig } from './types.js';
+import { logger } from '../logging.js';
 
 /**
  * ContextCache provides persistent caching of file contents and token counts

package/ts/context/enhanced-context.ts

@@ -69,181 +69,7 @@ export class EnhancedContext {
   public setTokenBudget(maxTokens: number): void {
     this.tokenBudget = maxTokens;
   }
-  
-  /**
-   * Gather files from the project
-   * @param includePaths Optional paths to include
-   * @param excludePaths Optional paths to exclude
-   */
-  public async gatherFiles(includePaths?: string[], excludePaths?: string[]): Promise<Record<string, plugins.smartfile.SmartFile | plugins.smartfile.SmartFile[]>> {
-    const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'package.json'),
-      this.projectDir,
-    );
-    
-    const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'readme.md'),
-      this.projectDir,
-    );
 
-    const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'readme.hints.md'),
-      this.projectDir,
-    );
-    
-    const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
-      plugins.path.join(this.projectDir, 'npmextra.json'),
-      this.projectDir,
-    );
-    
-    // Use provided include paths or default to all TypeScript files
-    const includeGlobs = includePaths?.map(path => `${path}/**/*.ts`) || ['ts*/**/*.ts'];
-    
-    // Get TypeScript files
-    const smartfilesModPromises = includeGlobs.map(glob => 
-      plugins.smartfile.fs.fileTreeToObject(this.projectDir, glob)
-    );
-    
-    const smartfilesModArrays = await Promise.all(smartfilesModPromises);
-    
-    // Flatten the arrays
-    const smartfilesMod: plugins.smartfile.SmartFile[] = [];
-    smartfilesModArrays.forEach(array => {
-      smartfilesMod.push(...array);
-    });
-    
-    // Get test files if not excluded
-    let smartfilesTest: plugins.smartfile.SmartFile[] = [];
-    if (!excludePaths?.includes('test/')) {
-      smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
-        this.projectDir,
-        'test/**/*.ts',
-      );
-    }
-    
-    return {
-      smartfilePackageJSON,
-      smartfilesReadme,
-      smartfilesReadmeHints,
-      smartfilesNpmextraJSON,
-      smartfilesMod,
-      smartfilesTest,
-    };
-  }
-  
-  /**
-   * Convert files to context string
-   * @param files The files to convert
-   * @param mode The context mode to use
-   */
-  public async convertFilesToContext(
-    files: plugins.smartfile.SmartFile[],
-    mode: ContextMode = this.contextMode
-  ): Promise<string> {
-    // Reset context result
-    this.contextResult = {
-      context: '',
-      tokenCount: 0,
-      includedFiles: [],
-      trimmedFiles: [],
-      excludedFiles: [],
-      tokenSavings: 0
-    };
-    
-    let totalTokenCount = 0;
-    let totalOriginalTokens = 0;
-
-    // Convert SmartFile objects to IFileMetadata for analysis
-    const metadata: IFileMetadata[] = files.map(sf => ({
-      path: sf.path,
-      relativePath: sf.relative,
-      size: sf.contents.toString().length,
-      mtime: Date.now(), // SmartFile doesn't expose mtime, use current time
-      estimatedTokens: this.countTokens(sf.contents.toString()),
-      importanceScore: 0
-    }));
-
-    // Analyze files using ContextAnalyzer to get smart prioritization
-    // (Note: This requires task type which we'll pass from buildContext)
-    // For now, sort files by estimated tokens (smaller files first for better efficiency)
-    const sortedFiles = [...files].sort((a, b) => {
-      const aTokens = this.countTokens(a.contents.toString());
-      const bTokens = this.countTokens(b.contents.toString());
-      return aTokens - bTokens;
-    });
-
-    const processedFiles: string[] = [];
-
-    for (const smartfile of sortedFiles) {
-      // Calculate original token count
-      const originalContent = smartfile.contents.toString();
-      const originalTokenCount = this.countTokens(originalContent);
-      totalOriginalTokens += originalTokenCount;
-      
-      // Apply trimming based on mode
-      let processedContent = originalContent;
-      
-      if (mode !== 'full') {
-        processedContent = this.trimmer.trimFile(
-          smartfile.relative,
-          originalContent,
-          mode
-        );
-      }
-      
-      // Calculate new token count
-      const processedTokenCount = this.countTokens(processedContent);
-      
-      // Check if we have budget for this file
-      if (totalTokenCount + processedTokenCount > this.tokenBudget) {
-        // We don't have budget for this file
-        this.contextResult.excludedFiles.push({
-          path: smartfile.path,
-          contents: originalContent,
-          relativePath: smartfile.relative,
-          tokenCount: originalTokenCount
-        });
-        continue;
-      }
-      
-      // Format the file for context
-      const formattedContent = `
-====== START OF FILE ${smartfile.relative} ======
-  
-${processedContent}
-  
-====== END OF FILE ${smartfile.relative} ======
-      `;
-      
-      processedFiles.push(formattedContent);
-      totalTokenCount += processedTokenCount;
-      
-      // Track file in appropriate list
-      const fileInfo: IFileInfo = {
-        path: smartfile.path,
-        contents: processedContent,
-        relativePath: smartfile.relative,
-        tokenCount: processedTokenCount
-      };
-      
-      if (mode === 'full' || processedContent === originalContent) {
-        this.contextResult.includedFiles.push(fileInfo);
-      } else {
-        this.contextResult.trimmedFiles.push(fileInfo);
-        this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
-      }
-    }
-    
-    // Join all processed files
-    const context = processedFiles.join('\n');
-    
-    // Update context result
-    this.contextResult.context = context;
-    this.contextResult.tokenCount = totalTokenCount;
-    
-    return context;
-  }
-  
   /**
    * Convert files to context with smart analysis and prioritization
    * @param metadata - File metadata to analyze
@@ -393,88 +219,45 @@ ${processedContent}
   }
 
   /**
-   * Build context for the project
-   * @param taskType Optional task type for task-specific context
+   * Build context for the project using smart analysis
+   * @param taskType Task type for context-aware prioritization (defaults to 'description')
    */
   public async buildContext(taskType?: TaskType): Promise<IContextResult> {
     // Initialize if needed
     if (this.tokenBudget === 0) {
       await this.initialize();
     }
-    
-    // Get task-specific configuration if a task type is provided
-    if (taskType) {
-      const taskConfig = this.configManager.getTaskConfig(taskType);
-      if (taskConfig.mode) {
-        this.setContextMode(taskConfig.mode);
-      }
-    }
-    
-    // Check if analyzer is enabled in config
-    const analyzerConfig = this.configManager.getAnalyzerConfig();
-    const useAnalyzer = analyzerConfig.enabled && taskType;
-
-    if (useAnalyzer) {
-      // Use new smart context building with lazy loading and analysis
-      const taskConfig = this.configManager.getTaskConfig(taskType!);
-
-      // Build globs for scanning
-      const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
-        'ts/**/*.ts',
-        'ts*/**/*.ts'
-      ];
-
-      // Add config files
-      const configGlobs = [
-        'package.json',
-        'readme.md',
-        'readme.hints.md',
-        'npmextra.json'
-      ];
-
-      // Scan files for metadata (fast, doesn't load contents)
-      const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
-
-      // Use analyzer to build context with smart prioritization
-      await this.convertFilesToContextWithAnalysis(metadata, taskType!, this.contextMode);
-    } else {
-      // Fall back to old method for backward compatibility
-      const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
-      const files = await this.gatherFiles(
-        taskConfig?.includePaths,
-        taskConfig?.excludePaths
-      );
-
-      // Convert files to context
-      // Create an array of all files to process
-      const allFiles: plugins.smartfile.SmartFile[] = [];
-
-      // Add individual files
-      if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
-      if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
-      if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
-      if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
-
-      // Add arrays of files
-      if (files.smartfilesMod) {
-        if (Array.isArray(files.smartfilesMod)) {
-          allFiles.push(...files.smartfilesMod);
-        } else {
-          allFiles.push(files.smartfilesMod);
-        }
-      }
 
-      if (files.smartfilesTest) {
-        if (Array.isArray(files.smartfilesTest)) {
-          allFiles.push(...files.smartfilesTest);
-        } else {
-          allFiles.push(files.smartfilesTest);
-        }
-      }
+    // Smart context building always requires a task type for optimal prioritization
+    // Default to 'description' if not provided
+    const effectiveTaskType = taskType || 'description';
 
-      await this.convertFilesToContext(allFiles);
+    // Get task-specific configuration
+    const taskConfig = this.configManager.getTaskConfig(effectiveTaskType);
+    if (taskConfig.mode) {
+      this.setContextMode(taskConfig.mode);
     }
 
+    // Build globs for scanning
+    const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
+      'ts/**/*.ts',
+      'ts*/**/*.ts'
+    ];
+
+    // Add config files
+    const configGlobs = [
+      'package.json',
+      'readme.md',
+      'readme.hints.md',
+      'npmextra.json'
+    ];
+
+    // Scan files for metadata (fast, doesn't load contents)
+    const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
+
+    // Use smart analyzer to build context with intelligent prioritization
+    await this.convertFilesToContextWithAnalysis(metadata, effectiveTaskType, this.contextMode);
+
     return this.contextResult;
   }
   

package/ts/context/index.ts

@@ -22,7 +22,9 @@ import type {
   ICacheEntry,
   IFileDependencies,
   IFileAnalysis,
-  IAnalysisResult
+  IAnalysisResult,
+  IIterativeConfig,
+  IIterativeContextResult
 } from './types.js';
 
 export {
@@ -54,5 +56,7 @@ export type {
   ICacheEntry,
   IFileDependencies,
   IFileAnalysis,
-  IAnalysisResult
+  IAnalysisResult,
+  IIterativeConfig,
+  IIterativeContextResult
 };