@git.zone/tsdoc 1.11.1 → 1.11.2

package/ts/aidocs_classes/description.ts

@@ -1,6 +1,7 @@
 import type { AiDoc } from '../classes.aidoc.js';
 import * as plugins from '../plugins.js';
 import { ProjectContext } from './projectcontext.js';
+import { logger } from '../logging.js';
 
 interface IDescriptionInterface {
   description: string;
@@ -18,50 +19,87 @@ export class Description {
   }
 
   public async build() {
-    // Use the new TaskContextFactory for optimized context
-    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
-      this.projectDir,
-      this.aiDocsRef.openaiInstance
-    );
-    await taskContextFactory.initialize();
-    
-    // Generate context specifically for description task
-    const contextResult = await taskContextFactory.createContextForDescription();
-    const contextString = contextResult.context;
-    
-    // Log token usage statistics
-    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
-
-    let result = await this.aiDocsRef.openaiInstance.chat({
-      systemMessage: `
-You create a json adhering the following interface:
+    // Use DualAgentOrchestrator with filesystem tool for agent-driven exploration
+    const descriptionOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
+      smartAiInstance: this.aiDocsRef.smartAiInstance,
+      defaultProvider: 'openai',
+      maxIterations: 15,
+      maxResultChars: 10000, // Limit tool output to prevent token explosion
+      maxHistoryMessages: 15, // Limit history window
+      logPrefix: '[Description]',
+      onProgress: (event) => logger.log(event.logLevel, event.logMessage),
+      guardianPolicyPrompt: `
+You validate description generation tool calls and outputs.
+
+APPROVE tool calls for:
+- Reading package.json, npmextra.json, or source files in the ts/ directory
+- Listing directory contents to understand project structure
+- Using tree to see project structure
+
+REJECT tool calls for:
+- Reading files outside the project directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+For final output, APPROVE if:
+- JSON is valid and parseable
+- Description is a clear, concise one-sentence summary
+- Keywords are relevant to the project's use cases
+- Both description and keywords fields are present
+
+REJECT final output if:
+- JSON is malformed or wrapped in markdown code blocks
+- Description is too long or vague
+- Keywords are irrelevant or generic
+`,
+    });
+
+    // Register scoped filesystem tool for agent exploration
+    descriptionOrchestrator.registerScopedFilesystemTool(this.projectDir);
+
+    await descriptionOrchestrator.start();
+
+    const descriptionTaskPrompt = `
+You create a project description and keywords for an npm package.
+
+PROJECT DIRECTORY: ${this.projectDir}
+
+Use the filesystem tool to explore the project and understand what it does:
+1. First, use tree to see the project structure
+2. Read package.json to understand the package name and current description
+3. Read npmextra.json if it exists for additional metadata
+4. Read key source files in ts/ directory to understand the implementation
+
+Then generate a description and keywords based on your exploration.
+
+Your FINAL response must be valid JSON adhering to this interface:
 {
   description: string; // a sensible short, one sentence description of the project
-  keywords: string[]; // an array of tags that describe the project
+  keywords: string[]; // an array of tags that describe the project based on use cases
 }
 
-The description should be based on what you understand from the project's files.
-The keywords should be based on use cases you see from the files.
-Don't be cheap about the way you think.
-
 Important: Answer only in valid JSON.
-You answer should be parseable with JSON.parse() without modifying anything.
+Your answer should be parseable with JSON.parse() without modifying anything.
+Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object.
+`;
 
-Don't wrap the JSON in three ticks json!!!
-    `,
-      messageHistory: [],
-      userMessage: contextString,
-    });
+    const descriptionResult = await descriptionOrchestrator.run(descriptionTaskPrompt);
+    await descriptionOrchestrator.stop();
 
-    console.log(result.message);
+    if (!descriptionResult.success) {
+      throw new Error(`Description generation failed: ${descriptionResult.status}`);
+    }
+
+    console.log(descriptionResult.result);
     const resultObject: IDescriptionInterface = JSON.parse(
-      result.message.replace('```json', '').replace('```', ''),
+      descriptionResult.result.replace('```json', '').replace('```', ''),
     );
 
-    // Create a standard ProjectContext instance for file operations
+    // Use ProjectContext to get file handles for writing
     const projectContext = new ProjectContext(this.projectDir);
     const files = await projectContext.gatherFiles();
-    
+
+    // Update npmextra.json
     const npmextraJson = files.smartfilesNpmextraJSON;
     const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
 
@@ -71,7 +109,7 @@ Don't wrap the JSON in three ticks json!!!
     npmextraJson.contents = Buffer.from(JSON.stringify(npmextraJsonContent, null, 2));
     await npmextraJson.write();
 
-    // do the same with packageJson
+    // Update package.json
     const packageJson = files.smartfilePackageJSON;
     const packageJsonContent = JSON.parse(packageJson.contents.toString());
     packageJsonContent.description = resultObject.description;
@@ -82,6 +120,6 @@ Don't wrap the JSON in three ticks json!!!
     console.log(`\n======================\n`);
     console.log(JSON.stringify(resultObject, null, 2));
     console.log(`\n======================\n`);
-    return result.message;
+    return descriptionResult.result;
   }
 }

package/ts/aidocs_classes/projectcontext.ts

@@ -64,21 +64,14 @@ ${smartfile.contents.toString()}
   }
 
   /**
-   * Calculate the token count for a string using the GPT tokenizer
-   * @param text The text to count tokens for
-   * @param model The model to use for token counting (default: gpt-3.5-turbo)
-   * @returns The number of tokens in the text
+   * Estimate token count for a string
+   * Uses a rough estimate of 4 characters per token
+   * @param text The text to estimate tokens for
+   * @returns Estimated number of tokens
    */
-  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
-    try {
-      // Use the gpt-tokenizer library to count tokens
-      const tokens = plugins.gptTokenizer.encode(text);
-      return tokens.length;
-    } catch (error) {
-      console.error('Error counting tokens:', error);
-      // Provide a rough estimate (4 chars per token) if tokenization fails
-      return Math.ceil(text.length / 4);
-    }
+  public countTokens(text: string): number {
+    // Rough estimate: ~4 characters per token for English text
+    return Math.ceil(text.length / 4);
   }
 
   private async buildContext(dirArg: string) {

package/ts/aidocs_classes/readme.ts

@@ -17,21 +17,7 @@ export class Readme {
   public async build() {
     let finalReadmeString = ``;
 
-    // Use the new TaskContextFactory for optimized context
-    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
-      this.projectDir,
-      this.aiDocsRef.openaiInstance
-    );
-    await taskContextFactory.initialize();
-    
-    // Generate context specifically for readme task
-    const contextResult = await taskContextFactory.createContextForReadme();
-    const contextString = contextResult.context;
-    
-    // Log token usage statistics
-    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
-
-    // lets first check legal before introducung any cost
+    // First check legal info before introducing any cost
     const projectContext = new ProjectContext(this.projectDir);
     const npmExtraJson = JSON.parse(
       (await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
@@ -42,50 +28,100 @@ export class Readme {
       console.log(error);
     }
 
-    let result = await this.aiDocsRef.openaiInstance.chat({
-      systemMessage: `
-You create markdown readmes for npm projects. You only output the markdown readme.
+    // Use DualAgentOrchestrator with filesystem tool for agent-driven exploration
+    const readmeOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
+      smartAiInstance: this.aiDocsRef.smartAiInstance,
+      defaultProvider: 'openai',
+      maxIterations: 25,
+      maxResultChars: 15000, // Limit tool output to prevent token explosion
+      maxHistoryMessages: 20, // Limit history window
+      logPrefix: '[README]',
+      onProgress: (event) => logger.log(event.logLevel, event.logMessage),
+      guardianPolicyPrompt: `
+You validate README generation tool calls and outputs.
+
+APPROVE tool calls for:
+- Reading any files within the project directory (package.json, ts/*.ts, readme.md, etc.)
+- Using tree to see project structure
+- Using glob to find source files
+- Listing directory contents
+
+REJECT tool calls for:
+- Reading files outside the project directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+For final README output, APPROVE if:
+- README follows proper markdown format
+- Contains Install and Usage sections
+- Code examples are correct TypeScript/ESM syntax
+- Documentation is comprehensive and helpful
+
+REJECT final output if:
+- README is incomplete or poorly formatted
+- Contains licensing information (added separately)
+- Uses CommonJS syntax instead of ESM
+- Contains "in conclusion" or similar filler
+`,
+    });
+
+    // Register scoped filesystem tool for agent exploration
+    readmeOrchestrator.registerScopedFilesystemTool(this.projectDir);
+
+    await readmeOrchestrator.start();
+
+    const readmeTaskPrompt = `
+You create markdown READMEs for npm projects. You only output the markdown readme.
 
-The Readme should follow the following template:
+PROJECT DIRECTORY: ${this.projectDir}
+
+Use the filesystem tool to explore the project and understand what it does:
+1. First, use tree to see the project structure (maxDepth: 3)
+2. Read package.json to understand the package name, description, and dependencies
+3. Read the existing readme.md if it exists (use it as a base, improve and expand)
+4. Read readme.hints.md if it exists (contains hints for documentation)
+5. Read key source files in ts/ directory to understand the API and implementation
+6. Focus on exported classes, interfaces, and functions
+
+Then generate a comprehensive README following this template:
 
 # Project Name
-[
-  The name is the module name of package.json
-  The description is in the description field of package.json
-]
+[The name from package.json and description]
 
 ## Install
-[
-  Write a short text on how to install the project
-]
+[Short text on how to install the project]
 
 ## Usage
-[ 
+[
   Give code examples here.
   Construct sensible scenarios for the user.
   Make sure to show a complete set of features of the module.
   Don't omit use cases.
-  It does not matter how much time you need.
   ALWAYS USE ESM SYNTAX AND TYPESCRIPT.
-  DON'T CHICKEN OUT. Write at least 4000 words. More if necessary.
-  If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts.
-  Super important: Check for completenes.
-  Don't include any licensing information. This will be added in a later step.
-  Avoid "in conclusions".
-
-  Good to know:
-  * npmextra.json contains overall module information.
-  * readme.hints.md provides valuable hints about module ideas.
+  Write at least 4000 words. More if necessary.
+  If there is already a readme, take the Usage section as base. Remove outdated content, expand and improve.
+  Check for completeness.
+  Don't include any licensing information. This will be added later.
+  Avoid "in conclusion" statements.
 ]
-            `,
-      messageHistory: [],
-      userMessage: contextString,
-    });
+`;
+
+    const readmeResult = await readmeOrchestrator.run(readmeTaskPrompt);
+    await readmeOrchestrator.stop();
+
+    if (!readmeResult.success) {
+      throw new Error(`README generation failed: ${readmeResult.status}`);
+    }
+
+    // Clean up markdown formatting if wrapped in code blocks
+    let resultMessage = readmeResult.result
+      .replace(/^```markdown\n?/i, '')
+      .replace(/\n?```$/i, '');
 
-    finalReadmeString += result.message + '\n' + legalInfo;
+    finalReadmeString += resultMessage + '\n' + legalInfo;
 
     console.log(`\n======================\n`);
-    console.log(result.message);
+    console.log(resultMessage);
     console.log(`\n======================\n`);
 
     const readme = (await projectContext.gatherFiles()).smartfilesReadme;
@@ -96,60 +132,99 @@ The Readme should follow the following template:
     const tsPublishInstance = new plugins.tspublish.TsPublish();
     const subModules = await tsPublishInstance.getModuleSubDirs(paths.cwd);
     logger.log('info', `Found ${Object.keys(subModules).length} sub modules`);
+
     for (const subModule of Object.keys(subModules)) {
       logger.log('info', `Building readme for ${subModule}`);
-      const subModuleContextString = await projectContext.update();
-      let result = await this.aiDocsRef.openaiInstance.chat({
-        systemMessage: `
-  You create markdown readmes for npm projects. You only output the markdown readme.
-
-  IMPORTANT: YOU ARE NOW CREATING THE README FOR THE FOLLOWING SUB MODULE: ${subModule} !!!!!!!!!!!
-  The Sub Module will be published with the following data:
-  ${JSON.stringify(await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'tspublish.json')).encoding('utf8').read(), null, 2)}
-
-  
-  The Readme should follow the following template:
-  
-  # Project Name
-  [
-    The name is the module name of package.json
-    The description is in the description field of package.json
-  ]
-  
-  ## Install
-  [
-    Write a short text on how to install the project
-  ]
-  
-  ## Usage
-  [ 
-    Give code examples here.
-    Construct sensible scenarios for the user.
-    Make sure to show a complete set of features of the module.
-    Don't omit use cases.
-    It does not matter how much time you need.
-    ALWAYS USE ESM SYNTAX AND TYPESCRIPT.
-    DON'T CHICKEN OUT. Write at least 4000 words. More if necessary.
-    If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts.
-    Super important: Check for completenes.
-    Don't include any licensing information. This will be added in a later step.
-    Avoid "in conclusions".
-  
-    Good to know:
-    * npmextra.json contains overall module information.
-    * readme.hints.md provides valuable hints about module ideas.
-    * Your output lands directly in the readme.md file.
-    * Don't use \`\`\` at the beginning or the end. It'll cause problems. Only use it for codeblocks. You are directly writing markdown. No need to introduce it weirdly.
-  ]
-              `,
-        messageHistory: [],
-        userMessage: subModuleContextString,
+
+      const subModulePath = plugins.path.join(paths.cwd, subModule);
+      const tspublishData = await plugins.fsInstance
+        .file(plugins.path.join(subModulePath, 'tspublish.json'))
+        .encoding('utf8')
+        .read();
+
+      // Create a new orchestrator with filesystem tool for each submodule
+      const subModuleOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
+        smartAiInstance: this.aiDocsRef.smartAiInstance,
+        defaultProvider: 'openai',
+        maxIterations: 20,
+        maxResultChars: 12000,
+        maxHistoryMessages: 15,
+        logPrefix: `[README:${subModule}]`,
+        onProgress: (event) => logger.log(event.logLevel, event.logMessage),
+        guardianPolicyPrompt: `
+You validate README generation for submodules.
+
+APPROVE tool calls for:
+- Reading any files within the submodule directory
+- Using tree to see structure
+- Using glob to find source files
+
+REJECT tool calls for:
+- Reading files outside the submodule directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+APPROVE final README if comprehensive, well-formatted markdown with ESM TypeScript examples.
+REJECT incomplete READMEs or those with licensing info.
+`,
       });
 
-      const subModuleReadmeString = result.message + '\n' + legalInfo;
-      await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'readme.md')).encoding('utf8').write(subModuleReadmeString);
-      logger.log('success', `Built readme for ${subModule}`);
+      // Register scoped filesystem tool for the submodule directory
+      subModuleOrchestrator.registerScopedFilesystemTool(subModulePath);
+
+      await subModuleOrchestrator.start();
+
+      const subModulePrompt = `
+You create markdown READMEs for npm projects. You only output the markdown readme.
+SUB MODULE: ${subModule}
+SUB MODULE DIRECTORY: ${subModulePath}
+
+IMPORTANT: YOU ARE CREATING THE README FOR THIS SUB MODULE: ${subModule}
+The Sub Module will be published with:
+${JSON.stringify(tspublishData, null, 2)}
+
+Use the filesystem tool to explore the submodule:
+1. Use tree to see the submodule structure
+2. Read package.json to understand the submodule
+3. Read source files in ts/ directory to understand the implementation
+
+Generate a README following the template:
+
+# Project Name
+[name and description from package.json]
+
+## Install
+[installation instructions]
+
+## Usage
+[
+  Code examples with complete features.
+  ESM TypeScript syntax only.
+  Write at least 4000 words.
+  No licensing information.
+  No "in conclusion".
+]
+
+Don't use \`\`\` at the beginning or end. Only for code blocks.
+`;
+
+      const subModuleResult = await subModuleOrchestrator.run(subModulePrompt);
+      await subModuleOrchestrator.stop();
+
+      if (subModuleResult.success) {
+        const subModuleReadmeString = subModuleResult.result
+          .replace(/^```markdown\n?/i, '')
+          .replace(/\n?```$/i, '') + '\n' + legalInfo;
+        await plugins.fsInstance
+          .file(plugins.path.join(subModulePath, 'readme.md'))
+          .encoding('utf8')
+          .write(subModuleReadmeString);
+        logger.log('success', `Built readme for ${subModule}`);
+      } else {
+        logger.log('error', `Failed to build readme for ${subModule}: ${subModuleResult.status}`);
+      }
     }
-    return result.message;
+
+    return resultMessage;
   }
 }

package/ts/classes.aidoc.ts

@@ -8,7 +8,7 @@ export class AiDoc {
   public npmextraKV: plugins.npmextra.KeyValueStore;
   public qenvInstance: plugins.qenv.Qenv;
   public aidocInteract: plugins.smartinteract.SmartInteract;
-  public openaiInstance: plugins.smartai.OpenAiProvider;
+  public smartAiInstance: plugins.smartai.SmartAi;
 
   argvArg: any;
 
@@ -85,20 +85,28 @@ export class AiDoc {
     }
 
     // lets assume we have an OPENAI_Token now
-    this.openaiInstance = new plugins.smartai.OpenAiProvider({
+    this.smartAiInstance = new plugins.smartai.SmartAi({
       openaiToken: this.openaiToken,
     });
-    await this.openaiInstance.start();
+    await this.smartAiInstance.start();
   }
 
   public async stop() {
-    if (this.openaiInstance) {
-      await this.openaiInstance.stop();
+    if (this.smartAiInstance) {
+      await this.smartAiInstance.stop();
     }
     // No explicit cleanup needed for npmextraKV or aidocInteract
     // They don't keep event loop alive
   }
 
+  /**
+   * Get the OpenAI provider for direct chat calls
+   * This is a convenience getter to access the provider from SmartAi
+   */
+  public get openaiProvider(): plugins.smartai.OpenAiProvider {
+    return this.smartAiInstance.openaiProvider;
+  }
+
   public getOpenaiToken(): string {
     return this.openaiToken;
   }
@@ -146,13 +154,12 @@ export class AiDoc {
   }
   
   /**
-   * Count tokens in a text string using GPT tokenizer
-   * @param text The text to count tokens for
-   * @param model The model to use for tokenization (default: gpt-3.5-turbo)
-   * @returns The number of tokens in the text
+   * Estimate token count in a text string
+   * @param text The text to estimate tokens for
+   * @returns Estimated number of tokens
    */
-  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
+  public countTokens(text: string): number {
     const projectContextInstance = new aiDocsClasses.ProjectContext('');
-    return projectContextInstance.countTokens(text, model);
+    return projectContextInstance.countTokens(text);
   }
 }