@git.zone/tsdoc 2.0.5 → 2.1.0

package/ts/aidocs_classes/commit.ts

@@ -1,8 +1,8 @@
 import * as plugins from '../plugins.js';
 import { AiDoc } from '../classes.aidoc.js';
-import { ProjectContext } from './projectcontext.js';
 import { DiffProcessor } from '../classes.diffprocessor.js';
 import { logger } from '../logging.js';
+import { createReadOnlyFileSystemTools } from '../helpers.agenttools.js';
 
 // Token budget configuration for OpenAI API limits
 const TOKEN_BUDGET = {
@@ -32,6 +32,88 @@ export interface INextCommitObject {
   changelog?: string; // the changelog for the next version
 }
 
+export class NoChangesError extends Error {
+  constructor(message = 'No uncommitted changes found for commit recommendation.') {
+    super(message);
+    this.name = 'NoChangesError';
+  }
+}
+
+const normalizeLevel = (level: unknown): INextCommitObject['recommendedNextVersionLevel'] => {
+  if (level === 'fix' || level === 'feat' || level === 'BREAKING CHANGE') return level;
+  throw new Error('recommendedNextVersionLevel must be fix, feat, or BREAKING CHANGE.');
+};
+
+const stripConventionalPrefix = (message: string): string => {
+  return message.replace(/^(fix|feat|BREAKING CHANGE)(\([^)]*\))?:\s*/i, '').trim();
+};
+
+const extractJsonObject = (text: string): Record<string, any> => {
+  const jsonString = text
+    .replace(/```json\n?/gi, '')
+    .replace(/```\n?/gi, '')
+    .match(/\{[\s\S]*\}/)?.[0];
+  if (!jsonString) {
+    throw new Error(`Could not find JSON object in result: ${text.substring(0, 100)}...`);
+  }
+  return JSON.parse(jsonString) as Record<string, any>;
+};
+
+const bumpVersion = (currentVersion: string, level: INextCommitObject['recommendedNextVersionLevel']): string => {
+  const [majorString, minorString, patchString] = currentVersion.split(/[+-]/)[0].split('.');
+  let major = Number.parseInt(majorString, 10);
+  let minor = Number.parseInt(minorString, 10);
+  let patch = Number.parseInt(patchString, 10);
+  if (!Number.isFinite(major)) major = 0;
+  if (!Number.isFinite(minor)) minor = 0;
+  if (!Number.isFinite(patch)) patch = 0;
+
+  if (level === 'BREAKING CHANGE') {
+    return `${major + 1}.0.0`;
+  }
+  if (level === 'feat') {
+    return `${major}.${minor + 1}.0`;
+  }
+  return `${major}.${minor}.${patch + 1}`;
+};
+
+const parseCommitResult = (text: string, currentVersion: string): INextCommitObject => {
+  const parsed = extractJsonObject(text);
+  const level = normalizeLevel(parsed.recommendedNextVersionLevel);
+  const scope = typeof parsed.recommendedNextVersionScope === 'string'
+    ? parsed.recommendedNextVersionScope.trim()
+    : '';
+  const message = typeof parsed.recommendedNextVersionMessage === 'string'
+    ? stripConventionalPrefix(parsed.recommendedNextVersionMessage)
+    : '';
+  const details = Array.isArray(parsed.recommendedNextVersionDetails)
+    ? parsed.recommendedNextVersionDetails.filter((detail): detail is string => typeof detail === 'string').map(detail => detail.trim()).filter(Boolean)
+    : [];
+
+  if (!scope) throw new Error('recommendedNextVersionScope must be a non-empty string.');
+  if (!message) throw new Error('recommendedNextVersionMessage must be a non-empty string.');
+
+  return {
+    recommendedNextVersionLevel: level,
+    recommendedNextVersionScope: scope,
+    recommendedNextVersionMessage: message,
+    recommendedNextVersionDetails: details,
+    recommendedNextVersion: bumpVersion(currentVersion, level),
+  };
+};
+
+const buildChangelog = (
+  commitObject: INextCommitObject,
+  oldChangelog: string,
+): string => {
+  const dateString = new plugins.smarttime.ExtendedDate().exportToHyphedSortableDate();
+  const details = commitObject.recommendedNextVersionDetails.length > 0
+    ? `\n\n${commitObject.recommendedNextVersionDetails.map(detail => `- ${detail}`).join('\n')}`
+    : '';
+  const previous = oldChangelog.replace(/^# Changelog\n\n?/, '').replace(/^\n+/, '');
+  return `# Changelog\n\n## ${dateString} - ${commitObject.recommendedNextVersion} - ${commitObject.recommendedNextVersionScope}\n${commitObject.recommendedNextVersionMessage}${details}\n\n${previous}`.trimEnd() + '\n';
+};
+
 export class Commit {
   private aiDocsRef: AiDoc;
   private projectDir: string;
@@ -94,62 +176,45 @@ export class Commit {
 
     // Pass glob patterns directly to smartgit - it handles matching internally
     const diffStringArray = await gitRepo.getUncommittedDiff(excludePatterns);
+    if (diffStringArray.length === 0) {
+      throw new NoChangesError();
+    }
+
+    const packageJsonPath = plugins.path.join(this.projectDir, 'package.json');
+    const packageJson = await plugins.fsInstance.file(packageJsonPath).exists()
+      ? JSON.parse(String(await plugins.fsInstance.file(packageJsonPath).encoding('utf8').read()))
+      : {};
+    const currentVersion = typeof packageJson.version === 'string' ? packageJson.version : '0.0.0';
 
     // Process diffs intelligently using DiffProcessor
-    let processedDiffString: string;
-
-    if (diffStringArray.length > 0) {
-      // Diagnostic logging for raw diff statistics
-      const totalChars = diffStringArray.join('\n\n').length;
-      const estimatedTokens = Math.ceil(totalChars / 4);
-
-      console.log(`Raw git diff statistics:`);
-      console.log(`   Files changed: ${diffStringArray.length}`);
-      console.log(`   Total characters: ${totalChars.toLocaleString()}`);
-      console.log(`   Estimated tokens: ${estimatedTokens.toLocaleString()}`);
-      console.log(`   Exclusion patterns: ${excludePatterns.length}`);
-
-      // Calculate available tokens for diff based on total budget
-      const maxDiffTokens = calculateMaxDiffTokens();
-      console.log(`Token budget: ${maxDiffTokens.toLocaleString()} tokens for diff (limit: ${TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT.toLocaleString()}, overhead: ${(TOKEN_BUDGET.SMARTAGENT_OVERHEAD + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD).toLocaleString()})`);
-
-      // Use DiffProcessor to intelligently handle large diffs
-      const diffProcessor = new DiffProcessor({
-        maxDiffTokens,               // Dynamic based on total budget
-        smallFileLines: 300,         // Most source files are under 300 lines
-        mediumFileLines: 800,        // Only very large files get head/tail treatment
-        sampleHeadLines: 75,         // When sampling, show more context
-        sampleTailLines: 75,         // When sampling, show more context
-      });
-
-      const processedDiff = diffProcessor.processDiffs(diffStringArray);
-      processedDiffString = diffProcessor.formatForContext(processedDiff);
-
-      console.log(`Processed diff statistics:`);
-      console.log(`   Full diffs: ${processedDiff.fullDiffs.length} files`);
-      console.log(`   Summarized: ${processedDiff.summarizedDiffs.length} files`);
-      console.log(`   Metadata only: ${processedDiff.metadataOnly.length} files`);
-      console.log(`   Final tokens: ${processedDiff.totalTokens.toLocaleString()}`);
-
-      if (estimatedTokens > 50000) {
-        console.log(`DiffProcessor reduced token usage: ${estimatedTokens.toLocaleString()} -> ${processedDiff.totalTokens.toLocaleString()}`);
-      }
-
-      // Validate total tokens won't exceed limit
-      const totalEstimatedTokens = processedDiff.totalTokens
-        + TOKEN_BUDGET.SMARTAGENT_OVERHEAD
-        + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
-
-      if (totalEstimatedTokens > TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT - TOKEN_BUDGET.SAFETY_MARGIN) {
-        console.log(`Warning: Estimated tokens (${totalEstimatedTokens.toLocaleString()}) approaching limit`);
-        console.log(`   Consider splitting into smaller commits`);
-      }
-    } else {
-      processedDiffString = 'No changes.';
+    const totalChars = diffStringArray.join('\n\n').length;
+    const estimatedTokens = Math.ceil(totalChars / 4);
+
+    logger.log('info', `Raw git diff: ${diffStringArray.length} files, ${estimatedTokens.toLocaleString()} estimated tokens, ${excludePatterns.length} exclusions.`);
+
+    const maxDiffTokens = calculateMaxDiffTokens();
+    const diffProcessor = new DiffProcessor({
+      maxDiffTokens,
+      smallFileLines: 300,
+      mediumFileLines: 800,
+      sampleHeadLines: 75,
+      sampleTailLines: 75,
+    });
+
+    const processedDiff = diffProcessor.processDiffs(diffStringArray);
+    const processedDiffString = diffProcessor.formatForContext(processedDiff);
+
+    logger.log('info', `Processed diff: ${processedDiff.fullDiffs.length} full, ${processedDiff.summarizedDiffs.length} summarized, ${processedDiff.metadataOnly.length} metadata-only, ${processedDiff.totalTokens.toLocaleString()} tokens.`);
+
+    const totalEstimatedTokens = processedDiff.totalTokens
+      + TOKEN_BUDGET.SMARTAGENT_OVERHEAD
+      + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
+
+    if (totalEstimatedTokens > TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT - TOKEN_BUDGET.SAFETY_MARGIN) {
+      logger.log('warn', `Estimated tokens (${totalEstimatedTokens.toLocaleString()}) approach the model context limit. Consider splitting into smaller commits.`);
     }
 
-    // Use runAgent for commit message generation with filesystem tool
-    const fsTools = plugins.smartagentTools.filesystemTool({ rootDir: this.projectDir });
+    const fsTools = await createReadOnlyFileSystemTools(this.projectDir);
 
     const commitSystemPrompt = `
 You create commit messages for git commits following semantic versioning conventions.
@@ -188,7 +253,7 @@ Here is the structure of the JSON you must return:
   "recommendedNextVersionScope": "string",
   "recommendedNextVersionMessage": "string (ONLY the description body WITHOUT the type(scope): prefix - e.g. 'bump dependency to ^1.2.6' NOT 'fix(deps): bump dependency to ^1.2.6')",
   "recommendedNextVersionDetails": ["string"],
-  "recommendedNextVersion": "x.x.x"
+  "recommendedNextVersion": "x.x.x (will be verified deterministically)"
 }
 
 For recommendedNextVersionDetails, only add entries that have obvious value to the reader.
@@ -202,97 +267,32 @@ Analyze these changes and output the JSON commit message object.
 
     logger.log('info', 'Starting commit message generation with agent...');
 
-    const commitResult = await plugins.smartagent.runAgent({
-      model: this.aiDocsRef.model,
+    const commitResult = await this.aiDocsRef.runAgent({
+      taskName: 'commit',
+      projectDir: this.projectDir,
       prompt: commitTaskPrompt,
       system: commitSystemPrompt,
       tools: fsTools,
       maxSteps: 10,
+      maxValidationRetries: 1,
+      validateCompletion: (result) => {
+        try {
+          parseCommitResult(result.text, currentVersion);
+        } catch (error) {
+          return `Return only valid JSON matching the requested commit object schema. Error: ${error instanceof Error ? error.message : String(error)}`;
+        }
+      },
       onToolCall: (toolName) => logger.log('info', `[Commit] Tool call: ${toolName}`),
     });
 
-    // Extract JSON from result - handle cases where AI adds text around it
-    let jsonString = commitResult.text
-      .replace(/```json\n?/gi, '')
-      .replace(/```\n?/gi, '');
-
-    // Try to find JSON object in the result
-    const jsonMatch = jsonString.match(/\{[\s\S]*\}/);
-    if (!jsonMatch) {
-      throw new Error(`Could not find JSON object in result: ${jsonString.substring(0, 100)}...`);
-    }
-    jsonString = jsonMatch[0];
-
-    const resultObject: INextCommitObject = JSON.parse(jsonString);
+    const resultObject = parseCommitResult(commitResult.text, currentVersion);
 
     const previousChangelogPath = plugins.path.join(this.projectDir, 'changelog.md');
-    let previousChangelog: plugins.smartfile.SmartFile | undefined;
+    let oldChangelog = '';
     if (await plugins.fsInstance.file(previousChangelogPath).exists()) {
-      previousChangelog = await plugins.smartfileFactory.fromFilePath(previousChangelogPath);
+      oldChangelog = String(await plugins.fsInstance.file(previousChangelogPath).encoding('utf8').read());
     }
-
-    if (!previousChangelog) {
-      // lets build the changelog based on that
-      const commitMessages = await gitRepo.getAllCommitMessages();
-      console.log(JSON.stringify(commitMessages, null, 2));
-
-      const changelogSystemPrompt = `
-You generate changelog.md files for software projects.
-
-RULES:
-- Changelog must follow proper markdown format with ## headers for each version
-- Entries must be chronologically ordered (newest first)
-- Version ranges for trivial commits should be properly summarized
-- No duplicate or empty entries
-- Format: ## yyyy-mm-dd - x.x.x - scope
-`;
-
-      const changelogTaskPrompt = `
-You are building a changelog.md file for the project.
-Omit commits and versions that lack relevant changes, but make sure to mention them as a range with a summarizing message instead.
-
-A changelog entry should look like this:
-
-    ## yyyy-mm-dd - x.x.x - scope here
-    main description here
-
-    - detailed bullet points follow
-
-You are given:
-* the commit messages of the project
-
-Only return the changelog file content, so it can be written directly to changelog.md.
-
-Here are the commit messages:
-
-${JSON.stringify(commitMessages, null, 2)}
-`;
-
-      const changelogResult = await plugins.smartagent.runAgent({
-        model: this.aiDocsRef.model,
-        prompt: changelogTaskPrompt,
-        system: changelogSystemPrompt,
-        maxSteps: 1,
-        onToolCall: (toolName) => logger.log('info', `[Changelog] Tool call: ${toolName}`),
-      });
-
-      previousChangelog = plugins.smartfileFactory.fromString(
-        previousChangelogPath,
-        changelogResult.text.replaceAll('```markdown', '').replaceAll('```', ''),
-        'utf8'
-      );
-    }
-
-    let oldChangelog = previousChangelog.contents.toString().replace('# Changelog\n\n', '');
-    if (oldChangelog.startsWith('\n')) {
-      oldChangelog = oldChangelog.replace('\n', '');
-    }
-    let newDateString = new plugins.smarttime.ExtendedDate().exportToHyphedSortableDate();
-    let newChangelog = `# Changelog\n\n${`## ${newDateString} - {{nextVersion}} - {{nextVersionScope}}
-{{nextVersionMessage}}
-
-{{nextVersionDetails}}`}\n\n${oldChangelog}`;
-    resultObject.changelog = newChangelog;
+    resultObject.changelog = buildChangelog(resultObject, oldChangelog);
 
     return resultObject;
   }

package/ts/aidocs_classes/description.ts

@@ -2,12 +2,38 @@ import type { AiDoc } from '../classes.aidoc.js';
 import * as plugins from '../plugins.js';
 import { ProjectContext } from './projectcontext.js';
 import { logger } from '../logging.js';
+import { createReadOnlyFileSystemTools } from '../helpers.agenttools.js';
 
 interface IDescriptionInterface {
   description: string;
   keywords: string[];
 }
 
+const parseDescriptionJson = (text: string): IDescriptionInterface => {
+  const jsonString = text
+    .replace(/```json\n?/gi, '')
+    .replace(/```\n?/gi, '')
+    .match(/\{[\s\S]*\}/)?.[0] ?? text;
+  const parsed = JSON.parse(jsonString) as IDescriptionInterface;
+  if (typeof parsed.description !== 'string' || parsed.description.trim().length === 0) {
+    throw new Error('description must be a non-empty string.');
+  }
+  if (!Array.isArray(parsed.keywords) || parsed.keywords.some(keyword => typeof keyword !== 'string')) {
+    throw new Error('keywords must be an array of strings.');
+  }
+  return {
+    description: parsed.description.trim(),
+    keywords: parsed.keywords.map(keyword => keyword.trim()).filter(Boolean),
+  };
+};
+
+const ensureModuleConfig = (config: Record<string, any>): Record<string, any> => {
+  if (!config.module || typeof config.module !== 'object') {
+    config.module = {};
+  }
+  return config.module;
+};
+
 export class Description {
   // INSTANCE
   private aiDocsRef: AiDoc;
@@ -19,8 +45,7 @@ export class Description {
   }
 
   public async build() {
-    // Use runAgent with filesystem tool for agent-driven exploration
-    const fsTools = plugins.smartagentTools.filesystemTool({ rootDir: this.projectDir });
+    const fsTools = await createReadOnlyFileSystemTools(this.projectDir);
 
     const descriptionSystemPrompt = `
 You create project descriptions and keywords for npm packages.
@@ -28,7 +53,7 @@ You create project descriptions and keywords for npm packages.
 You have access to filesystem tools to explore the project.
 
 IMPORTANT RULES:
-- Only READ files (package.json, .smartconfig.json, source files in ts/)
+- Only READ files (package.json, .smartconfig.json, source files in ts/ and ts_web/)
 - Do NOT write, delete, or modify any files
 - Your final response must be valid JSON only
 - Description must be a clear, concise one-sentence summary
@@ -44,7 +69,7 @@ Use the filesystem tools to explore the project and understand what it does:
 1. First, use list_directory to see the project structure
 2. Read package.json to understand the package name and current description
 3. Read .smartconfig.json if it exists for additional metadata
-4. Read key source files in ts/ directory to understand the implementation
+4. Read key source files in ts/ and ts_web/ directories to understand the implementation
 
 Then generate a description and keywords based on your exploration.
 
@@ -61,19 +86,26 @@ Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object.
 
     logger.log('info', 'Starting description generation with agent...');
 
-    const descriptionResult = await plugins.smartagent.runAgent({
-      model: this.aiDocsRef.model,
+    const descriptionResult = await this.aiDocsRef.runAgent({
+      taskName: 'description',
+      projectDir: this.projectDir,
       prompt: descriptionTaskPrompt,
       system: descriptionSystemPrompt,
       tools: fsTools,
       maxSteps: 15,
+      useCompaction: true,
+      maxValidationRetries: 1,
+      validateCompletion: (result) => {
+        try {
+          parseDescriptionJson(result.text);
+        } catch (error) {
+          return `Return only valid JSON matching { "description": string, "keywords": string[] }. Error: ${error instanceof Error ? error.message : String(error)}`;
+        }
+      },
       onToolCall: (toolName) => logger.log('info', `[Description] Tool call: ${toolName}`),
     });
 
-    console.log(descriptionResult.text);
-    const resultObject: IDescriptionInterface = JSON.parse(
-      descriptionResult.text.replace('```json', '').replace('```', ''),
-    );
+    const resultObject = parseDescriptionJson(descriptionResult.text);
 
     // Use ProjectContext to get file handles for writing
     const projectContext = new ProjectContext(this.projectDir);
@@ -81,12 +113,23 @@ Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object.
 
     // Update smartconfig.json
     const smartconfigJson = files.smartfilesSmartconfigJSON;
-    const smartconfigJsonContent = JSON.parse(smartconfigJson.contents.toString());
-
-    smartconfigJsonContent['gitzone'].module.description = resultObject.description;
-    smartconfigJsonContent['gitzone'].module.keywords = resultObject.keywords;
-
-    smartconfigJson.contents = Buffer.from(JSON.stringify(smartconfigJsonContent, null, 2));
+    const smartconfigJsonContent = smartconfigJson.contents.length > 0
+      ? JSON.parse(smartconfigJson.contents.toString())
+      : {};
+    if (!smartconfigJsonContent['@git.zone/cli'] || typeof smartconfigJsonContent['@git.zone/cli'] !== 'object') {
+      smartconfigJsonContent['@git.zone/cli'] = {};
+    }
+    const modernModuleConfig = ensureModuleConfig(smartconfigJsonContent['@git.zone/cli']);
+    modernModuleConfig.description = resultObject.description;
+    modernModuleConfig.keywords = resultObject.keywords;
+
+    if (smartconfigJsonContent.gitzone && typeof smartconfigJsonContent.gitzone === 'object') {
+      const legacyModuleConfig = ensureModuleConfig(smartconfigJsonContent.gitzone);
+      legacyModuleConfig.description = resultObject.description;
+      legacyModuleConfig.keywords = resultObject.keywords;
+    }
+
+    smartconfigJson.contents = Buffer.from(`${JSON.stringify(smartconfigJsonContent, null, 2)}\n`);
     await smartconfigJson.write();
 
     // Update package.json
@@ -94,7 +137,7 @@ Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object.
     const packageJsonContent = JSON.parse(packageJson.contents.toString());
     packageJsonContent.description = resultObject.description;
     packageJsonContent.keywords = resultObject.keywords;
-    packageJson.contents = Buffer.from(JSON.stringify(packageJsonContent, null, 2));
+    packageJson.contents = Buffer.from(`${JSON.stringify(packageJsonContent, null, 2)}\n`);
     await packageJson.write();
 
     console.log(`\n======================\n`);

package/ts/aidocs_classes/projectcontext.ts

@@ -1,7 +1,11 @@
 import * as plugins from '../plugins.js';
 
 export class ProjectContext {
-  public static async fromDir(dirArg: string) {}
+  public static async fromDir(dirArg: string) {
+    const projectContext = new ProjectContext(dirArg);
+    await projectContext.update();
+    return projectContext;
+  }
 
   // INSTANCE
   public projectDir: string;
@@ -12,30 +16,26 @@ export class ProjectContext {
     this.projectDir = projectDirArg;
   }
 
-  public async gatherFiles() {
-    const smartfilePackageJSON = await plugins.smartfileFactory.fromFilePath(
-      plugins.path.join(this.projectDir, 'package.json'),
-      this.projectDir,
-    );
-    const smartfilesReadme = await plugins.smartfileFactory.fromFilePath(
-      plugins.path.join(this.projectDir, 'readme.md'),
-      this.projectDir,
-    );
+  private async getSmartFile(fileName: string): Promise<plugins.smartfile.SmartFile> {
+    const filePath = plugins.path.join(this.projectDir, fileName);
+    if (await plugins.fsInstance.file(filePath).exists()) {
+      return await plugins.smartfileFactory.fromFilePath(filePath, this.projectDir);
+    }
+    return plugins.smartfileFactory.fromString(filePath, '', 'utf8');
+  }
 
-    const smartfilesReadmeHints = await plugins.smartfileFactory.fromFilePath(
-      plugins.path.join(this.projectDir, 'readme.hints.md'),
-      this.projectDir,
-    );
-    const smartfilesSmartconfigJSON = await plugins.smartfileFactory.fromFilePath(
-      plugins.path.join(this.projectDir, '.smartconfig.json'),
-      this.projectDir,
-    );
-    const smartfilesMod = await plugins.smartfileFactory.virtualDirectoryFromPath(
-      this.projectDir,
-    ).then(vd => vd.filter(f => f.relative.startsWith('ts') && f.relative.endsWith('.ts')).listFiles());
-    const smartfilesTest = await plugins.smartfileFactory.virtualDirectoryFromPath(
-      this.projectDir,
-    ).then(vd => vd.filter(f => f.relative.startsWith('test/') && f.relative.endsWith('.ts')).listFiles());
+  public async gatherFiles() {
+    const smartfilePackageJSON = await this.getSmartFile('package.json');
+    const smartfilesReadme = await this.getSmartFile('readme.md');
+    const smartfilesReadmeHints = await this.getSmartFile('readme.hints.md');
+    const smartfilesSmartconfigJSON = await this.getSmartFile('.smartconfig.json');
+    const virtualDirectory = await plugins.smartfileFactory.virtualDirectoryFromPath(this.projectDir);
+    const smartfilesMod = await virtualDirectory
+      .filter(f => (f.relative.startsWith('ts/') || f.relative.startsWith('ts_web/')) && f.relative.endsWith('.ts'))
+      .listFiles();
+    const smartfilesTest = await virtualDirectory
+      .filter(f => f.relative.startsWith('test/') && f.relative.endsWith('.ts'))
+      .listFiles();
     return {
       smartfilePackageJSON,
       smartfilesReadme,

package/ts/aidocs_classes/readme.ts

@@ -1,8 +1,12 @@
 import type { AiDoc } from '../classes.aidoc.js';
 import * as plugins from '../plugins.js';
-import * as paths from '../paths.js';
 import { ProjectContext } from './projectcontext.js';
 import { logger } from '../logging.js';
+import { createReadOnlyFileSystemTools } from '../helpers.agenttools.js';
+
+const getLegalInfo = (smartconfigJson: any): string | undefined => {
+  return smartconfigJson?.['@git.zone/tsdoc']?.legal ?? smartconfigJson?.tsdoc?.legal;
+};
 
 export class Readme {
   // INSTANCE
@@ -19,17 +23,16 @@ export class Readme {
 
     // First check legal info before introducing any cost
     const projectContext = new ProjectContext(this.projectDir);
-    const smartconfigJson = JSON.parse(
-      (await projectContext.gatherFiles()).smartfilesSmartconfigJSON.contents.toString()
-    );
-    const legalInfo = smartconfigJson?.['tsdoc']?.legal;
+    const smartconfigFile = (await projectContext.gatherFiles()).smartfilesSmartconfigJSON;
+    const smartconfigJson = smartconfigFile.contents.length > 0
+      ? JSON.parse(smartconfigFile.contents.toString())
+      : {};
+    const legalInfo = getLegalInfo(smartconfigJson);
     if (!legalInfo) {
-      const error = new Error(`No legal information found in .smartconfig.json`);
-      console.log(error);
+      throw new Error('No legal information found in .smartconfig.json under @git.zone/tsdoc.legal or tsdoc.legal.');
     }
 
-    // Use runAgent with filesystem tool for agent-driven exploration
-    const fsTools = plugins.smartagentTools.filesystemTool({ rootDir: this.projectDir });
+    const fsTools = await createReadOnlyFileSystemTools(this.projectDir);
 
     const readmeSystemPrompt = `
 You create markdown READMEs for npm projects. You only output the markdown readme.
@@ -42,7 +45,7 @@ IMPORTANT RULES:
 - README must follow proper markdown format
 - Must contain Install and Usage sections
 - Code examples must use correct TypeScript/ESM syntax
-- Documentation must be comprehensive and helpful
+- Documentation must be comprehensive and helpful without unnecessary filler
 - Do NOT include licensing information (added separately)
 - Do NOT use CommonJS syntax - only ESM
 - Do NOT include "in conclusion" or similar filler
@@ -56,7 +59,7 @@ Use the filesystem tools to explore the project and understand what it does:
 2. Read package.json to understand the package name, description, and dependencies
 3. Read the existing readme.md if it exists (use it as a base, improve and expand)
 4. Read readme.hints.md if it exists (contains hints for documentation)
-5. Read key source files in ts/ directory to understand the API and implementation
+5. Read key source files in ts/ and ts_web/ directories to understand the API and implementation
 6. Focus on exported classes, interfaces, and functions
 
 Then generate a comprehensive README following this template:
@@ -74,7 +77,7 @@ Then generate a comprehensive README following this template:
   Make sure to show a complete set of features of the module.
   Don't omit use cases.
   ALWAYS USE ESM SYNTAX AND TYPESCRIPT.
-  Write at least 4000 words. More if necessary.
+  Size the documentation to the actual project. Do not pad with filler.
   If there is already a readme, take the Usage section as base. Remove outdated content, expand and improve.
   Check for completeness.
   Don't include any licensing information. This will be added later.
@@ -84,12 +87,14 @@ Then generate a comprehensive README following this template:
 
     logger.log('info', 'Starting README generation with agent...');
 
-    const readmeResult = await plugins.smartagent.runAgent({
-      model: this.aiDocsRef.model,
+    const readmeResult = await this.aiDocsRef.runAgent({
+      taskName: 'readme',
+      projectDir: this.projectDir,
       prompt: readmeTaskPrompt,
       system: readmeSystemPrompt,
       tools: fsTools,
       maxSteps: 25,
+      useCompaction: true,
       onToolCall: (toolName) => logger.log('info', `[README] Tool call: ${toolName}`),
     });
 
@@ -110,19 +115,19 @@ Then generate a comprehensive README following this template:
 
     // lets care about monorepo aspects
     const tsPublishInstance = new plugins.tspublish.TsPublish();
-    const subModules = await tsPublishInstance.getModuleSubDirs(paths.cwd);
+    const subModules = await tsPublishInstance.getModuleSubDirs(this.projectDir);
     logger.log('info', `Found ${Object.keys(subModules).length} sub modules`);
 
     for (const subModule of Object.keys(subModules)) {
       logger.log('info', `Building readme for ${subModule}`);
 
-      const subModulePath = plugins.path.join(paths.cwd, subModule);
+      const subModulePath = plugins.path.join(this.projectDir, subModule);
       const tspublishData = await plugins.fsInstance
         .file(plugins.path.join(subModulePath, 'tspublish.json'))
         .encoding('utf8')
         .read();
 
-      const subModuleFsTools = plugins.smartagentTools.filesystemTool({ rootDir: subModulePath });
+      const subModuleFsTools = await createReadOnlyFileSystemTools(subModulePath);
 
       const subModuleSystemPrompt = `
 You create markdown READMEs for npm projects. You only output the markdown readme.
@@ -130,7 +135,7 @@ You create markdown READMEs for npm projects. You only output the markdown readm
 IMPORTANT RULES:
 - Only READ files within the submodule directory
 - Do NOT write, delete, or modify any files
-- README must be comprehensive, well-formatted markdown with ESM TypeScript examples
+- README must be comprehensive, well-formatted markdown with ESM TypeScript examples and no filler
 - Do NOT include licensing information (added separately)
 `;
 
@@ -159,7 +164,7 @@ Generate a README following the template:
 [
   Code examples with complete features.
   ESM TypeScript syntax only.
-  Write at least 4000 words.
+  Size the documentation to the submodule. Do not pad with filler.
   No licensing information.
   No "in conclusion".
 ]
@@ -167,12 +172,14 @@ Generate a README following the template:
 Don't use \`\`\` at the beginning or end. Only for code blocks.
 `;
 
-      const subModuleResult = await plugins.smartagent.runAgent({
-        model: this.aiDocsRef.model,
+      const subModuleResult = await this.aiDocsRef.runAgent({
+        taskName: `readme:${subModule}`,
+        projectDir: subModulePath,
         prompt: subModulePrompt,
         system: subModuleSystemPrompt,
         tools: subModuleFsTools,
         maxSteps: 20,
+        useCompaction: true,
         onToolCall: (toolName) => logger.log('info', `[README:${subModule}] Tool call: ${toolName}`),
       });