@aigne/doc-smith 0.8.12-beta.2 → 0.8.12-beta.4

package/agents/generate/check-need-generate-structure.mjs

@@ -1,31 +1,22 @@
-import { access } from "node:fs/promises";
-import { join } from "node:path";
 import chalk from "chalk";
 import { getActiveRulesForScope } from "../../utils/preferences-utils.mjs";
-import {
-  getCurrentGitHead,
-  getProjectInfo,
-  hasFileChangesBetweenCommits,
-  loadConfigFromFile,
-  saveValueToConfig,
-} from "../../utils/utils.mjs";
+import { getProjectInfo, loadConfigFromFile, saveValueToConfig } from "../../utils/utils.mjs";
 
 export default async function checkNeedGenerateStructure(
-  { originalDocumentStructure, feedback, lastGitHead, docsDir, forceRegenerate, ...rest },
+  { originalDocumentStructure, forceRegenerate, isLargeContext, ...rest },
   options,
 ) {
   // Check if originalDocumentStructure is empty and prompt user
   if (!originalDocumentStructure) {
     const choice = await options.prompts.select({
-      message:
-        "Your project configuration is complete. Would you like to generate the documentation structure now?",
+      message: "Project configured successfully. Generate the documentation structure now?",
       choices: [
         {
-          name: "Generate now - Start generating the documentation structure",
+          name: "Yes, generate now",
           value: "generate",
         },
         {
-          name: "Review configuration first - Edit configuration before generating",
+          name: "No, review configuration first",
           value: "later",
         },
       ],
@@ -50,65 +41,28 @@ export default async function checkNeedGenerateStructure(
   }
 
   // Check if we need to regenerate documentation structure
-  let shouldRegenerate = false;
-  let finalFeedback = feedback;
-
-  // If no feedback and originalDocumentStructure exists, check for git changes
-  if (originalDocumentStructure) {
-    // If no lastGitHead, check if _sidebar.md exists to determine if we should regenerate
-    if (!lastGitHead) {
-      try {
-        // Check if _sidebar.md exists in docsDir
-        const sidebarPath = join(docsDir, "_sidebar.md");
-        await access(sidebarPath);
-        // If _sidebar.md exists, it means last execution was completed, need to regenerate
-        shouldRegenerate = true;
-      } catch {
-        // If _sidebar.md doesn't exist, it means last execution was interrupted, no need to regenerate
-        shouldRegenerate = false;
-      }
-    } else {
-      // Check if there are relevant file changes since last generation
-      const currentGitHead = getCurrentGitHead();
-      if (currentGitHead && currentGitHead !== lastGitHead) {
-        const hasChanges = hasFileChangesBetweenCommits(lastGitHead, currentGitHead);
-        if (hasChanges) {
-          shouldRegenerate = true;
-        }
-      }
-    }
-
-    if (shouldRegenerate) {
-      finalFeedback = `
-      ${finalFeedback || ""}
-
-      Update documentation structure based on the latest DataSources:
-        1. For new content, add new sections as needed or supplement existing section displays
-        2. Be cautious when deleting sections, unless all associated sourceIds have been removed
-        3. Do not modify the path of existing sections
-        4. Update section sourceIds as needed based on the latest Data Sources
-      `;
-    }
-  }
+  let finalFeedback = "";
 
   // user requested regeneration
   if (forceRegenerate) {
-    shouldRegenerate = true;
     finalFeedback = `
-    ${finalFeedback || ""}
-
     User requested forced regeneration of documentation structure. Please regenerate based on the latest Data Sources and user requirements, **allowing any modifications**.
     `;
   }
 
   // If no regeneration needed, return original documentation structure
-  if (originalDocumentStructure && !finalFeedback && !shouldRegenerate) {
+  if (originalDocumentStructure && !forceRegenerate) {
     return {
       documentStructure: originalDocumentStructure,
     };
   }
 
-  const planningAgent = options.context.agents["refineDocumentStructure"];
+  // Performance optimization:
+  // Using both structured output and Tool with Gemini model causes redundant calls
+  // Only use Tool when context is very large
+  const generateStructureAgent = isLargeContext
+    ? options.context.agents["generateStructure"]
+    : options.context.agents["generateStructureWithoutTools"];
 
   // Get user preferences for documentation structure and global scope
   const structureRules = getActiveRulesForScope("structure", []);
@@ -121,11 +75,12 @@ export default async function checkNeedGenerateStructure(
   // Convert rule texts to string format for passing to the agent
   const userPreferences = ruleTexts.length > 0 ? ruleTexts.join("\n\n") : "";
 
-  const result = await options.context.invoke(planningAgent, {
+  const result = await options.context.invoke(generateStructureAgent, {
     ...rest,
     originalDocumentStructure,
     userPreferences,
     feedback: finalFeedback || "",
+    isLargeContext,
   });
 
   let message = "";

package/agents/generate/document-structure-tools/generate-sub-structure.mjs

@@ -0,0 +1,131 @@
+import {
+  buildSourcesContent,
+  calculateFileStats,
+  loadFilesFromPaths,
+  readFileContents,
+} from "../../../utils/file-utils.mjs";
+import {
+  INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD,
+  DEFAULT_EXCLUDE_PATTERNS,
+  DEFAULT_INCLUDE_PATTERNS,
+} from "../../../utils/constants/index.mjs";
+import { toRelativePath } from "../../../utils/utils.mjs";
+
+export default async function generateSubStructure(
+  {
+    parentDocument,
+    subSourcePaths,
+    includePatterns,
+    excludePatterns,
+    useDefaultPatterns = true,
+    ...rest
+  },
+  options,
+) {
+  const sourcePaths = subSourcePaths?.map((item) => item.path);
+  if (!sourcePaths || sourcePaths.length === 0) {
+    return {
+      subStructure: [],
+    };
+  }
+
+  let files = await loadFilesFromPaths(sourcePaths, {
+    includePatterns,
+    excludePatterns,
+    useDefaultPatterns,
+    defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
+    defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
+  });
+  files = [...new Set(files)];
+
+  // all files path
+  const allFilesPaths = files.map((file) => `- ${toRelativePath(file)}`).join("\n");
+
+  // Read all source files using the utility function
+  const sourceFiles = await readFileContents(files, process.cwd());
+
+  // Count tokens and lines using utility function
+  const { totalTokens } = calculateFileStats(sourceFiles);
+
+  // check if totalTokens is too large
+  let isLargeContext = false;
+  if (totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD) {
+    isLargeContext = true;
+  }
+
+  // Build allSources string using utility function
+  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
+
+  // Performance optimization:
+  // Using both structured output and Tool with Gemini model causes redundant calls
+  // Only use Tool when context is very large
+  const generateStructureAgent = isLargeContext
+    ? options.context.agents["generateStructure"]
+    : options.context.agents["generateStructureWithoutTools"];
+  const result = await options.context.invoke(generateStructureAgent, {
+    ...rest,
+    isSubStructure: true,
+    parentDocument,
+    datasources: allSources,
+    allFilesPaths,
+    isLargeContext,
+    files,
+    totalTokens,
+  });
+
+  return {
+    subStructure: result.documentStructure || [],
+    message: `Generated a sub structure for '${parentDocument.path}' successfully. Please merge all sub-structures to output the complete document structure.`,
+  };
+}
+
+generateSubStructure.description = `
+Generates a sub-structure. 
+Handles large file sets by splitting them into smaller sub-document structures when the context size exceeds limits. This approach ensures more focused and complete documentation generation.
+`;
+
+generateSubStructure.inputSchema = {
+  type: "object",
+  properties: {
+    parentDocument: {
+      type: "object",
+      description: "The parent node to generate a sub structure for",
+      properties: {
+        title: { type: "string", description: "The title of the parent node" },
+        description: { type: "string", description: "The description of the parent node" },
+        path: {
+          type: "string",
+          description:
+            "The path of the parent node, Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about ",
+        },
+        parentId: { type: "string", description: "The parent ID of the parent node" },
+        sourceIds: { type: "array", description: "The source IDs of the parent node" },
+      },
+    },
+    subSourcePaths: {
+      type: "array",
+      description: "The source paths of the sub structure",
+      items: {
+        type: "object",
+        properties: {
+          path: { type: "string", description: "The source path of the sub structure" },
+          reason: { type: "string", description: "The reason for selecting the source path" },
+        },
+        required: ["path", "reason"],
+      },
+    },
+  },
+};
+
+generateSubStructure.outputSchema = {
+  type: "object",
+  properties: {
+    subStructure: {
+      type: "array",
+      description:
+        "The sub structure of the parent node, need merge all sub-structures and output the complete document structure.",
+    },
+    message: { type: "string", description: "The message of the sub structure" },
+  },
+  required: ["subStructure"],
+};

package/agents/generate/generate-structure-without-tools.yaml

@@ -0,0 +1,65 @@
+name: generateStructureWithoutTools
+description: Generate the structure and organization of your documentation
+instructions:
+  - role: system
+    url: ../../prompts/structure/generate/system-prompt.md
+  - role: user
+    url: ../../prompts/structure/generate/user-prompt.md
+task_render_mode: collapse
+task_title: Generate the structure of the documentation
+input_schema:
+  type: object
+  properties:
+    rules:
+      type: string
+      description: Your specific requirements for documentation structure
+    locale:
+      type: string
+      description: Primary language for documentation (e.g., zh, en, ja)
+    datasources:
+      type: string
+      description: Project content and context to help generate documentation structure
+    targetAudience:
+      type: string
+      description: Target audience for the documentation
+    nodeName:
+      type: string
+      description: Specific section or page name to focus on
+    glossary:
+      type: string
+      description: Glossary for consistent terminology
+    feedback:
+      type: string
+      description: Tell us how to improve the documentation structure
+    userPreferences:
+      type: string
+      description: Your saved preferences for structure and documentation style
+    docsType:
+      type: string
+      description: "Documentation type (options: general, getting-started, reference, faq)"
+      default: general
+  required:
+    - rules
+    - datasources
+output_schema:
+  type: object
+  properties:
+    projectName:
+      type: string
+      description: Project name identified from your content sources
+    projectDesc:
+      type: string
+      description: Brief project description generated from content analysis (under 50 words)
+    documentStructure: ../schema/document-structure.yaml
+    documentStructureTree:
+      type: string
+      description: |
+        Visual tree structure showing documentation hierarchy with indented levels for easy review:
+        ```
+        - Home
+          - Getting Started
+            - Installation
+              - Requirements
+        ```
+  required:
+    - documentStructure

package/agents/generate/generate-structure.yaml

@@ -5,7 +5,11 @@ instructions:
     url: ../../prompts/structure/generate/system-prompt.md
   - role: user
     url: ../../prompts/structure/generate/user-prompt.md
-
+skills:
+  - ./document-structure-tools/generate-sub-structure.mjs
+task_render_mode: collapse
+task_title: Generate the structure of the documentation
+tool_calls_concurrency: 5
 input_schema:
   type: object
   properties:
@@ -60,3 +64,5 @@ output_schema:
             - Installation
               - Requirements
         ```
+  required:
+    - documentStructure

package/agents/generate/index.yaml

@@ -54,9 +54,6 @@ input_schema:
     glossary:
       type: string
       description: Glossary file for consistent terminology (use @filename.md)
-    feedback:
-      type: string
-      description: Tell us how to improve the documentation structure
     forceRegenerate:
       type: boolean
       description: Rebuild all documentation from scratch

package/agents/generate/update-document-structure.yaml

@@ -37,3 +37,6 @@ skills:
   - ./document-structure-tools/delete-document.mjs
   - ./document-structure-tools/update-document.mjs
   - ./document-structure-tools/move-document.mjs
+  - ../update/fs-tools/glob.mjs
+  - ../update/fs-tools/grep.mjs
+  - ../update/fs-tools/read-file.mjs

package/agents/generate/user-review-document-structure.mjs

@@ -76,15 +76,14 @@ export default async function userReviewDocumentStructure({ documentStructure, .
 
   // Ask user if they want to review the documentation structure
   const needReview = await options.prompts.select({
-    message:
-      "Would you like to optimize the documentation structure?\n  You can edit titles, reorganize sections.",
+    message: "Would you like to optimize the documentation structure?",
     choices: [
       {
-        name: "Looks good - proceed with current structure",
+        name: "No, looks good",
         value: "no",
       },
       {
-        name: "Yes, optimize the structure",
+        name: "Yes, optimize the structure (e.g. rename 'Getting Started' to 'Quick Start', move 'API Reference' before 'Configuration')",
         value: "yes",
       },
     ],
@@ -108,7 +107,10 @@ export default async function userReviewDocumentStructure({ documentStructure, .
     const feedback = await options.prompts.input({
       message:
         "How would you like to improve the structure?\n" +
-        "  • Rename, reorganize, add, or remove sections\n\n" +
+        "Examples:\n" +
+        "  • Add a new document 'Troubleshooting'\n" +
+        "  • Remove the 'Legacy Features' document\n" +
+        "  • Move 'Installation' to the top of the structure\n\n" +
         "  Press Enter to finish reviewing:",
     });
 

package/agents/init/index.mjs

@@ -143,15 +143,7 @@ export default async function init(
   // Save target audience choices as keys
   input.targetAudienceTypes = audienceChoices;
 
-  // 3. Custom rules - any specific requirements for the documentation?
-  const rulesInput = await options.prompts.input({
-    message:
-      "📋 [3/9]: Any custom rules or requirements for your documentation? (Optional, press Enter to skip)",
-    default: "",
-  });
-  input.rules = rulesInput.trim();
-
-  // 4. Reader knowledge level - what do readers typically know when they arrive?
+  // 3. Reader knowledge level - what do readers typically know when they arrive?
   // Determine default based on selected purposes using mapping
   const mappedPurpose = prioritizedPurposes.find(
     (purpose) => PURPOSE_TO_KNOWLEDGE_MAPPING[purpose],
@@ -166,7 +158,7 @@ export default async function init(
   );
 
   const knowledgeChoice = await options.prompts.select({
-    message: "🧠 [4/9]: How much do readers already know about your project?",
+    message: "🧠 [3/9]: How much do readers already know about your project?",
     choices: Object.entries(filteredKnowledgeOptions).map(([key, level]) => ({
       name: `${level.name}`,
       description: level.description,
@@ -211,7 +203,7 @@ export default async function init(
   );
 
   const depthChoice = await options.prompts.select({
-    message: "📊 [5/9]: How detailed should your documentation be?",
+    message: "📊 [4/9]: How detailed should your documentation be?",
     choices: Object.entries(filteredDepthOptions).map(([key, depth]) => ({
       name: `${depth.name}`,
       description: depth.description,
@@ -229,7 +221,7 @@ export default async function init(
 
   // Let user select primary language from supported list
   const primaryLanguageChoice = await options.prompts.select({
-    message: "🌐 [6/9]: What's your main documentation language?",
+    message: "🌐 [5/9]: What's your main documentation language?",
     choices: SUPPORTED_LANGUAGES.map((lang) => ({
       name: `${lang.label} - ${lang.sample}`,
       value: lang.code,
@@ -246,7 +238,7 @@ export default async function init(
   );
 
   const translateLanguageChoices = await options.prompts.checkbox({
-    message: "🔄 [7/9]: Which languages should we translate to?",
+    message: "🔄 [6/9]: Which languages should we translate to?",
     choices: availableTranslationLanguages.map((lang) => ({
       name: `${lang.label} - ${lang.sample}`,
       value: lang.code,
@@ -257,13 +249,13 @@ export default async function init(
 
   // 7. Documentation directory
   const docsDirInput = await options.prompts.input({
-    message: `📁 [8/9]: Where should we save your documentation?`,
+    message: `📁 [7/9]: Where should we save your documentation?`,
     default: `${outputPath}/docs`,
   });
   input.docsDir = docsDirInput.trim() || `${outputPath}/docs`;
 
   // 8. Content sources
-  console.log("\n🔍 [9/9]: Content Sources");
+  console.log("\n🔍 [8/9]: Content Sources");
   console.log(
     "What folders/files should we analyze for documentation? (e.g., ./src, ./docs, ./README.md)",
   );
@@ -347,6 +339,14 @@ export default async function init(
   // If no paths entered, use default
   input.sourcesPath = sourcePaths.length > 0 ? sourcePaths : ["./"];
 
+  // 9. Custom rules - any specific requirements for the documentation?
+  const rulesInput = await options.prompts.input({
+    message:
+      "📋 [9/9]: Any custom rules or requirements for your documentation? (Optional, press Enter to skip)",
+    default: "",
+  });
+  input.rules = rulesInput.trim();
+
   // Save project info to config
   const projectInfo = await getProjectInfo();
   input.projectName = projectInfo.name;