@aigne/doc-smith 0.8.15-beta.7 → 0.8.15-beta.8

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.8.15-beta.8](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta.7...v0.8.15-beta.8) (2025-11-01)
+
+
+### Features
+
+* smarter structure generation with team-based architecture ([#225](https://github.com/AIGNE-io/aigne-doc-smith/issues/225)) ([eb3404a](https://github.com/AIGNE-io/aigne-doc-smith/commit/eb3404a8889364912a077e84688cfcd48d69ef47))
+
 ## [0.8.15-beta.7](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta.6...v0.8.15-beta.7) (2025-10-31)
 
 

package/agents/generate/check-need-generate-structure.mjs

@@ -3,7 +3,7 @@ import { getActiveRulesForScope } from "../../utils/preferences-utils.mjs";
 import { getProjectInfo, loadConfigFromFile, saveValueToConfig } from "../../utils/utils.mjs";
 
 export default async function checkNeedGenerateStructure(
-  { originalDocumentStructure, forceRegenerate, isLargeContext, ...rest },
+  { originalDocumentStructure, forceRegenerate, ...rest },
   options,
 ) {
   // Check if originalDocumentStructure is empty and prompt user
@@ -53,11 +53,7 @@ export default async function checkNeedGenerateStructure(
     };
   }
 
-  // Performance optimization: Using both structured output and tools with the Gemini model can cause redundant calls.
-  // Only use tools when the context is very large.
-  const generateStructureAgent = isLargeContext
-    ? options.context.agents["generateStructure"]
-    : options.context.agents["generateStructureWithoutTools"];
+  const generateStructureAgent = options.context.agents["generateStructure"];
 
   const structureRules = getActiveRulesForScope("structure", []);
   const globalRules = getActiveRulesForScope("global", []);
@@ -72,7 +68,6 @@ export default async function checkNeedGenerateStructure(
     originalDocumentStructure,
     userPreferences,
     feedback: finalFeedback || "",
-    isLargeContext,
   });
 
   let message = "";

package/agents/generate/generate-structure.yaml

@@ -1,68 +1,162 @@
+type: team
 name: generateStructure
 description: Generate the structure and organization of your documentation
-instructions:
-  - role: system
-    url: ../../prompts/structure/generate/system-prompt.md
-  - role: user
-    url: ../../prompts/structure/generate/user-prompt.md
 skills:
-  - ./document-structure-tools/generate-sub-structure.mjs
-task_render_mode: collapse
-task_title: Generate the structure of the documentation
-tool_calls_concurrency: 5
-input_schema:
-  type: object
-  properties:
-    rules:
-      type: string
-      description: Your specific requirements for documentation structure
-    locale:
-      type: string
-      description: Primary language for documentation (e.g., zh, en, ja)
-    datasources:
-      type: string
-      description: Project content and context to help generate documentation structure
-    targetAudience:
-      type: string
-      description: Target audience for the documentation
-    nodeName:
-      type: string
-      description: Specific section or page name to focus on
-    glossary:
-      type: string
-      description: Glossary for consistent terminology
-    feedback:
-      type: string
-      description: Tell us how to improve the documentation structure
-    userPreferences:
-      type: string
-      description: Your saved preferences for structure and documentation style
-    docsType:
-      type: string
-      description: "Documentation type (options: general, getting-started, reference, faq)"
-      default: general
-  required:
-    - rules
-    - datasources
-output_schema:
-  type: object
-  properties:
-    projectName:
-      type: string
-      description: Project name identified from your content sources
-    projectDesc:
-      type: string
-      description: Brief project description generated from content analysis (under 50 words)
-    documentStructure: ../schema/document-structure.yaml
-    documentStructureTree:
-      type: string
-      description: |
-        Visual tree structure showing documentation hierarchy with indented levels for easy review:
-        ```
-        - Home
-          - Getting Started
-            - Installation
-              - Requirements
-        ```
-  required:
-    - documentStructure
+  - type: team
+    name: generateStructureWorker
+    iterate_on: datasources
+    skills:
+      - type: ai
+        model:
+          reasoning_effort: 500
+        instructions:
+          - role: system
+            url: ../../prompts/structure/generate/system-prompt.md
+          - role: user
+            url: ../../prompts/structure/generate/user-prompt.md
+        task_render_mode: collapse
+        task_title: Generate the structure of the documentation
+        tool_calls_concurrency: 5
+        input_schema:
+          type: object
+          properties:
+            rules:
+              type: string
+              description: Your specific requirements for documentation structure
+            locale:
+              type: string
+              description: Primary language for documentation (e.g., zh, en, ja)
+            datasources:
+              type: string
+              description: Project content and context to help generate documentation structure
+            targetAudience:
+              type: string
+              description: Target audience for the documentation
+            nodeName:
+              type: string
+              description: Specific section or page name to focus on
+            glossary:
+              type: string
+              description: Glossary for consistent terminology
+            feedback:
+              type: string
+              description: Tell us how to improve the documentation structure
+            userPreferences:
+              type: string
+              description: Your saved preferences for structure and documentation style
+            docsType:
+              type: string
+              description: "Documentation type (options: general, getting-started, reference, faq)"
+              default: general
+          required:
+            - rules
+            - datasources
+        output_schema:
+          type: object
+          properties:
+            projectName:
+              type: string
+              description: Project name identified from your content sources
+            projectDesc:
+              type: string
+              description: Brief project description generated from content analysis (under 50 words)
+            add:
+              type: array
+              description: List of document structure items to add, null or empty array means no addition
+              items:
+                type: object
+                properties:
+                  index:
+                    type: integer
+                    description: Position to insert the new item, null means append to the end
+                  item: ../schema/document-structure-item.yaml
+                required:
+                  - item
+            update:
+              type: array
+              description: List of document structure items to update, replace the item with the same path, null or empty array means no update
+              items:
+                type: object
+                properties:
+                  path:
+                    type: string
+                    description: Path of the document structure item to update or replace
+                  item: ../schema/document-structure-item.yaml
+                required:
+                  - path
+                  - item
+
+      - ./utils/merge-document-structures.mjs
+
+  - type: function
+    name: aggregateDocumentStructure
+    process: |
+      return {
+        documentStructure: options.context.userContext.originalDocumentStructure.map(i => ({
+          ...i,
+          id: i.title.toLowerCase().replace(/\s+/g, '-'),
+        })),
+        projectName: options.context.userContext.projectName,
+        projectDesc: options.context.userContext.projectDesc,
+      }
+
+  - type: ai
+    name: refineStructure
+    model:
+      reasoning_effort: 500
+    instructions:
+      - role: system
+        url: ../../prompts/structure/review/structure-review-system.md
+    output_schema:
+      type: object
+      properties:
+        refinedStructure:
+          type: array
+          description: Optimized document structure array
+          items:
+            type: object
+            description: Document structure item representing a node in the document hierarchy
+            properties:
+              id:
+                type: string
+                description: Unique identifier for the document structure item
+              newIndex:
+                type: integer
+                description: Used for ordering purposes, indicates the new position index of the document structure item
+              newPath:
+                type: string
+                description: The new path of the document structure item if it has been changed, otherwise can be omitted
+              newParentPath:
+                type: string
+                description: The new parentPath of the document structure item if it has been changed, otherwise can be omitted
+            required:
+              - id
+      required:
+        - refinedStructure
+
+  - type: function
+    name: finalizeDocumentStructure
+    process: |
+      return {
+        projectName: input.projectName,
+        projectDesc: input.projectDesc,
+        documentStructure: input.documentStructure
+          .map((item) => {
+            const refined = input.refinedStructure?.find(i => i.id === item.id)
+
+            return {
+              ...item,
+              index: refined?.newIndex || item.index,
+              path: refined?.newPath || item.path,
+              parentId: refined?.newParentPath || item.parentPath,
+            }
+          })
+          .sort((a, b) => a.index - b.index)
+          .map(i => {
+            const newItem = { ...i }
+            delete newItem.index
+            delete newItem.id
+            delete newItem.parentPath
+            return newItem
+          }),
+      }

package/agents/generate/user-review-document-structure.mjs

@@ -140,6 +140,7 @@ export default async function userReviewDocumentStructure({ documentStructure, .
       // Call refineDocumentStructure agent with feedback
       await options.context.invoke(refineAgent, {
         ...rest,
+        datasources: rest.datasources[0].datasources,
         feedback: feedback.trim(),
         documentStructure: currentStructure,
         userPreferences,

package/agents/generate/utils/merge-document-structures.mjs

@@ -0,0 +1,54 @@
+export default async function mergeDocumentStructures(input, options) {
+  if (input.projectName) {
+    options.context.userContext.projectName = input.projectName;
+  }
+  if (input.projectDesc) {
+    options.context.userContext.projectDesc = input.projectDesc;
+  }
+
+  input.projectName = options.context.userContext.projectName;
+  input.projectDesc = options.context.userContext.projectDesc;
+
+  options.context.userContext.originalDocumentStructure ??= [];
+
+  const structure = options.context.userContext.originalDocumentStructure;
+
+  if (input.add) {
+    for (const { index, item } of input.add) {
+      if (index != null && index >= 0 && index < structure.length) {
+        structure.splice(index, 0, item);
+      } else {
+        structure.push(item);
+      }
+    }
+  }
+
+  if (input.update) {
+    for (const upd of input.update) {
+      const idx = structure.findIndex((i) => i.path === upd.path);
+      if (idx !== -1) {
+        structure[idx] = upd.item;
+      }
+    }
+  }
+
+  if (input.delete) {
+    for (const del of input.delete) {
+      const idx = structure.findIndex((i) => i.path === del.path);
+      if (idx !== -1) {
+        structure.splice(idx, 1);
+      }
+    }
+  }
+
+  options.context.userContext.originalDocumentStructure = structure.map((i, index) => {
+    delete i.index;
+
+    return {
+      index,
+      ...i,
+    };
+  });
+
+  return {};
+}

package/agents/schema/document-structure-item.yaml

@@ -0,0 +1,23 @@
+type: object
+description: Document structure item representing a node in the document hierarchy
+properties:
+  title:
+    type: string
+  description:
+    type: string
+  path:
+    type: string
+    description: Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about
+  parentPath:
+    type: string
+    description: Parent node path, if null indicates it is a top-level node
+  sourceIds:
+    type: array
+    description: Associated sourceId from dataSources for subsequent translation and content generation, must come from sourceId in datasources, cannot have fake ids, **cannot be empty**
+    items:
+      type: string
+required:
+  - title
+  - description
+  - path
+  - sourceIds

package/agents/schema/document-structure.yaml

@@ -10,9 +10,7 @@ items:
       type: string
       description: Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about
     parentId:
-      type:
-        - string
-        - "null"
+      type: string
       description: Parent node path, if null indicates it is a top-level node
     sourceIds:
       type: array

package/agents/utils/load-sources.mjs

@@ -4,11 +4,11 @@ import path from "node:path";
 import imageSize from "image-size";
 import {
   buildSourcesContent,
-  calculateFileStats,
   loadFilesFromPaths,
   readFileContents,
   getMimeType,
   isRemoteFile,
+  calculateTokens,
 } from "../../utils/file-utils.mjs";
 import {
   getCurrentGitHead,
@@ -196,13 +196,10 @@ export default async function loadSources(
   }
 
   // Read all source files using the utility function
-  let sourceFiles = await readFileContents(sourceFilesPaths, process.cwd());
-
-  // Count tokens and lines using utility function
-  const { totalTokens, totalLines } = calculateFileStats(sourceFiles);
-
-  // check if totalTokens is too large
-  const isLargeContext = totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD;
+  let sourceFiles = (await readFileContents(sourceFilesPaths, process.cwd())).map((i) => ({
+    ...i,
+    tokens: calculateTokens(`\n${i.sourceId}\n${i.content}`),
+  }));
 
   // filter OpenAPI doc should after check isLargeContext
   sourceFiles = sourceFiles.filter((file) => {
@@ -215,6 +212,16 @@ export default async function loadSources(
     return !isOpenAPI;
   });
 
+  const totalTokens = sourceFiles.reduce((sum, file) => sum + file.tokens, 0);
+  const totalLines = sourceFiles.reduce(
+    (sum, file) => sum + file.content.split("\n").filter(Boolean).length,
+    0,
+  );
+
+  const datasources = splitSourcesToChunks(sourceFiles, INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD).map(
+    (i) => ({ datasources: buildSourcesContent(i) }),
+  );
+
   const remoteFileList = [];
 
   sourceFiles.forEach((file) => {
@@ -226,8 +233,6 @@ export default async function loadSources(
     options.context.userContext.remoteFileList = remoteFileList;
   }
 
-  // Build allSources string using utility function
-  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
   // all files path
   const allFilesPaths = sourceFiles.map((x) => `- ${toRelativePath(x.sourceId)}`).join("\n");
 
@@ -285,7 +290,7 @@ export default async function loadSources(
   }
 
   return {
-    datasources: allSources,
+    datasources,
     content,
     originalDocumentStructure,
     files,
@@ -293,7 +298,6 @@ export default async function loadSources(
     totalTokens,
     totalLines,
     mediaFiles,
-    isLargeContext,
     allFilesPaths,
   };
 }
@@ -342,7 +346,13 @@ loadSources.output_schema = {
   type: "object",
   properties: {
     datasources: {
-      type: "string",
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          datasources: { type: "string" },
+        },
+      },
     },
     files: {
       type: "array",
@@ -373,3 +383,33 @@ loadSources.output_schema = {
 };
 
 loadSources.task_render_mode = "hide";
+
+function splitSourcesToChunks(sources, maxTokens) {
+  const chunks = [];
+
+  let currentChunk = [];
+  let currentTokens = 0;
+
+  for (const source of sources) {
+    const sourceTokens = source.tokens;
+
+    if (currentTokens + sourceTokens > maxTokens) {
+      // Start a new chunk
+      if (currentChunk.length > 0) {
+        chunks.push(currentChunk);
+      }
+      currentChunk = [source];
+      currentTokens = sourceTokens;
+    } else {
+      // Add to current chunk
+      currentChunk.push(source);
+      currentTokens += sourceTokens;
+    }
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+
+  return chunks;
+}

package/aigne.yaml

@@ -1,8 +1,7 @@
 #!/usr/bin/env aigne
 
-chat_model:
-  provider: google
-  name: gemini-2.5-pro
+model:
+  model: google/gemini-2.5-pro
   # name: gemini-2.5-flash
   temperature: 0.8
 agents:
@@ -11,7 +10,6 @@ agents:
 
   # Documentation Structure Generation
   - ./agents/generate/generate-structure.yaml
-  - ./agents/generate/generate-structure-without-tools.yaml
   - ./agents/generate/update-document-structure.yaml
   - ./agents/generate/check-need-generate-structure.mjs
   - ./agents/generate/refine-document-structure.yaml

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/doc-smith",
-  "version": "0.8.15-beta.7",
+  "version": "0.8.15-beta.8",
   "description": "AI-driven documentation generation tool built on the AIGNE Framework",
   "publishConfig": {
     "access": "public"
@@ -56,6 +56,7 @@
     "remark-lint": "^10.0.1",
     "remark-parse": "^11.0.0",
     "terminal-link": "^4.0.0",
+    "typescript": "^5.9.3",
     "ufo": "^1.6.1",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",

package/prompts/structure/generate/system-prompt.md

@@ -14,34 +14,4 @@ You are an AI document strategist with the personality of an **INTJ (The Archite
 {% include "../../common/document-structure/conflict-resolution-guidance.md" %}
 
 
-<sub_structure>
-{% if isLargeContext %}
-Analyze the provided file list and DataSources to complete the document structure planning:
-  - If the DataSources contain sufficient context and already include content from all files in the file list, you can directly generate a detailed document structure.
-  - First plan the document structure based on DataSources and <file_list>, ensuring all user-provided information will be presented in the document
-  - Ensure initial planning has sufficient content separation to avoid oversized data sources when generating sub-documents
-  - For sections with extensive content, use the `generateSubStructure` tool to generate detailed sub-structures
-  - Trigger all Tool calls at once whenever possible
-  - When triggering Tool calls, only output Tool call related information
-  - Carefully check the data returned by the `generateSubStructure` tool, integrate all data, merge the complete document structure, and finally verify that it meets the requirements in <output_constraints>
-
-Using `generateSubStructure`:
-- When the provided file list is large and DataSources don't contain all file contents, resulting in an oversized context, split the generation into sub-document structures to make the context more focused and complete
-- Generate sub-documents to more effectively and fully utilize the data source files provided in <file_list>
-- Requires `parentDocument` and `subSourcePaths` as context parameters
-- `subSourcePaths` supports individual files and Glob Patterns, generation process:
-  - Analyze relevant files from the file list, include as many related files as possible to ensure complete context
-  - Selected files must come from <file_list>, ensure file paths are correct
-  - Consolidation Rules:
-    1. If all files from a single directory (e.g., src/) have been selected, consolidate them into a pattern like src/\*.
-    2. If multiple files with a common naming convention are selected (e.g., README.md, README-dockerfile.md, README-turbo.md), consolidate them into a pattern like README\*.md.
-    3. Ensure only files correctly matched by the pattern are removed, while unmatched files must be preserved
-- Merge the returned subStructure into the overall document structure plan, **ensuring all subStructures returned by the tool are included**.
-
-{% else %}
-The current context is sufficient, proceed directly with document structure planning based on DataSources.
-{% endif %}
-</sub_structure>
-
-
 {% include "../../common/document-structure/output-constraints.md" %}

package/prompts/structure/generate/user-prompt.md

@@ -1,15 +1,10 @@
+<datasources>
+Following are the partial or complete data sources provided by the user to help you design the document structure. Use these data sources to inform your structural planning.
 
-{% include "../../common/document-structure/user-locale-rules.md" %}
-
-{% include "../../common/document-structure/user-preferences.md" %}
-
+{{ datasources }}
 
-<file_list>
-{{allFilesPaths}}
-</file_list>
 
-<datasources>
-{{ datasources }}
+NOTICE: There are additional data source contents not displayed. When operating on the document structure, be sure to consider these undisplayed contents and do not easily delete any nodes unless users explicitly request deletion.
 </datasources>
 
 {% if userContext.openAPISpec %}
@@ -17,7 +12,7 @@
 
 **Goal:** Use the provided OpenAPI (Swagger) specification to design how the OpenAPI content and the overall document should be structured together.
 
-**OpenAPI File Content:** 
+**OpenAPI File Content:**
 <openapi_doc>
 
 {{ userContext.openAPISpec }}
@@ -47,20 +42,31 @@
 {% endif %}
 
 
-{% if originalDocumentStructure %}
 <last_document_structure>
-{{originalDocumentStructure}}
+projectName: |
+  {{projectName}}
+projectDesc: |
+  {{projectDesc}}
+
+{% if originalDocumentStructure %}
+{{ originalDocumentStructure | yaml.stringify }}
+{% else %}
+No previous document structure provided. generate a new structure based on the data sources!
+{% endif %}
+
 </last_document_structure>
 
 
+{% include "../../common/document-structure/user-locale-rules.md" %}
+
+{% include "../../common/document-structure/user-preferences.md" %}
+
 <last_document_structure_rule>
 If a previous structural plan (last_document_structure) is provided, follow these rules:
   1.  **Feedback Implementation**: The new structural plan **must** correctly implement all changes requested in user feedback.
   2.  **Unrelated Node Stability**: Nodes not mentioned in user feedback **must not have their path or sourcesIds attributes modified**. `path` and `sourcesIds` are critical identifiers linking existing content, and their stability is paramount.
     Ideally, other attributes (such as `title`, `description`) should also remain stable, unless these changes are directly caused by a requested modification or result from DataSource updates.
 </last_document_structure_rule>
-{% endif %}
-
 
 {% if documentStructure %}
 <review_document_structure>
@@ -92,27 +98,62 @@ Sub-structures must meet the following requirements:
 - Sub-structures are planned based on DataSources and the parent document's description
 - The parent document provides an overview of the planned content, while sub-structures directly plan the specific content to be displayed
 - Further break down and comprehensively display the content planned in the parent document
-- All sub-structures must have their parentId value set to {{parentDocument.path}}
+- All sub-structures must have their parentPath value set to {{parentDocument.path}}
 </parent_document>
 {% endif %}
 
 <instructions>
-Your task is to design a detailed structural plan for the document to be generated. This plan will serve as a "blueprint" for subsequent content generation, guiding the LLM on how to organize and present information, ensuring the document is logically clear, easy to understand, well-structured, and comprehensive.
+Your task is to **analyze, refine, and adjust** the existing document structure (`last_document_structure`) based on the partial code repository content currently provided, generating a structural update plan.
+You are not creating a structure from scratch, but rather **performing intelligent updates based on understanding the existing structure** to make the document structure more accurately reflect the latest code content, architectural changes, and logical relationships.
+
+## When using <datasource> data sources, please note the following:
+
+- Fully respect the project descriptions and usage instructions in README files, as these typically summarize the project's core functionality and objectives.
+- Pay attention to comments and docstrings in source code files, as these reveal the design intent and usage methods of the code.
+- Understand the relationships between various modules and files, which helps build a logically clear and well-structured document hierarchy.
+- Notice key concepts, APIs, and configuration options in the code, as these are typically important components of the document structure.
+- The generated document structure must include all public modules, interfaces, and features to ensure document completeness and usability.
+
+
+## Objective
+
+Your output should be a structured change plan containing the following three sections to indicate how to modify the existing document structure:
+
+- **add**: New structure items (array), can use index to specify insertion position (optional), each item is an object containing:
+  - `index` (optional): Insertion position index, if not specified, append to the end;
+  - `item`: New structure definition
+- **update**: Structure items that need modification (array), each item is an object containing:
+  - `path`: Path pointing to the node being updated;
+  - `update`: New structure definition
+
+## Behavior Rules
+
+1. **Understanding and Inheritance**
+   - Fully understand the hierarchical logic, section divisions, and naming style in <last_document_structure>.
+   - Perform incremental updates based on this foundation, not complete rewrites.
+   - Preserve existing reasonable structures, only modify or extend when there is clear justification.
+
+2. **Contextual Association Analysis**
+   - You will receive part of the code repository content (such as partial source files or directory content), please analyze their **documentation value and structural impact**.
+   - Identify which parts represent new concepts, APIs, modules, configurations, or features; determine if they require adding or modifying corresponding sections in the document structure.
 
-Key capabilities and behavioral principles:
-  - Data Comprehension: Ability to parse and understand structured and unstructured data, identifying key concepts, entities, attributes, relationships, and processes within them.
-  - Structured Thinking: Strong logical analysis capabilities to decompose complex information into clear chapters, sections, and items, establishing reasonable hierarchical relationships.
-  - User-Oriented Approach: Ability to flexibly adjust the focus and level of detail in structural planning based on document objectives and audience characteristics provided by users.
-  - Modular Design: Tendency to divide documents into independent, reusable modules or sections for easy content population and subsequent maintenance.
-  - Flexibility and Adaptability: Ability to handle multiple types of data sources and design the most suitable documentation structure based on data source characteristics (such as code function/class structures, API endpoints/parameters, text paragraphs/themes).
-  - Clarity and Completeness: Ensure the final structural plan is easy to understand and can guide the LLM to generate a comprehensive and well-organized document.
+3. **Structure Adjustment Strategy**
+   - If new content supplements details of existing sections, use `update`.
+   - If new content introduces new topics, modules, or hierarchies, use `add`.
+   - Ensure the position, hierarchy, and naming of new nodes align with the overall document logic.
 
+4. **Consistency and Clarity**
+   - Ensure new or modified structure items are consistent with existing structure style.
+   - Each structure node (whether new or updated) should include:
+     - **Title**
+     - **Brief description in one sentence**, describing main content and purpose
+   - Maintain clear hierarchy, avoid duplication, ensure logical coherence. Excellent documentation should allow users to quickly understand project structure and content distribution, organized by modules, functional features, and other dimensions.
 
-Objectives:
-  - Create a clear and logical structural plan that comprehensively presents information from the user-provided context while providing users with intuitive navigation paths.
-  - Each {{nodeName}} should include: a {{nodeName}} title, a one-sentence introduction describing its main content, with presentation and organization methods tailored to the target audience.
+5. **Requirements**
+  - Follow all rules and guidelines in <document_structure_rules>.
+  - Generate rich document structure where functional modules must have sub-documents, comprehensively covering the codebase's functionality and modules, ensuring users can easily get started, understand, and use various modules and main features of the project through documentation.
 
 {% include "../../common/document-structure/intj-traits.md" %}
 
-Always follow one principle: You must ensure the final structural plan meets user requirements.
+You must make reasonable incremental modifications based solely on the new information provided while respecting the existing structure, ensuring the final structure remains complete, clear, and extensible.
 </instructions>

package/prompts/structure/review/structure-review-system.md

@@ -0,0 +1,73 @@
+<role_and_goal>
+You are an AI document strategist with the personality of an **INTJ (The Architect)**. Your core strengths are strategic thinking, understanding complex systems, and creating logically sound blueprints. You are a perfectionist, rigorously logical, and can anticipate future challenges.
+
+</role_and_goal>
+
+<document_structure>
+projectName: |
+  {{projectName}}
+projectDesc: |
+  {{projectDesc}}
+
+documentStructure:
+{{ documentStructure | yaml.stringify }}
+</document_structure>
+
+<instructions>
+You are a Documentation Structure Refiner — an expert in technical documentation architecture and information design.
+
+Your task:
+Given an existing document structure (a JSON array or tree of sections), refine and optimize its **hierarchy and order** to improve clarity, usability, and conventional organization.
+️ You must not add, delete, rename, or rewrite any nodes. Only adjust the **order** and **nesting levels** of existing nodes.
+
+---
+
+## Optimization Goals
+
+1. **Logical Order**
+   - Introductory materials should always appear at the beginning:
+     - “Overview”, “Introduction”, “Quick Start”, “Getting Started”, “Setup” should be near the top.
+   - Meta and community-related sections (e.g., “Community”, “Contributing”, “License”, “Changelog”) should always be at the end.
+   - Technical reference and configuration sections should appear after conceptual and usage sections.
+
+2. **Hierarchy Correction**
+   - Ensure proper depth:
+     - “Overview” and “Quick Start” should have **1–2 levels max**.
+     - Remove deeply nested technical details from “Overview” or “Quick Start”.
+     - Relocate such details under “Architecture”, “API Reference”, or “Modules”.
+   - Preserve all nodes — only change their parent-child relationships when needed for clarity.
+
+3. **Grouping and Alignment**
+   - Align similar nodes logically (e.g., group “Usage”, “Examples”, “Tutorials” together).
+   - Avoid duplication or overlap by reordering, not by deletion.
+
+4. **Naming and Identity**
+   - You are **not allowed to rename or reword** any section titles or descriptions.
+   - Keep all existing keys, identifiers, and text intact.
+
+5. **Balance**
+   - Maintain a clean, well-organized hierarchy.
+   - Keep top-level nodes concise (≤ 8 preferred).
+   - Avoid over-nesting (≤ 4 levels deep).
+
+---
+
+## Behavior Rules
+
+- Do **not** add new nodes.
+- Do **not** delete existing nodes.
+- Do **not** rename or rewrite content.
+- You **may** move nodes to different parents or reorder siblings to achieve better logical flow.
+- You **must** maintain all data and structural integrity.
+- The final structure must remain fully valid and machine-readable (same schema as input).
+
+---
+
+## Objective
+
+Output a single **optimized JSON structure** (same format as input), where:
+1. The hierarchy and order are improved.
+2. All nodes are preserved exactly as given.
+3. The structure reflects a natural and professional documentation layout
+4. Only return the nodes need to be changed to achieve the above goals.
+</instructions>

package/types/document-structure-schema.mjs

@@ -6,7 +6,7 @@ export const documentItemSchema = z.object({
   title: z.string().min(1, "Title is required"),
   description: z.string().min(1, "Description is required"),
   path: z.string().startsWith("/", 'Path must start with "/"'),
-  parentId: z.string().nullable(),
+  parentId: z.string().nullish(),
   sourceIds: z.array(z.string()).min(1, "At least one source ID is required"),
 });
 
@@ -18,7 +18,7 @@ export const addDocumentInputSchema = z.object({
   title: z.string().min(1, "Title is required"),
   description: z.string().min(1, "Description is required"),
   path: z.string().startsWith("/", 'Path must start with "/"'),
-  parentId: z.string().nullable().optional(),
+  parentId: z.string().nullish(),
   sourceIds: z.array(z.string()).min(1, "At least one source ID is required"),
 });
 
@@ -44,7 +44,7 @@ export const deleteDocumentOutputSchema = z.object({
 // Move document schemas
 export const moveDocumentInputSchema = z.object({
   path: z.string().min(1, "Path is required"),
-  newParentId: z.string().nullable().optional(),
+  newParentId: z.string().nullish(),
 });
 
 export const moveDocumentOutputSchema = z.object({

package/utils/extract-api.mjs

@@ -0,0 +1,32 @@
+import { readFile } from "node:fs/promises";
+import { transpileDeclaration } from "typescript";
+
+export async function extractApi(path) {
+  const content = await readFile(path, "utf8");
+
+  const lang = languages.find((lang) => lang.match(path, content));
+  if (lang) {
+    return lang.extract(path, content);
+  }
+
+  return content;
+}
+
+const languages = [
+  {
+    match: (path) => /\.m?(js|ts)x?$/.test(path),
+    extract: extractJsApi,
+  },
+];
+
+async function extractJsApi(_path, content) {
+  const res = transpileDeclaration(content, {
+    compilerOptions: {
+      declaration: true,
+      emitDeclarationOnly: true,
+      allowJs: true,
+    },
+  });
+
+  return res.outputText.trim();
+}

package/utils/file-utils.mjs

@@ -11,8 +11,8 @@ import { gunzipSync } from "node:zlib";
 
 import { debug } from "./debug.mjs";
 import { isGlobPattern } from "./utils.mjs";
-import { INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD } from "./constants/index.mjs";
 import { uploadFiles } from "./upload-files.mjs";
+import { extractApi } from "./extract-api.mjs";
 
 /**
  * Check if a directory is inside a git repository using git command
@@ -508,7 +508,9 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
 
           return null;
         } else {
-          const content = await readFile(file, "utf8");
+          const content = await extractApi(file);
+          if (!content) return null;
+
           const relativePath = path.relative(baseDir, file);
           return {
             sourceId: relativePath,
@@ -527,6 +529,11 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
   return results.filter((result) => result !== null);
 }
 
+export function calculateTokens(text) {
+  const tokens = encode(text);
+  return tokens.length;
+}
+
 /**
  * Calculate total lines and tokens from file contents
  * @param {Array<{content: string}>} sourceFiles - Array of objects containing content property
@@ -552,97 +559,17 @@ export function calculateFileStats(sourceFiles) {
 }
 
 /**
- * Build sources content string based on context size
- * For large contexts, only include core project files to avoid token limit issues
+ * Build sources content string
  * @param {Array<{sourceId: string, content: string}>} sourceFiles - Array of source file objects
- * @param {boolean} isLargeContext - Whether the context is large
  * @returns {string} Concatenated sources content with sourceId comments
  */
-export function buildSourcesContent(sourceFiles, isLargeContext = false) {
-  // Define core file patterns that represent project structure and key information
-  const coreFilePatterns = [
-    // Configuration files
-    /package\.json$/,
-    /tsconfig\.json$/,
-    /jsconfig\.json$/,
-    /\.env\.example$/,
-    /Cargo\.toml$/,
-    /go\.mod$/,
-    /pom\.xml$/,
-    /build\.gradle$/,
-    /Gemfile$/,
-    /requirements\.txt$/,
-    /Pipfile$/,
-    /composer\.json$/,
-    /pyproject\.toml$/,
-
-    // Documentation
-    /README\.md$/i,
-    /CHANGELOG\.md$/i,
-    /CONTRIBUTING\.md$/i,
-    /\.github\/.*\.md$/i,
-
-    // Entry points and main files
-    /index\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /main\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /app\.(js|ts|jsx|tsx|py)$/,
-    /server\.(js|ts|jsx|tsx|py)$/,
-
-    // API definitions
-    /api\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /routes\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /controllers\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-
-    // Type definitions and schemas
-    /types\.(ts|d\.ts)$/,
-    /schema\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /.*\.d\.ts$/,
-
-    // Core utilities
-    /utils\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /lib\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /helpers\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-  ];
-
-  // Function to check if a file is a core file
-  const isCoreFile = (filePath) => {
-    return coreFilePatterns.some((pattern) => pattern.test(filePath));
-  };
-
+export function buildSourcesContent(sourceFiles) {
   // Build sources string
   let allSources = "";
 
-  if (isLargeContext) {
-    // Only include core files for large contexts
-    const coreFiles = sourceFiles.filter((source) => isCoreFile(source.sourceId));
-
-    // Determine which files to use and set appropriate message
-    const filesToInclude = coreFiles.length > 0 ? coreFiles : sourceFiles;
-    const noteMessage =
-      coreFiles.length > 0
-        ? "// Note: Context is large, showing only core project files.\n"
-        : "// Note: Context is large, showing a sample of files.\n";
-
-    allSources += noteMessage;
-    let accumulatedTokens = 0;
-
-    for (const source of filesToInclude) {
-      const fileContent = `// sourceId: ${source.sourceId}\n${source.content}\n`;
-      const fileTokens = encode(fileContent);
-
-      // Check if adding this file would exceed the token limit
-      if (accumulatedTokens + fileTokens.length > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD) {
-        break;
-      }
-
-      allSources += fileContent;
-      accumulatedTokens += fileTokens.length;
-    }
-  } else {
-    // Include all files for normal contexts
-    for (const source of sourceFiles) {
-      allSources += `// sourceId: ${source.sourceId}\n${source.content}\n`;
-    }
+  // Include all files for normal contexts
+  for (const source of sourceFiles) {
+    allSources += `\n// sourceId: ${source.sourceId}\n${source.content}\n`;
   }
 
   return allSources;

package/agents/generate/document-structure-tools/generate-sub-structure.mjs

@@ -1,131 +0,0 @@
-import {
-  buildSourcesContent,
-  calculateFileStats,
-  loadFilesFromPaths,
-  readFileContents,
-} from "../../../utils/file-utils.mjs";
-import {
-  INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD,
-  DEFAULT_EXCLUDE_PATTERNS,
-  DEFAULT_INCLUDE_PATTERNS,
-} from "../../../utils/constants/index.mjs";
-import { toRelativePath } from "../../../utils/utils.mjs";
-
-export default async function generateSubStructure(
-  {
-    parentDocument,
-    subSourcePaths,
-    includePatterns,
-    excludePatterns,
-    useDefaultPatterns = true,
-    ...rest
-  },
-  options,
-) {
-  const sourcePaths = subSourcePaths?.map((item) => item.path);
-  if (!sourcePaths || sourcePaths.length === 0) {
-    return {
-      subStructure: [],
-    };
-  }
-
-  let files = await loadFilesFromPaths(sourcePaths, {
-    includePatterns,
-    excludePatterns,
-    useDefaultPatterns,
-    defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
-    defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
-  });
-  files = [...new Set(files)];
-
-  // all files path
-  const allFilesPaths = files.map((file) => `- ${toRelativePath(file)}`).join("\n");
-
-  // Read all source files using the utility function
-  const sourceFiles = await readFileContents(files, process.cwd());
-
-  // Count tokens and lines using utility function
-  const { totalTokens } = calculateFileStats(sourceFiles);
-
-  // check if totalTokens is too large
-  let isLargeContext = false;
-  if (totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD) {
-    isLargeContext = true;
-  }
-
-  // Build allSources string using utility function
-  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
-
-  // Performance optimization:
-  // Using both structured output and Tool with Gemini model causes redundant calls
-  // Only use Tool when context is very large
-  const generateStructureAgent = isLargeContext
-    ? options.context.agents["generateStructure"]
-    : options.context.agents["generateStructureWithoutTools"];
-  const result = await options.context.invoke(generateStructureAgent, {
-    ...rest,
-    isSubStructure: true,
-    parentDocument,
-    datasources: allSources,
-    allFilesPaths,
-    isLargeContext,
-    files,
-    totalTokens,
-  });
-
-  return {
-    subStructure: result.documentStructure || [],
-    message: `Generated a sub structure for '${parentDocument.path}' successfully. Please merge all sub-structures to output the complete document structure.`,
-  };
-}
-
-generateSubStructure.description = `
-Generates a sub-structure. 
-Handles large file sets by splitting them into smaller sub-document structures when the context size exceeds limits. This approach ensures more focused and complete documentation generation.
-`;
-
-generateSubStructure.inputSchema = {
-  type: "object",
-  properties: {
-    parentDocument: {
-      type: "object",
-      description: "The parent node to generate a sub structure for",
-      properties: {
-        title: { type: "string", description: "The title of the parent node" },
-        description: { type: "string", description: "The description of the parent node" },
-        path: {
-          type: "string",
-          description:
-            "The path of the parent node, Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about ",
-        },
-        parentId: { type: "string", description: "The parent ID of the parent node" },
-        sourceIds: { type: "array", description: "The source IDs of the parent node" },
-      },
-    },
-    subSourcePaths: {
-      type: "array",
-      description: "The source paths of the sub structure",
-      items: {
-        type: "object",
-        properties: {
-          path: { type: "string", description: "The source path of the sub structure" },
-          reason: { type: "string", description: "The reason for selecting the source path" },
-        },
-        required: ["path", "reason"],
-      },
-    },
-  },
-};
-
-generateSubStructure.outputSchema = {
-  type: "object",
-  properties: {
-    subStructure: {
-      type: "array",
-      description:
-        "The sub structure of the parent node, need merge all sub-structures and output the complete document structure.",
-    },
-    message: { type: "string", description: "The message of the sub structure" },
-  },
-  required: ["subStructure"],
-};

package/agents/generate/generate-structure-without-tools.yaml

@@ -1,65 +0,0 @@
-name: generateStructureWithoutTools
-description: Generate the structure and organization of your documentation
-instructions:
-  - role: system
-    url: ../../prompts/structure/generate/system-prompt.md
-  - role: user
-    url: ../../prompts/structure/generate/user-prompt.md
-task_render_mode: collapse
-task_title: Generate the structure of the documentation
-input_schema:
-  type: object
-  properties:
-    rules:
-      type: string
-      description: Your specific requirements for documentation structure
-    locale:
-      type: string
-      description: Primary language for documentation (e.g., zh, en, ja)
-    datasources:
-      type: string
-      description: Project content and context to help generate documentation structure
-    targetAudience:
-      type: string
-      description: Target audience for the documentation
-    nodeName:
-      type: string
-      description: Specific section or page name to focus on
-    glossary:
-      type: string
-      description: Glossary for consistent terminology
-    feedback:
-      type: string
-      description: Tell us how to improve the documentation structure
-    userPreferences:
-      type: string
-      description: Your saved preferences for structure and documentation style
-    docsType:
-      type: string
-      description: "Documentation type (options: general, getting-started, reference, faq)"
-      default: general
-  required:
-    - rules
-    - datasources
-output_schema:
-  type: object
-  properties:
-    projectName:
-      type: string
-      description: Project name identified from your content sources
-    projectDesc:
-      type: string
-      description: Brief project description generated from content analysis (under 50 words)
-    documentStructure: ../schema/document-structure.yaml
-    documentStructureTree:
-      type: string
-      description: |
-        Visual tree structure showing documentation hierarchy with indented levels for easy review:
-        ```
-        - Home
-          - Getting Started
-            - Installation
-              - Requirements
-        ```
-  required:
-    - documentStructure