@aigne/doc-smith 0.8.15-beta.1 → 0.8.15-beta.10

package/agents/generate/generate-structure.yaml

@@ -1,68 +1,120 @@
+type: team
 name: generateStructure
 description: Generate the structure and organization of your documentation
-instructions:
-  - role: system
-    url: ../../prompts/structure/generate/system-prompt.md
-  - role: user
-    url: ../../prompts/structure/generate/user-prompt.md
 skills:
-  - ./document-structure-tools/generate-sub-structure.mjs
-task_render_mode: collapse
-task_title: Generate the structure of the documentation
-tool_calls_concurrency: 5
-input_schema:
-  type: object
-  properties:
-    rules:
-      type: string
-      description: Your specific requirements for documentation structure
-    locale:
-      type: string
-      description: Primary language for documentation (e.g., zh, en, ja)
-    datasources:
-      type: string
-      description: Project content and context to help generate documentation structure
-    targetAudience:
-      type: string
-      description: Target audience for the documentation
-    nodeName:
-      type: string
-      description: Specific section or page name to focus on
-    glossary:
-      type: string
-      description: Glossary for consistent terminology
-    feedback:
-      type: string
-      description: Tell us how to improve the documentation structure
-    userPreferences:
-      type: string
-      description: Your saved preferences for structure and documentation style
-    docsType:
-      type: string
-      description: "Documentation type (options: general, getting-started, reference, faq)"
-      default: general
-  required:
-    - rules
-    - datasources
-output_schema:
-  type: object
-  properties:
-    projectName:
-      type: string
-      description: Project name identified from your content sources
-    projectDesc:
-      type: string
-      description: Brief project description generated from content analysis (under 50 words)
-    documentStructure: ../schema/document-structure.yaml
-    documentStructureTree:
-      type: string
-      description: |
-        Visual tree structure showing documentation hierarchy with indented levels for easy review:
-        ```
-        - Home
-          - Getting Started
-            - Installation
-              - Requirements
-        ```
-  required:
-    - documentStructure
+  - type: team
+    name: generateStructureWorker
+    iterate_on: dataSources
+    skills:
+      - type: ai
+        model:
+          reasoning_effort: 500
+        instructions:
+          - role: system
+            url: ../../prompts/structure/generate/system-prompt.md
+          - role: user
+            url: ../../prompts/structure/generate/user-prompt.md
+        task_render_mode: collapse
+        task_title: Generate the structure of the documentation
+        tool_calls_concurrency: 5
+        input_schema:
+          type: object
+          properties:
+            rules:
+              type: string
+              description: Your specific requirements for documentation structure
+            locale:
+              type: string
+              description: Primary language for documentation (e.g., zh, en, ja)
+            dataSourceChunk:
+              type: string
+              description: Project content and context to help generate documentation structure
+            targetAudience:
+              type: string
+              description: Target audience for the documentation
+            nodeName:
+              type: string
+              description: Specific section or page name to focus on
+            glossary:
+              type: string
+              description: Glossary for consistent terminology
+            feedback:
+              type: string
+              description: Tell us how to improve the documentation structure
+            userPreferences:
+              type: string
+              description: Your saved preferences for structure and documentation style
+            docsType:
+              type: string
+              description: "Documentation type (options: general, getting-started, reference, faq)"
+              default: general
+          required:
+            - rules
+            - dataSourceChunk
+        output_schema:
+          type: object
+          properties:
+            projectName:
+              type: string
+              description: Project name identified from your content sources
+            projectDesc:
+              type: string
+              description: Brief project description generated from content analysis (under 50 words)
+            structures:
+              type: array
+              description: List of document structure items to add or update
+              items: ../schema/document-structure-item.yaml
+
+      - ./utils/merge-document-structures.mjs
+
+  - type: function
+    name: aggregateDocumentStructure
+    process: |
+      return {
+        documentStructure: options.context.userContext.originalDocumentStructure.map((i, index) => ({
+          ...i,
+          id: i.title.toLowerCase().replace(/\s+/g, '-'),
+          index
+        })),
+        projectName: options.context.userContext.projectName,
+        projectDesc: options.context.userContext.projectDesc,
+      }
+
+  - type: ai
+    name: refineStructure
+    model:
+      reasoning_effort: 500
+    instructions:
+      - role: system
+        url: ../../prompts/structure/review/structure-review-system.md
+    output_schema:
+      type: object
+      properties:
+        structures:
+          type: array
+          description: Document structure items that need to be updated or changed
+          items: ../schema/document-structure-refine-item.yaml
+      required:
+        - structures
+
+  - type: function
+    name: finalizeDocumentStructure
+    process: |
+      return {
+        projectName: input.projectName,
+        projectDesc: input.projectDesc,
+        documentStructure: input.structures.map(refinedItem => {
+          // Find original item to get sourceIds and other fields
+          const originalItem = input.documentStructure.find(orig => orig.id === refinedItem.id)
+
+          const newItem = {
+            title: refinedItem.title,
+            description: refinedItem.description,
+            path: refinedItem.path,
+            parentId: refinedItem.parentPath || null,  // Convert parentPath to parentId
+            sourceIds: originalItem?.sourceIds || [],
+          }
+
+          return newItem
+        })
+      }

package/agents/generate/index.yaml

@@ -23,6 +23,7 @@ skills:
       savePath:
         $get: outputDir
       fileName: structure-plan.json
+  - ../utils/save-sidebar.mjs
   - type: transform
     name: transformData
     task_render_mode: hide
@@ -37,8 +38,7 @@ skills:
                 'translates': [$map(translateLanguages, function($lang) { {"language": $lang} })]
               }
             ])
-          }),
-          "datasources": ""
+          })
         }
       ])
   - ../utils/format-document-structure.mjs
@@ -47,7 +47,7 @@ skills:
   - url: ../utils/check-feedback-refiner.mjs
     default_input:
       stage: document_structure
-  - ../utils/save-docs.mjs
+  - ../utils/post-generate.mjs
 
 input_schema:
   type: object

package/agents/generate/{merge-d2-diagram.yaml → merge-diagram.yaml}

@@ -1,14 +1,15 @@
-name: mergeD2DiagramToDocument
-description: Merge D2 Diagram source code into document
+name: mergeDiagramToDocument
+description: Merge Diagram source code into document
 instructions: |
   You are an AI assistant that helps to merge d2 diagram into document.
 
-  <datasources>
+  <detail_data_source>
   {{ content }}
-  </datasources>
-  <datasources>
+  </detail_data_source>
+
+  <diagram_source_code>
   {{ diagramSourceCode }}
-  </datasources>
+  </diagram_source_code>
 
   Given the source content of a document and the D2 diagram source code, your task is to:
   - **Keep the original content as soon as possible.**

package/agents/generate/update-document-structure.yaml

@@ -16,7 +16,7 @@ input_schema:
     locale:
       type: string
       description: User language, e.g. zh, en
-    datasources:
+    dataSourceChunk:
       type: string
       description: Context for documentation structure
     glossary:

package/agents/generate/user-review-document-structure.mjs

@@ -140,6 +140,7 @@ export default async function userReviewDocumentStructure({ documentStructure, .
       // Call refineDocumentStructure agent with feedback
       await options.context.invoke(refineAgent, {
         ...rest,
+        dataSourceChunk: rest.dataSources[0].dataSourceChunk,
         feedback: feedback.trim(),
         documentStructure: currentStructure,
         userPreferences,

package/agents/generate/utils/merge-document-structures.mjs

@@ -0,0 +1,30 @@
+export default async function mergeDocumentStructures(input, options) {
+  if (input.projectName) {
+    options.context.userContext.projectName = input.projectName;
+  }
+  if (input.projectDesc) {
+    options.context.userContext.projectDesc = input.projectDesc;
+  }
+
+  input.projectName = options.context.userContext.projectName;
+  input.projectDesc = options.context.userContext.projectDesc;
+
+  options.context.userContext.originalDocumentStructure ??= [];
+
+  const originalStructures = options.context.userContext.originalDocumentStructure;
+
+  if (input.structures) {
+    for (const item of input.structures) {
+      const old = originalStructures.find((s) => s.path === item.path);
+      if (old) {
+        Object.assign(old, item);
+      } else {
+        originalStructures.push(item);
+      }
+    }
+  }
+
+  options.context.userContext.originalDocumentStructure = originalStructures;
+
+  return {};
+}

package/agents/init/check.mjs

@@ -10,5 +10,7 @@ export default async function checkNeedGenerate({ docsDir, locale, documentExecu
     );
     process.exit(0);
   }
-  return {};
+  return {
+    message: 'Documents found in the docs directory, skipping "generate" step',
+  };
 }

package/agents/init/index.mjs

@@ -20,6 +20,7 @@ import {
   isGlobPattern,
   validatePath,
 } from "../../utils/utils.mjs";
+import { isRemoteFile } from "../../utils/file-utils.mjs";
 
 const _PRESS_ENTER_TO_FINISH = "Press Enter to finish";
 
@@ -244,19 +245,21 @@ export default async function init(
   input.docsDir = docsDirInput.trim() || `${outputPath}/docs`;
 
   // 8. Content sources
-  console.log("\n🔍 [8/9]: Content Sources");
+  console.log("\n🔍 [8/9]: Data Sources");
+  console.log("Please specify the data source we should analyze to generate your documentation.");
   console.log(
-    "Please specify the folders and files we should analyze to generate your documentation (e.g., ./src, ./docs, ./README.md).",
+    `  1. Use paths like ${chalk.green("./src")}, ${chalk.green("./README.md")} or ${chalk.green("!./src/private")}.`,
   );
   console.log(
-    "💡 You can also use glob patterns like src/**/*.js or docs/**/*.md for more specific file matching.",
+    `  2. Use globs like ${chalk.green("src/**/*.js")} or ${chalk.green("!private/**/*.js")} for more specific file matching.`,
   );
+  console.log(`  3. Use URLs like ${chalk.green("https://example.com/openapi.yaml")}.`);
   console.log("💡 If you leave this empty, we will scan the entire directory.");
 
   const sourcePaths = [];
   while (true) {
     const selectedPath = await options.prompts.search({
-      message: "Please enter a file or folder path, or a glob pattern:",
+      message: "Please enter a valid data source:",
       source: async (input) => {
         if (!input || input.trim() === "") {
           return [
@@ -268,13 +271,23 @@ export default async function init(
           ];
         }
 
+        let isIgnore = false;
         const searchTerm = input.trim();
+        let cleanSearchTerm = searchTerm;
+        if (cleanSearchTerm.startsWith("!")) {
+          isIgnore = true;
+          cleanSearchTerm = searchTerm.slice(1);
+        }
 
         // Search for matching files and folders in current directory
-        const availablePaths = getAvailablePaths(searchTerm);
+        const availablePaths = getAvailablePaths(cleanSearchTerm);
 
         // Also add option to use as glob pattern
-        const options = [...availablePaths];
+        const options = [...availablePaths].map((x) => ({
+          ...x,
+          name: isIgnore ? `!${x.name}` : x.name,
+          value: isIgnore ? `!${x.value}` : x.value,
+        }));
 
         // Check if input looks like a glob pattern
         const isGlobPatternResult = isGlobPattern(searchTerm);
@@ -287,6 +300,14 @@ export default async function init(
           });
         }
 
+        if (!isIgnore && isRemoteFile(searchTerm)) {
+          options.push({
+            name: searchTerm,
+            value: searchTerm,
+            description: "Use this remote url for content source.",
+          });
+        }
+
         return options;
       },
     });
@@ -301,6 +322,14 @@ export default async function init(
     // Check if it's a glob pattern
     const isGlobPatternResult = isGlobPattern(trimmedPath);
 
+    if (isRemoteFile(trimmedPath)) {
+      // For remote urls, just add them without validation
+      if (sourcePaths.includes(trimmedPath)) {
+        console.log(`⚠️ URL already exists: ${trimmedPath}`);
+        continue;
+      }
+      sourcePaths.push(trimmedPath);
+    }
     if (isGlobPatternResult) {
       // For glob patterns, just add them without validation
       if (sourcePaths.includes(trimmedPath)) {
@@ -309,8 +338,9 @@ export default async function init(
       }
       sourcePaths.push(trimmedPath);
     } else {
+      const cleanTrimmedPath = trimmedPath.startsWith("!") ? trimmedPath.slice(1) : trimmedPath;
       // Use validatePath to check if path is valid for regular paths
-      const validation = validatePath(trimmedPath);
+      const validation = validatePath(cleanTrimmedPath);
 
       if (!validation.isValid) {
         console.log(`⚠️ ${validation.error}`);

package/agents/media/load-media-description.mjs

@@ -163,31 +163,19 @@ export default async function loadMediaDescription(input, options) {
   let enhancedAssetsContent = "# Available Media Assets for Documentation\n\n";
 
   if (mediaFiles.length > 0) {
-    enhancedAssetsContent += "```yaml\n";
-    enhancedAssetsContent += "assets:\n";
-
-    for (const asset of mediaFiles) {
-      enhancedAssetsContent += `  - name: "${asset.name}"\n`;
-      enhancedAssetsContent += `    path: "${asset.path}"\n`;
-      enhancedAssetsContent += `    type: "${asset.type}"\n`;
-
-      // Add description for images and videos
-      if (asset.type === "image" || asset.type === "video") {
-        const mediaHash = mediaHashMap.get(asset.path);
-        const cachedDesc = cache[mediaHash];
-        if (cachedDesc?.description) {
-          enhancedAssetsContent += `    description: "${cachedDesc.description}"\n`;
-        }
-      }
-
-      // Add dimensions for images and videos
-      if (asset.width && asset.height) {
-        enhancedAssetsContent += `    width: ${asset.width}\n`;
-        enhancedAssetsContent += `    height: ${asset.height}\n`;
+    const assets = mediaFiles.map((x) => {
+      const mediaHash = mediaHashMap.get(x.path);
+      const description = cache[mediaHash]?.description;
+      const result = {
+        name: x.name,
+        path: x.path,
+      };
+      if (description) {
+        result.description = description;
       }
-    }
-
-    enhancedAssetsContent += "```\n";
+      return result;
+    });
+    enhancedAssetsContent += stringify(assets);
   }
 
   return {

package/agents/publish/publish-docs.mjs

@@ -14,14 +14,9 @@ import {
 } from "../../utils/constants/index.mjs";
 import { beforePublishHook, ensureTmpDir } from "../../utils/d2-utils.mjs";
 import { deploy } from "../../utils/deploy.mjs";
-import {
-  getGithubRepoUrl,
-  isHttp,
-  loadConfigFromFile,
-  saveValueToConfig,
-} from "../../utils/utils.mjs";
+import { getGithubRepoUrl, loadConfigFromFile, saveValueToConfig } from "../../utils/utils.mjs";
 import updateBranding from "../utils/update-branding.mjs";
-import { downloadAndUploadImage } from "../../utils/file-utils.mjs";
+import { isRemoteFile, downloadAndUploadImage } from "../../utils/file-utils.mjs";
 
 const BASE_URL = process.env.DOC_SMITH_BASE_URL || CLOUD_SERVICE_URL_PROD;
 
@@ -205,7 +200,7 @@ export default async function publishDocs(
     let finalPath = null;
 
     // Handle project logo download if it's a URL
-    if (projectInfo.icon && isHttp(projectInfo.icon)) {
+    if (projectInfo.icon && isRemoteFile(projectInfo.icon)) {
       const { url: uploadedImageUrl, downloadFinalPath } = await downloadAndUploadImage(
         projectInfo.icon,
         docsDir,

package/agents/schema/document-execution-structure.yaml

@@ -25,7 +25,7 @@ items:
       type: array
       items:
         type: string
-      description: Associated sourceId from dataSources for subsequent translation and content generation, must come from sourceId in datasources, cannot have fake ids, cannot be empty
+      description: Associated sourceId from `<data_sources>` for subsequent translation and content generation, must come from sourceId in `<data_sources>`, cannot have fake ids, cannot be empty
   required:
     - title
     - description

package/agents/schema/document-structure-item.yaml

@@ -0,0 +1,23 @@
+type: object
+description: Document structure item representing a node in the document hierarchy
+properties:
+  title:
+    type: string
+  description:
+    type: string
+  path:
+    type: string
+    description: Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about
+  parentPath:
+    type: string
+    description: Parent node path, if null indicates it is a top-level node
+  sourceIds:
+    type: array
+    description: Associated sourceId from `<data_sources>` for subsequent translation and content generation, must come from sourceId in `<data_sources>`, cannot have fake ids, **cannot be empty**
+    items:
+      type: string
+required:
+  - title
+  - description
+  - path
+  - sourceIds

package/agents/schema/document-structure-refine-item.yaml

@@ -0,0 +1,20 @@
+type: object
+description: Document structure item representing a node in the document hierarchy
+properties:
+  id:
+    type: string
+    description: Unique identifier for the document structure item
+  title:
+    type: string
+  description:
+    type: string
+  path:
+    type: string
+    description: Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about
+  parentPath:
+    type: string
+    description: Parent node path, if null indicates it is a top-level node
+required:
+  - title
+  - description
+  - path

package/agents/schema/document-structure.yaml

@@ -16,7 +16,7 @@ items:
       description: Parent node path, if null indicates it is a top-level node
     sourceIds:
       type: array
-      description: Associated sourceId from dataSources for subsequent translation and content generation, must come from sourceId in datasources, cannot have fake ids, **cannot be empty**
+      description: Associated sourceId from `<data_sources>` for subsequent translation and content generation, must come from sourceId in `<data_sources>`, cannot have fake ids, **cannot be empty**
       items:
         type: string
   required:

package/agents/translate/index.yaml

@@ -33,15 +33,12 @@ skills:
     name: batchTranslateDocument
     skills:
       - ../translate/translate-multilingual.yaml
-      - url: ./record-translation-history.mjs
-      - url: ../utils/save-single-doc.mjs
-        default_input:
-          isTranslate: true
     iterate_on: selectedDocs
     concurrency: 10
   - url: ../utils/check-feedback-refiner.mjs
     default_input:
       stage: translation_refine
+  - url: ./record-translation-history.mjs
   - url: ../utils/action-success.mjs
     default_input:
       action: '✅ Translation completed'

package/agents/translate/record-translation-history.mjs

@@ -1,16 +1,20 @@
 import { recordUpdate } from "../../utils/history-utils.mjs";
 
-export default function recordTranslationHistory({ feedback, path }) {
+export default function recordTranslationHistory({ selectedPaths, feedback }) {
   // Skip if no feedback provided
   if (!feedback?.trim()) {
     return {};
   }
 
+  if (!Array.isArray(selectedPaths) || selectedPaths.length === 0) {
+    return {};
+  }
+
   // Record translation history for this document
   recordUpdate({
     operation: "translation_update",
     feedback: feedback.trim(),
-    documentPath: path,
+    docPaths: selectedPaths,
   });
 
   return {};

package/agents/translate/translate-multilingual.yaml

@@ -22,6 +22,7 @@ skills:
       max_iterations: 5
       return_last_on_max_iterations: true
     task_title: Translate '{{ title }}' to '{{ language }}'
+  - ../utils/save-doc-translation.mjs
 input_schema:
   type: object
   properties:
@@ -48,4 +49,3 @@ output_schema:
             type: string
 iterate_on: translates
 concurrency: 3
-

package/agents/update/batch-generate-document.yaml

@@ -6,7 +6,7 @@ skills:
 input_schema:
   type: object
   properties:
-    datasources:
+    detailDataSource:
       type: string
       description: Context for documentation structure generation, used to assist generate documentation structure
     documentExecutionStructure: ../schema/document-execution-structure.yaml

package/agents/update/batch-update-document.yaml

@@ -4,4 +4,4 @@ name: batchUpdateDocument
 skills:
   - ../update/handle-document-update.yaml
 iterate_on: selectedDocs
-concurrency: 10
+concurrency: 10