@aigne/doc-smith 0.8.15-beta.2 → 0.8.15-beta.4

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [0.8.15-beta.4](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta.3...v0.8.15-beta.4) (2025-10-28)
+
+
+### Bug Fixes
+
+* fix update translate not working ([#221](https://github.com/AIGNE-io/aigne-doc-smith/issues/221)) ([95bc49e](https://github.com/AIGNE-io/aigne-doc-smith/commit/95bc49ec8b1e18fe20dd1360d9707afdd6629bad))
+
+## [0.8.15-beta.3](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta.2...v0.8.15-beta.3) (2025-10-28)
+
+
+### Features
+
+* glossary support load from remote-url ([#220](https://github.com/AIGNE-io/aigne-doc-smith/issues/220)) ([5d52746](https://github.com/AIGNE-io/aigne-doc-smith/commit/5d527469819a4dc08f17338b1d1fb2cd7ccf07c0))
+
+
+### Bug Fixes
+
+* polish generate/update/translate save doc logic ([#218](https://github.com/AIGNE-io/aigne-doc-smith/issues/218)) ([dada1a4](https://github.com/AIGNE-io/aigne-doc-smith/commit/dada1a422d589512f2a0f6e5c69df4845eca44ae))
+
 ## [0.8.15-beta.2](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta.1...v0.8.15-beta.2) (2025-10-24)
 
 

package/agents/generate/index.yaml

@@ -23,6 +23,7 @@ skills:
       savePath:
         $get: outputDir
       fileName: structure-plan.json
+  - ../utils/save-sidebar.mjs
   - type: transform
     name: transformData
     task_render_mode: hide
@@ -47,7 +48,7 @@ skills:
   - url: ../utils/check-feedback-refiner.mjs
     default_input:
       stage: document_structure
-  - ../utils/save-docs.mjs
+  - ../utils/post-generate.mjs
 
 input_schema:
   type: object

package/agents/init/check.mjs

@@ -10,5 +10,7 @@ export default async function checkNeedGenerate({ docsDir, locale, documentExecu
     );
     process.exit(0);
   }
-  return {};
+  return {
+    message: 'Documents found in the docs directory, skipping "generate" step',
+  };
 }

package/agents/init/index.mjs

@@ -20,6 +20,7 @@ import {
   isGlobPattern,
   validatePath,
 } from "../../utils/utils.mjs";
+import { isRemoteFile } from "../../utils/file-utils.mjs";
 
 const _PRESS_ENTER_TO_FINISH = "Press Enter to finish";
 
@@ -246,17 +247,23 @@ export default async function init(
   // 8. Content sources
   console.log("\n🔍 [8/9]: Content Sources");
   console.log(
-    "Please specify the folders and files we should analyze to generate your documentation (e.g., ./src, ./docs, ./README.md).",
+    "Please specify the folders and files we should analyze to generate your documentation.",
   );
   console.log(
-    "💡 You can also use glob patterns like src/**/*.js or docs/**/*.md for more specific file matching.",
+    `  1. You can use local file paths like ${chalk.green("./src")}, ${chalk.green("./docs")}, ${chalk.green("./README.md")} (prefix with '!' to ignore a file or folder like ${chalk.green("!./src/private")}).`,
+  );
+  console.log(
+    `  2. You can also use glob patterns like ${chalk.green("src/**/*.js")} or ${chalk.green("docs/**/*.md")} for more specific file matching. (prefix with '!' to ignore a file or folder like ${chalk.green("!private/**/*.js")}).`,
+  );
+  console.log(
+    `  3. You can also use remote url like ${chalk.green("https://example.com/openapi.yaml")}.`,
   );
   console.log("💡 If you leave this empty, we will scan the entire directory.");
 
   const sourcePaths = [];
   while (true) {
     const selectedPath = await options.prompts.search({
-      message: "Please enter a file or folder path, or a glob pattern:",
+      message: "Please enter a file or folder path, or a glob pattern or remote url:",
       source: async (input) => {
         if (!input || input.trim() === "") {
           return [
@@ -268,13 +275,23 @@ export default async function init(
           ];
         }
 
+        let isIgnore = false;
         const searchTerm = input.trim();
+        let cleanSearchTerm = searchTerm;
+        if (cleanSearchTerm.startsWith("!")) {
+          isIgnore = true;
+          cleanSearchTerm = searchTerm.slice(1);
+        }
 
         // Search for matching files and folders in current directory
-        const availablePaths = getAvailablePaths(searchTerm);
+        const availablePaths = getAvailablePaths(cleanSearchTerm);
 
         // Also add option to use as glob pattern
-        const options = [...availablePaths];
+        const options = [...availablePaths].map((x) => ({
+          ...x,
+          name: isIgnore ? `!${x.name}` : x.name,
+          value: isIgnore ? `!${x.value}` : x.value,
+        }));
 
         // Check if input looks like a glob pattern
         const isGlobPatternResult = isGlobPattern(searchTerm);
@@ -287,6 +304,14 @@ export default async function init(
           });
         }
 
+        if (!isIgnore && isRemoteFile(searchTerm)) {
+          options.push({
+            name: searchTerm,
+            value: searchTerm,
+            description: "Use this remote url for content source.",
+          });
+        }
+
         return options;
       },
     });
@@ -301,6 +326,14 @@ export default async function init(
     // Check if it's a glob pattern
     const isGlobPatternResult = isGlobPattern(trimmedPath);
 
+    if (isRemoteFile(trimmedPath)) {
+      // For remote urls, just add them without validation
+      if (sourcePaths.includes(trimmedPath)) {
+        console.log(`⚠️ URL already exists: ${trimmedPath}`);
+        continue;
+      }
+      sourcePaths.push(trimmedPath);
+    }
     if (isGlobPatternResult) {
       // For glob patterns, just add them without validation
       if (sourcePaths.includes(trimmedPath)) {
@@ -309,8 +342,9 @@ export default async function init(
       }
       sourcePaths.push(trimmedPath);
     } else {
+      const cleanTrimmedPath = trimmedPath.startsWith("!") ? trimmedPath.slice(1) : trimmedPath;
       // Use validatePath to check if path is valid for regular paths
-      const validation = validatePath(trimmedPath);
+      const validation = validatePath(cleanTrimmedPath);
 
       if (!validation.isValid) {
         console.log(`⚠️ ${validation.error}`);

package/agents/publish/publish-docs.mjs

@@ -14,14 +14,9 @@ import {
 } from "../../utils/constants/index.mjs";
 import { beforePublishHook, ensureTmpDir } from "../../utils/d2-utils.mjs";
 import { deploy } from "../../utils/deploy.mjs";
-import {
-  getGithubRepoUrl,
-  isHttp,
-  loadConfigFromFile,
-  saveValueToConfig,
-} from "../../utils/utils.mjs";
+import { getGithubRepoUrl, loadConfigFromFile, saveValueToConfig } from "../../utils/utils.mjs";
 import updateBranding from "../utils/update-branding.mjs";
-import { downloadAndUploadImage } from "../../utils/file-utils.mjs";
+import { isRemoteFile, downloadAndUploadImage } from "../../utils/file-utils.mjs";
 
 const BASE_URL = process.env.DOC_SMITH_BASE_URL || CLOUD_SERVICE_URL_PROD;
 
@@ -205,7 +200,7 @@ export default async function publishDocs(
     let finalPath = null;
 
     // Handle project logo download if it's a URL
-    if (projectInfo.icon && isHttp(projectInfo.icon)) {
+    if (projectInfo.icon && isRemoteFile(projectInfo.icon)) {
       const { url: uploadedImageUrl, downloadFinalPath } = await downloadAndUploadImage(
         projectInfo.icon,
         docsDir,

package/agents/translate/index.yaml

@@ -34,9 +34,6 @@ skills:
     skills:
       - ../translate/translate-multilingual.yaml
       - url: ./record-translation-history.mjs
-      - url: ../utils/save-single-doc.mjs
-        default_input:
-          isTranslate: true
     iterate_on: selectedDocs
     concurrency: 10
   - url: ../utils/check-feedback-refiner.mjs

package/agents/translate/translate-multilingual.yaml

@@ -22,6 +22,7 @@ skills:
       max_iterations: 5
       return_last_on_max_iterations: true
     task_title: Translate '{{ title }}' to '{{ language }}'
+  - ../utils/save-doc-translation.mjs
 input_schema:
   type: object
   properties:
@@ -48,4 +49,3 @@ output_schema:
             type: string
 iterate_on: translates
 concurrency: 3
-

package/agents/update/batch-update-document.yaml

@@ -4,4 +4,4 @@ name: batchUpdateDocument
 skills:
   - ../update/handle-document-update.yaml
 iterate_on: selectedDocs
-concurrency: 10
+concurrency: 10

package/agents/update/check-document.mjs

@@ -2,7 +2,10 @@ import { access, readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { TeamAgent } from "@aigne/core";
+import fs from "fs-extra";
+import pMap from "p-map";
 
+import { getFileName } from "../../utils/utils.mjs";
 import checkDetailResult from "../utils/check-detail-result.mjs";
 
 // Get current script directory
@@ -18,13 +21,13 @@ export default async function checkDocument(
     modifiedFiles,
     forceRegenerate,
     locale,
+    translates,
     ...rest
   },
   options,
 ) {
   // Check if the detail file already exists
-  const flatName = path.replace(/^\//, "").replace(/\//g, "-");
-  const fileFullName = locale === "en" ? `${flatName}.md` : `${flatName}.${locale}.md`;
+  const fileFullName = getFileName(path, locale);
   const filePath = join(docsDir, fileFullName);
   let detailGenerated = true;
   let fileContent = null;
@@ -84,24 +87,42 @@ export default async function checkDocument(
       contentValidationFailed = true;
     }
   }
+  const languages = translates.map((x) => x.language);
+  const lackLanguages = new Set(languages);
+  const skills = [];
 
   // If file exists, sourceIds haven't changed, source files haven't changed, and content validation passes, no need to regenerate
   if (detailGenerated && !sourceIdsChanged && !contentValidationFailed && !forceRegenerate) {
-    return {
-      path,
-      docsDir,
-      ...rest,
-      detailGenerated: true,
-    };
+    await pMap(
+      languages,
+      async (x) => {
+        const languageFileName = getFileName(path, x);
+        const languageFilePath = join(docsDir, languageFileName);
+        if (await fs.exists(languageFilePath)) {
+          lackLanguages.delete(x);
+        }
+      },
+      { concurrency: 10 },
+    );
+    if (lackLanguages.size === 0) {
+      return {
+        path,
+        docsDir,
+        ...rest,
+        detailGenerated: true,
+      };
+    }
+    // translations during generation don't need feedback, content is satisfactory
+    rest.content = fileContent;
+  } else {
+    skills.push(options.context.agents["handleDocumentUpdate"]);
   }
 
+  skills.push(options.context.agents["translateMultilingual"]);
+
   const teamAgent = TeamAgent.from({
     name: "generateDocument",
-    skills: [
-      options.context.agents["handleDocumentUpdate"],
-      options.context.agents["translateMultilingual"],
-      options.context.agents["saveSingleDoc"],
-    ],
+    skills,
   });
   let openAPISpec = null;
 
@@ -119,6 +140,7 @@ export default async function checkDocument(
 
   const result = await options.context.invoke(teamAgent, {
     ...rest,
+    translates: translates.filter((x) => lackLanguages.has(x.language)),
     locale,
     docsDir,
     path,

package/agents/update/generate-diagram.yaml

@@ -0,0 +1,30 @@
+type: team
+task_render_mode: collapse
+name: generateDiagram
+skills:
+  - ../generate/draw-diagram.yaml
+  - ../generate/wrap-diagram-code.mjs
+reflection:
+  reviewer: ../generate/check-diagram.mjs
+  is_approved: isValid
+  max_iterations: 5
+  return_last_on_max_iterations: false
+  custom_error_message: "MUST NOT generate any diagram: validation failed after max iterations."
+input_schema:
+  type: object
+  properties:
+    documentContent:
+      type: string
+      description: The **raw text content** of the current document. (**Note:** This is the original document and **does not include** any diagram source code.)
+    locale:
+      type: string
+      description: Language for diagram labels and text
+      default: en
+  required:
+    - documentContent
+output_schema:
+  type: object
+  properties:
+    diagramSourceCode:
+      type: string
+      description: The **diagram source code** generated from the input text.

package/agents/update/generate-document.yaml

@@ -58,33 +58,4 @@ afs:
           should search and read as needed while generating document content
 keep_text_in_tool_uses: false
 skills:
-  - type: team
-    task_render_mode: collapse
-    name: generateDiagram
-    skills:
-      - ../generate/draw-diagram.yaml
-      - ../generate/wrap-diagram-code.mjs
-    reflection:
-      reviewer: ../generate/check-diagram.mjs
-      is_approved: isValid
-      max_iterations: 5
-      return_last_on_max_iterations: false
-      custom_error_message: "MUST NOT generate any diagram: validation failed after max iterations."
-    input_schema:
-      type: object
-      properties:
-        documentContent:
-          type: string
-          description: The **raw text content** of the current document. (**Note:** This is the original document and **does not include** any diagram source code.)
-        locale:
-          type: string
-          description: Language for diagram labels and text
-          default: en
-      required:
-        - documentContent
-    output_schema:
-      type: object
-      properties:
-        diagramSourceCode:
-          type: string
-          description: The **diagram source code** generated from the input text.
+  - ./generate-diagram.yaml

package/agents/update/handle-document-update.yaml

@@ -23,6 +23,7 @@ skills:
       max_iterations: 5
       return_last_on_max_iterations: true
     task_title: Generate document for '{{ title }}'
+  - ../utils/save-doc.mjs
 input_schema:
   type: object
   properties:

package/agents/update/save-and-translate-document.mjs

@@ -1,4 +1,4 @@
-import { recordUpdate } from "../../utils/history-utils.mjs";
+import pMap from "p-map";
 
 export default async function saveAndTranslateDocument(input, options) {
   const { selectedDocs, docsDir, translateLanguages, locale } = input;
@@ -7,21 +7,6 @@ export default async function saveAndTranslateDocument(input, options) {
     return {};
   }
 
-  // Saves a document with optional translation data
-  const saveDocument = async (doc, translates = null, isTranslate = false) => {
-    const saveAgent = options.context.agents["saveSingleDoc"];
-
-    return await options.context.invoke(saveAgent, {
-      path: doc.path,
-      content: doc.content,
-      docsDir: docsDir,
-      locale: locale,
-      translates: translates || doc.translates,
-      labels: doc.labels,
-      isTranslate: isTranslate,
-    });
-  };
-
   // Only prompt user if translation is actually needed
   let shouldTranslate = false;
   if (
@@ -46,28 +31,6 @@ export default async function saveAndTranslateDocument(input, options) {
 
   // Save documents in batches
   const batchSize = 3;
-  for (let i = 0; i < selectedDocs.length; i += batchSize) {
-    const batch = selectedDocs.slice(i, i + batchSize);
-
-    const savePromises = batch.map(async (doc) => {
-      try {
-        await saveDocument(doc);
-
-        // Record history for each document if feedback is provided
-        if (doc.feedback?.trim()) {
-          recordUpdate({
-            operation: "document_update",
-            feedback: doc.feedback.trim(),
-            documentPath: doc.path,
-          });
-        }
-      } catch (error) {
-        console.error(`❌ Failed to save document ${doc.path}:`, error.message);
-      }
-    });
-
-    await Promise.all(savePromises);
-  }
 
   // Return results if user chose to skip translation
   if (!shouldTranslate) {
@@ -77,30 +40,27 @@ export default async function saveAndTranslateDocument(input, options) {
   // Translate documents in batches
   const translateAgent = options.context.agents["translateMultilingual"];
 
-  for (let i = 0; i < selectedDocs.length; i += batchSize) {
-    const batch = selectedDocs.slice(i, i + batchSize);
-
-    const translatePromises = batch.map(async (doc) => {
+  await pMap(
+    selectedDocs,
+    async (doc) => {
       try {
         // Clear feedback to ensure translation is not affected by update feedback
         doc.feedback = "";
 
-        const result = await options.context.invoke(translateAgent, {
+        await options.context.invoke(translateAgent, {
           ...input, // context is required
           content: doc.content,
           translates: doc.translates,
           title: doc.title,
+          path: doc.path,
+          docsDir,
         });
-
-        // Save the translated content
-        await saveDocument(doc, result.translates, true);
       } catch (error) {
         console.error(`❌ Failed to translate document ${doc.path}:`, error.message);
       }
-    });
-
-    await Promise.all(translatePromises);
-  }
+    },
+    { concurrency: batchSize },
+  );
 
   return {};
 }

package/agents/update/update-document-detail.yaml

@@ -57,4 +57,5 @@ afs:
 keep_text_in_tool_uses: false
 skills:
   - ./document-tools/update-document-content.mjs
+  - ./generate-diagram.yaml
 task_render_mode: collapse

package/agents/update/update-single-document.yaml

@@ -3,5 +3,6 @@ name: updateSingleDocument
 skills:
   - ../utils/transform-detail-datasources.mjs
   - ../update/user-review-document.mjs
+  - ../utils/save-doc.mjs
 iterate_on: selectedDocs
 concurrency: 1

package/agents/utils/load-sources.mjs

@@ -8,7 +8,7 @@ import {
   loadFilesFromPaths,
   readFileContents,
   getMimeType,
-  checkIsRemoteFile,
+  isRemoteFile,
 } from "../../utils/file-utils.mjs";
 import {
   getCurrentGitHead,
@@ -144,7 +144,7 @@ export default async function loadSources(
     files.map(async (file) => {
       const ext = path.extname(file).toLowerCase();
 
-      if (mediaExtensions.includes(ext) && !checkIsRemoteFile(file)) {
+      if (mediaExtensions.includes(ext) && !isRemoteFile(file)) {
         // This is a media file
         const relativePath = path.relative(docsDir, file);
         const fileName = path.basename(file);
@@ -214,15 +214,15 @@ export default async function loadSources(
     return !isOpenAPI;
   });
 
-  const httpFileList = [];
+  const remoteFileList = [];
 
   sourceFiles.forEach((file) => {
-    if (checkIsRemoteFile(file.sourceId)) {
-      httpFileList.push(file);
+    if (isRemoteFile(file.sourceId)) {
+      remoteFileList.push(file);
     }
   });
   if (options?.context?.userContext) {
-    options.context.userContext.httpFileList = httpFileList;
+    options.context.userContext.remoteFileList = remoteFileList;
   }
 
   // Build allSources string using utility function

package/agents/utils/{save-docs.mjs → post-generate.mjs}

@@ -1,4 +1,4 @@
-import { readdir, unlink, writeFile } from "node:fs/promises";
+import { readdir, unlink } from "node:fs/promises";
 import { join } from "node:path";
 import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
 import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
@@ -10,7 +10,7 @@ import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
  * @param {Array<string>} [params.translateLanguages] - Translation languages
  * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
  */
-export default async function saveDocs({
+export default async function postGenerate({
   documentExecutionStructure: documentStructure,
   docsDir,
   translateLanguages = [],
@@ -26,15 +26,6 @@ export default async function saveDocs({
     console.warn("Failed to save git HEAD:", err.message);
   }
 
-  // Generate _sidebar.md
-  try {
-    const sidebar = generateSidebar(documentStructure);
-    const sidebarPath = join(docsDir, "_sidebar.md");
-    await writeFile(sidebarPath, sidebar, "utf8");
-  } catch (err) {
-    console.error("Failed to save _sidebar.md:", err.message);
-  }
-
   // Clean up invalid .md files that are no longer in the documentation structure
   try {
     await cleanupInvalidFiles(documentStructure, docsDir, translateLanguages, locale);
@@ -140,43 +131,3 @@ async function cleanupInvalidFiles(documentStructure, docsDir, translateLanguage
 }
 
 // Generate sidebar content, support nested structure, and the order is consistent with documentStructure
-function generateSidebar(documentStructure) {
-  // Build tree structure
-  const root = {};
-  for (const { path, title, parentId } of documentStructure) {
-    const relPath = path.replace(/^\//, "");
-    const segments = relPath.split("/");
-    let node = root;
-    for (let i = 0; i < segments.length; i++) {
-      const seg = segments[i];
-      if (!node[seg])
-        node[seg] = {
-          __children: {},
-          __title: null,
-          __fullPath: segments.slice(0, i + 1).join("/"),
-          __parentId: parentId,
-        };
-      if (i === segments.length - 1) node[seg].__title = title;
-      node = node[seg].__children;
-    }
-  }
-  // Recursively generate sidebar text, the link path is the flattened file name
-  function walk(node, parentSegments = [], indent = "") {
-    let out = "";
-    for (const key of Object.keys(node)) {
-      const item = node[key];
-      const fullSegments = [...parentSegments, key];
-      const flatFile = `${fullSegments.join("-")}.md`;
-      if (item.__title) {
-        const realIndent = item.__parentId === null ? "" : indent;
-        out += `${realIndent}* [${item.__title}](/${flatFile})\n`;
-      }
-      const children = item.__children;
-      if (Object.keys(children).length > 0) {
-        out += walk(children, fullSegments, `${indent}  `);
-      }
-    }
-    return out;
-  }
-  return walk(root).replace(/\n+$/, "");
-}

package/agents/utils/save-doc-translation.mjs

@@ -0,0 +1,27 @@
+import { saveDocTranslation as _saveDocTranslation } from "../../utils/utils.mjs";
+
+export default async function saveDocTranslation({
+  path,
+  docsDir,
+  translation,
+  language,
+  labels,
+  isShowMessage = false,
+}) {
+  await _saveDocTranslation({
+    path,
+    docsDir,
+    language,
+    translation,
+    labels,
+  });
+
+  if (isShowMessage) {
+    const message = `✅ Translation completed successfully.`;
+    return { message };
+  }
+
+  return {};
+}
+
+saveDocTranslation.task_render_mode = "hide";

package/agents/utils/save-doc.mjs

@@ -0,0 +1,55 @@
+import { recordUpdate } from "../../utils/history-utils.mjs";
+import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
+import { saveDoc as _saveDoc } from "../../utils/utils.mjs";
+
+export default async function saveDoc({
+  path,
+  content,
+  docsDir,
+  labels,
+  locale,
+  feedback,
+  isShowMessage = false,
+  ...rest
+}) {
+  await _saveDoc({
+    path,
+    content,
+    docsDir,
+    labels,
+    locale,
+  });
+
+  if (feedback?.trim()) {
+    recordUpdate({
+      operation: "document_update",
+      feedback: feedback.trim(),
+      documentPath: path,
+    });
+  }
+
+  if (isShowMessage) {
+    // Shutdown mermaid worker pool to ensure clean exit
+    try {
+      await shutdownMermaidWorkerPool();
+    } catch (error) {
+      console.warn("Failed to shutdown mermaid worker pool:", error.message);
+    }
+
+    const message = `✅ Document updated successfully.`;
+    return { message };
+  }
+
+  return {
+    path,
+    content,
+    docsDir,
+    labels,
+    locale,
+    feedback,
+    isShowMessage,
+    ...rest,
+  };
+}
+
+saveDoc.task_render_mode = "hide";

package/agents/utils/save-sidebar.mjs

@@ -0,0 +1,59 @@
+import { join } from "node:path";
+import fs from "fs-extra";
+
+export default async function saveSidebar({ documentStructure, docsDir }) {
+  // Generate _sidebar.md
+  try {
+    const sidebar = generateSidebar(documentStructure);
+    const sidebarPath = join(docsDir, "_sidebar.md");
+
+    await fs.ensureDir(docsDir);
+    await fs.writeFile(sidebarPath, sidebar, "utf8");
+  } catch (err) {
+    console.error("Failed to save _sidebar.md:", err.message);
+  }
+  return {};
+}
+
+// Recursively generate sidebar text, the link path is the flattened file name
+function walk(node, parentSegments = [], indent = "") {
+  let out = "";
+  for (const key of Object.keys(node)) {
+    const item = node[key];
+    const fullSegments = [...parentSegments, key];
+    const flatFile = `${fullSegments.join("-")}.md`;
+    if (item.__title) {
+      const realIndent = item.__parentId === null ? "" : indent;
+      out += `${realIndent}* [${item.__title}](/${flatFile})\n`;
+    }
+    const children = item.__children;
+    if (Object.keys(children).length > 0) {
+      out += walk(children, fullSegments, `${indent}  `);
+    }
+  }
+  return out;
+}
+
+function generateSidebar(documentStructure) {
+  // Build tree structure
+  const root = {};
+  for (const { path, title, parentId } of documentStructure) {
+    const relPath = path.replace(/^\//, "");
+    const segments = relPath.split("/");
+    let node = root;
+    for (let i = 0; i < segments.length; i++) {
+      const seg = segments[i];
+      if (!node[seg])
+        node[seg] = {
+          __children: {},
+          __title: null,
+          __fullPath: segments.slice(0, i + 1).join("/"),
+          __parentId: parentId,
+        };
+      if (i === segments.length - 1) node[seg].__title = title;
+      node = node[seg].__children;
+    }
+  }
+
+  return walk(root).replace(/\n+$/, "");
+}

package/agents/utils/transform-detail-datasources.mjs

@@ -1,11 +1,11 @@
 import fs from "node:fs";
+import { isRemoteFile } from "../../utils/file-utils.mjs";
 import { normalizePath, toRelativePath } from "../../utils/utils.mjs";
-import { checkIsRemoteFile } from "../../utils/file-utils.mjs";
 
 export default function transformDetailDatasources({ sourceIds }, options = {}) {
   // Read file content for each sourceId, ignoring failures
   let openAPISpec;
-  const httpFileList = options?.context?.userContext?.httpFileList || [];
+  const remoteFileList = options?.context?.userContext?.remoteFileList || [];
   const contents = (sourceIds || [])
     .filter((id) => {
       const openApiSourceId = options?.context?.userContext?.openAPISpec?.sourceId;
@@ -17,8 +17,8 @@ export default function transformDetailDatasources({ sourceIds }, options = {})
     })
     .map((id) => {
       try {
-        if (checkIsRemoteFile(id)) {
-          const findFile = httpFileList.find((f) => f.sourceId === id);
+        if (isRemoteFile(id)) {
+          const findFile = remoteFileList.find((f) => f.sourceId === id);
           if (findFile) {
             return `// sourceId: ${id}\n${findFile.content}\n`;
           }

package/aigne.yaml

@@ -18,7 +18,7 @@ agents:
   - ./agents/generate/check-document-structure.yaml
   - ./agents/generate/user-review-document-structure.mjs
   - ./agents/generate/index.yaml
-  
+
   # Documentation Structure Tools
   - ./agents/generate/document-structure-tools/add-document.mjs
   - ./agents/generate/document-structure-tools/delete-document.mjs
@@ -47,6 +47,7 @@ agents:
 
   # Publishing
   - ./agents/publish/publish-docs.mjs
+  - ./agents/publish/translate-meta.mjs
   - ./agents/publish/index.yaml
 
   # Media
@@ -65,9 +66,11 @@ agents:
 
   # Utilities
   - ./agents/utils/load-sources.mjs
-  - ./agents/utils/save-docs.mjs
+  - ./agents/utils/post-generate.mjs
+  - ./agents/utils/save-sidebar.mjs
   - ./agents/utils/transform-detail-datasources.mjs
-  - ./agents/utils/save-single-doc.mjs
+  - ./agents/utils/save-doc.mjs
+  - ./agents/utils/save-doc-translation.mjs
   - ./agents/utils/save-output.mjs
   - ./agents/utils/format-document-structure.mjs
   - ./agents/utils/find-item-by-path.mjs

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/doc-smith",
-  "version": "0.8.15-beta.2",
+  "version": "0.8.15-beta.4",
   "description": "AI-driven documentation generation tool built on the AIGNE Framework",
   "publishConfig": {
     "access": "public"

package/prompts/detail/generate/user-prompt.md

@@ -97,6 +97,7 @@ Generate detailed and well-structured document for the current {{nodeName}} base
 YOU SHOULD:
 - Use AFS tools `afs_list` `afs_search` or `afs_read` to gather relevant and accurate information to enhance the content.
 - Follow rules in `<diagram_generation_guide>`: use `generateDiagram` tool to create and embed a diagram when appropriate, following the diagram generation guidelines.
+- If the `generateDiagram` tool is not called, do not attempt to add any diagrams.
 
 <steps>
 1. Analyze the provided document structure and user requirements to plan the content.

package/utils/file-utils.mjs

@@ -286,7 +286,7 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
         continue;
       }
 
-      if (checkIsRemoteFile(dir)) {
+      if (isRemoteFile(dir)) {
         allFiles.push(dir);
         continue;
       }
@@ -387,8 +387,8 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
  * @returns {Promise<boolean>} True if file appears to be a text file
  */
 async function isTextFile(filePath) {
-  if (checkIsRemoteFile(filePath)) {
-    return checkIsHttpTextFile(filePath);
+  if (isRemoteFile(filePath)) {
+    return isRemoteTextFile(filePath);
   }
 
   try {
@@ -400,14 +400,21 @@ async function isTextFile(filePath) {
   }
 }
 
-export function checkIsRemoteFile(filepath) {
-  if (filepath.startsWith("http://") || filepath.startsWith("https://")) {
+/**
+ * Check if a string is an HTTP/HTTPS URL
+ * @param {string} fileUrl - The string to check
+ * @returns {boolean} - True if the string starts with http:// or https://
+ */
+export function isRemoteFile(fileUrl) {
+  if (typeof fileUrl !== "string") return false;
+
+  if (fileUrl.startsWith("http://") || fileUrl.startsWith("https://")) {
     return true;
   }
   return false;
 }
 
-export async function checkIsHttpTextFile(fileUrl) {
+export async function isRemoteTextFile(fileUrl) {
   try {
     const res = await fetch(fileUrl, {
       method: "HEAD",
@@ -435,14 +442,14 @@ export async function checkIsHttpTextFile(fileUrl) {
   }
 }
 
-export async function getHttpFileContent(file) {
-  if (!file) return null;
+export async function getRemoteFileContent(fileUrl) {
+  if (!fileUrl) return null;
   try {
-    const res = await fetch(file);
+    const res = await fetch(fileUrl);
     const text = await res.text();
     return text;
   } catch (error) {
-    debug(`Failed to fetch HTTP file content: ${file} - ${error.message}`);
+    debug(`Failed to fetch HTTP file content: ${fileUrl} - ${error.message}`);
     return null;
   }
 }
@@ -469,8 +476,8 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
       }
 
       try {
-        if (checkIsRemoteFile(file)) {
-          const content = await getHttpFileContent(file);
+        if (isRemoteFile(file)) {
+          const content = await getRemoteFileContent(file);
           if (content) {
             return {
               sourceId: file,

package/utils/load-config.mjs

@@ -28,7 +28,7 @@ export default async function loadConfig({ config, appUrl }) {
     }
 
     // Parse new configuration fields and convert keys to actual content
-    const processedConfig = processConfigFields(parsedConfig);
+    const processedConfig = await processConfigFields(parsedConfig);
 
     return {
       lastGitHead: parsedConfig.lastGitHead || "",

package/utils/utils.mjs

@@ -18,6 +18,7 @@ import {
   SUPPORTED_LANGUAGES,
   TARGET_AUDIENCES,
 } from "./constants/index.mjs";
+import { isRemoteFile, getRemoteFileContent } from "./file-utils.mjs";
 
 /**
  * Normalize path to absolute path for consistent comparison
@@ -47,16 +48,6 @@ export function isGlobPattern(pattern) {
   return /[*?[\]]|(\*\*)/.test(pattern);
 }
 
-/**
- * Check if a string is an HTTP/HTTPS URL
- * @param {string} url - The string to check
- * @returns {boolean} - True if the string starts with http:// or https://
- */
-export function isHttp(url) {
-  if (typeof url !== "string") return false;
-  return url.startsWith("http://") || url.startsWith("https://");
-}
-
 export function processContent({ content }) {
   // Match markdown regular links [text](link), exclude images ![text](link)
   return content.replace(/(?<!!)\[([^\]]+)\]\(([^)]+)\)/g, (match, text, link) => {
@@ -81,77 +72,72 @@ export function processContent({ content }) {
   });
 }
 
+// Helper function to generate filename based on language
+export function getFileName(docPath, language) {
+  // Flatten path: remove leading /, replace all / with -
+  const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
+  const isEnglish = language === "en";
+  return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
+}
+
 /**
- * Save a single document and its translations to files
+ * Save a single document to files
  * @param {Object} params
  * @param {string} params.path - Relative path (without extension)
  * @param {string} params.content - Main document content
  * @param {string} params.docsDir - Root directory
  * @param {string} params.locale - Main content language (e.g., 'en', 'zh', 'fr')
- * @param {Array<{language: string, translation: string}>} [params.translates] - Translation content
  * @param {Array<string>} [params.labels] - Document labels for front matter
- * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
+ * @returns {Promise<{ path: string, success: boolean, error?: string }>}
  */
-export async function saveDocWithTranslations({
-  path: docPath,
-  content,
-  docsDir,
-  locale,
-  translates = [],
-  labels,
-  isTranslate = false,
-}) {
-  const results = [];
+export async function saveDoc({ path: docPath, content, docsDir, locale, labels }) {
   try {
-    // Flatten path: remove leading /, replace all / with -
-    const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
     await fs.mkdir(docsDir, { recursive: true });
+    const mainFileName = getFileName(docPath, locale);
+    const mainFilePath = path.join(docsDir, mainFileName);
 
-    // Helper function to generate filename based on language
-    const getFileName = (language) => {
-      const isEnglish = language === "en";
-      return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
-    };
-
-    // Save main content with appropriate filename based on locale (skip if isTranslate is true)
-    if (!isTranslate) {
-      const mainFileName = getFileName(locale);
-      const mainFilePath = path.join(docsDir, mainFileName);
-
-      // Add labels front matter if labels are provided
-      let finalContent = processContent({ content });
+    // Add labels front matter if labels are provided
+    let finalContent = processContent({ content });
 
-      if (labels && labels.length > 0) {
-        const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
-        finalContent = frontMatter + finalContent;
-      }
-
-      await fs.writeFile(mainFilePath, finalContent, "utf8");
-      results.push({ path: mainFilePath, success: true });
+    if (labels && labels.length > 0) {
+      const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
+      finalContent = frontMatter + finalContent;
     }
 
-    // Process all translations
-    for (const translate of translates) {
-      const translateFileName = getFileName(translate.language);
-      const translatePath = path.join(docsDir, translateFileName);
+    await fs.writeFile(mainFilePath, finalContent, "utf8");
+    return { path: mainFilePath, success: true };
+  } catch (err) {
+    return { path: docPath, success: false, error: err.message };
+  }
+}
 
-      // Add labels front matter to translation content if labels are provided
-      let finalTranslationContent = processContent({
-        content: translate.translation,
-      });
+export async function saveDocTranslation({
+  path: docPath,
+  docsDir,
+  translation,
+  language,
+  labels,
+}) {
+  try {
+    await fs.mkdir(docsDir, { recursive: true });
+    const translateFileName = getFileName(docPath, language);
+    const translatePath = path.join(docsDir, translateFileName);
 
-      if (labels && labels.length > 0) {
-        const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
-        finalTranslationContent = frontMatter + finalTranslationContent;
-      }
+    // Add labels front matter to translation content if labels are provided
+    let finalTranslationContent = processContent({
+      content: translation,
+    });
 
-      await fs.writeFile(translatePath, finalTranslationContent, "utf8");
-      results.push({ path: translatePath, success: true });
+    if (labels && labels.length > 0) {
+      const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
+      finalTranslationContent = frontMatter + finalTranslationContent;
     }
+
+    await fs.writeFile(translatePath, finalTranslationContent, "utf8");
+    return { path: translatePath, success: true };
   } catch (err) {
-    results.push({ path: docPath, success: false, error: err.message });
+    return { path: docPath, success: false, error: err.message };
   }
-  return results;
 }
 
 /**
@@ -963,7 +949,7 @@ export function processTargetAudience(targetAudienceTypes, existingTargetAudienc
  * @param {Object} config - Parsed configuration
  * @returns {Object} Processed configuration with content fields
  */
-export function processConfigFields(config) {
+export async function processConfigFields(config) {
   const processed = {};
   const allRulesContent = [];
 
@@ -995,7 +981,15 @@ export function processConfigFields(config) {
     if (typeof config.rules === "string") {
       const existingRules = config.rules.trim();
       if (existingRules) {
-        allRulesContent.push(existingRules);
+        // load rules from remote url
+        if (isRemoteFile(existingRules)) {
+          const remoteFileContent = await getRemoteFileContent(existingRules);
+          if (remoteFileContent) {
+            allRulesContent.push(remoteFileContent);
+          }
+        } else {
+          allRulesContent.push(existingRules);
+        }
       }
     } else if (Array.isArray(config.rules)) {
       // Handle array of rules - join them with newlines
@@ -1054,6 +1048,12 @@ export function processConfigFields(config) {
     }
   }
 
+  if (config.glossary) {
+    if (isRemoteFile(config.glossary)) {
+      processed.glossary = await getRemoteFileContent(config.glossary);
+    }
+  }
+
   // Detect and handle conflicts in user selections
   const conflicts = detectResolvableConflicts(config);
   if (conflicts.length > 0) {
@@ -1097,8 +1097,10 @@ export function processConfigFields(config) {
  * @returns {Promise<any>} - The processed configuration with file content loaded in place of references.
  */
 export async function resolveFileReferences(obj, basePath = process.cwd()) {
-  if (typeof obj === "string" && obj.startsWith("@")) {
-    return await loadFileContent(obj.slice(1), basePath);
+  if (typeof obj === "string") {
+    if (obj.startsWith("@")) {
+      return await loadFileContent(obj.slice(1), basePath);
+    }
   }
 
   if (Array.isArray(obj)) {

package/agents/utils/save-single-doc.mjs

@@ -1,41 +0,0 @@
-import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
-import { saveDocWithTranslations } from "../../utils/utils.mjs";
-
-export default async function saveSingleDoc({
-  path,
-  content,
-  docsDir,
-  translates,
-  labels,
-  locale,
-  isTranslate = false,
-  isShowMessage = false,
-}) {
-  const _results = await saveDocWithTranslations({
-    path,
-    content,
-    docsDir,
-    translates,
-    labels,
-    locale,
-    isTranslate,
-  });
-
-  if (isShowMessage) {
-    // Shutdown mermaid worker pool to ensure clean exit
-    try {
-      await shutdownMermaidWorkerPool();
-    } catch (error) {
-      console.warn("Failed to shutdown mermaid worker pool:", error.message);
-    }
-
-    const message = isTranslate
-      ? `✅ Translation completed successfully`
-      : `✅ Document updated successfully`;
-    return { message };
-  }
-
-  return {};
-}
-
-saveSingleDoc.task_render_mode = "hide";