@aigne/doc-smith 0.8.15-beta.1 → 0.8.15-beta.10

package/agents/update/check-document.mjs

@@ -2,7 +2,10 @@ import { access, readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { TeamAgent } from "@aigne/core";
+import fs from "fs-extra";
+import pMap from "p-map";
 
+import { getFileName } from "../../utils/utils.mjs";
 import checkDetailResult from "../utils/check-detail-result.mjs";
 
 // Get current script directory
@@ -18,13 +21,13 @@ export default async function checkDocument(
     modifiedFiles,
     forceRegenerate,
     locale,
+    translates,
     ...rest
   },
   options,
 ) {
   // Check if the detail file already exists
-  const flatName = path.replace(/^\//, "").replace(/\//g, "-");
-  const fileFullName = locale === "en" ? `${flatName}.md` : `${flatName}.${locale}.md`;
+  const fileFullName = getFileName(path, locale);
   const filePath = join(docsDir, fileFullName);
   let detailGenerated = true;
   let fileContent = null;
@@ -84,24 +87,42 @@ export default async function checkDocument(
       contentValidationFailed = true;
     }
   }
+  const languages = translates.map((x) => x.language);
+  const lackLanguages = new Set(languages);
+  const skills = [];
 
   // If file exists, sourceIds haven't changed, source files haven't changed, and content validation passes, no need to regenerate
   if (detailGenerated && !sourceIdsChanged && !contentValidationFailed && !forceRegenerate) {
-    return {
-      path,
-      docsDir,
-      ...rest,
-      detailGenerated: true,
-    };
+    await pMap(
+      languages,
+      async (x) => {
+        const languageFileName = getFileName(path, x);
+        const languageFilePath = join(docsDir, languageFileName);
+        if (await fs.exists(languageFilePath)) {
+          lackLanguages.delete(x);
+        }
+      },
+      { concurrency: 10 },
+    );
+    if (lackLanguages.size === 0) {
+      return {
+        path,
+        docsDir,
+        ...rest,
+        detailGenerated: true,
+      };
+    }
+    // translations during generation don't need feedback, content is satisfactory
+    rest.content = fileContent;
+  } else {
+    skills.push(options.context.agents["handleDocumentUpdate"]);
   }
 
+  skills.push(options.context.agents["translateMultilingual"]);
+
   const teamAgent = TeamAgent.from({
     name: "generateDocument",
-    skills: [
-      options.context.agents["handleDocumentUpdate"],
-      options.context.agents["translateMultilingual"],
-      options.context.agents["saveSingleDoc"],
-    ],
+    skills,
   });
   let openAPISpec = null;
 
@@ -119,6 +140,7 @@ export default async function checkDocument(
 
   const result = await options.context.invoke(teamAgent, {
     ...rest,
+    translates: translates.filter((x) => lackLanguages.has(x.language)),
     locale,
     docsDir,
     path,

package/agents/update/check-generate-diagram.mjs

@@ -0,0 +1,26 @@
+const placeholder = "DIAGRAM_PLACEHOLDER";
+
+export default async function checkGenerateDiagram(
+  { needDiagram, documentContent, locale },
+  options,
+) {
+  if (!needDiagram) {
+    return {};
+  }
+
+  const generateAgent = options.context?.agents?.["generateDiagram"];
+  let content = documentContent;
+
+  try {
+    const { diagramSourceCode } = await options.context.invoke(generateAgent, {
+      documentContent,
+      locale,
+    });
+    content = content.replace(placeholder, diagramSourceCode);
+  } catch (error) {
+    // FIXME: @zhanghan should regenerate document without diagram
+    content = content.replace(placeholder, "");
+    console.log(`⚠️  Skip generate any diagram: ${error.message}`);
+  }
+  return { content };
+}

package/agents/update/generate-diagram.yaml

@@ -0,0 +1,29 @@
+type: team
+task_render_mode: collapse
+name: generateDiagram
+skills:
+  - ../generate/draw-diagram.yaml
+  - ../generate/wrap-diagram-code.mjs
+reflection:
+  reviewer: ../generate/check-diagram.mjs
+  is_approved: isValid
+  max_iterations: 5
+  return_last_on_max_iterations: false
+input_schema:
+  type: object
+  properties:
+    documentContent:
+      type: string
+      description: The **raw text content** of the current document. (**Note:** This is the original document and **does not include** any diagram source code.)
+    locale:
+      type: string
+      description: Language for diagram labels and text
+      default: en
+  required:
+    - documentContent
+output_schema:
+  type: object
+  properties:
+    diagramSourceCode:
+      type: string
+      description: The **diagram source code** generated from the input text.

package/agents/update/generate-document.yaml

@@ -16,7 +16,7 @@ input_schema:
     locale:
       type: string
       description: User language, such as zh, en
-    datasources:
+    detailDataSource:
       type: string
       description: Source data and context for document content generation
     targetAudience:
@@ -44,9 +44,21 @@ input_schema:
       description: Additional supplementary information
   required:
     - rules
-    - datasources
+    - detailDataSource
     - originalDocumentStructure
-output_key: content
+# output_key: content
+output_schema:
+  type: object
+  properties:
+    content:
+      type: string
+      description: Document content
+    needDiagram:
+      type: boolean
+      description: Does this document need to generate diagram?
+  required:
+    - content
+    - detailDataSource
 afs:
   modules:
     - module: system-fs
@@ -57,30 +69,5 @@ afs:
           Codebase of the project to be documented used as context for document generation,
           should search and read as needed while generating document content
 keep_text_in_tool_uses: false
-skills:
-  - type: team
-    task_render_mode: collapse
-    name: generateDiagram
-    skills:
-      - ../generate/draw-diagram.yaml
-      - ../generate/wrap-diagram-code.mjs
-    reflection:
-      reviewer: ../generate/check-diagram.mjs
-      is_approved: isValid
-      max_iterations: 5
-      return_last_on_max_iterations: false
-      custom_error_message: "MUST NOT generate any diagram: validation failed after max iterations."
-    input_schema:
-      type: object
-      properties:
-        documentContent:
-          type: string
-          description: The **raw text content** of the current document. (**Note:** This is the original document and **does not include** any diagram source code.)
-      required:
-        - documentContent
-    output_schema:
-      type: object
-      properties:
-        diagramSourceCode:
-          type: string
-          description: The **diagram source code** generated from the input text.
+# skills:
+#   - ./generate-diagram.yaml

package/agents/update/handle-document-update.yaml

@@ -2,7 +2,7 @@ type: team
 name: handleDocumentUpdate
 description: Update a document in a batch
 skills:
-  - ../utils/transform-detail-datasources.mjs
+  - ../utils/transform-detail-data-sources.mjs
   - type: team
     task_render_mode: collapse
     name: generateDocumentContent
@@ -23,6 +23,15 @@ skills:
       max_iterations: 5
       return_last_on_max_iterations: true
     task_title: Generate document for '{{ title }}'
+  - type: transform
+    jsonata: |
+      $merge([
+        $,
+        { "documentContent": content }
+      ])
+  - ./check-generate-diagram.mjs
+  # - ../generate/merge-diagram.yaml
+  - ../utils/save-doc.mjs
 input_schema:
   type: object
   properties:

package/agents/update/save-and-translate-document.mjs

@@ -1,3 +1,4 @@
+import pMap from "p-map";
 import { recordUpdate } from "../../utils/history-utils.mjs";
 
 export default async function saveAndTranslateDocument(input, options) {
@@ -7,20 +8,15 @@ export default async function saveAndTranslateDocument(input, options) {
     return {};
   }
 
-  // Saves a document with optional translation data
-  const saveDocument = async (doc, translates = null, isTranslate = false) => {
-    const saveAgent = options.context.agents["saveSingleDoc"];
-
-    return await options.context.invoke(saveAgent, {
-      path: doc.path,
-      content: doc.content,
-      docsDir: docsDir,
-      locale: locale,
-      translates: translates || doc.translates,
-      labels: doc.labels,
-      isTranslate: isTranslate,
+  // Record history if feedback is provided
+  const doc = selectedDocs[0];
+  if (doc.feedback?.trim()) {
+    recordUpdate({
+      operation: "document_update",
+      feedback: doc.feedback.trim(),
+      docPaths: selectedDocs.map((v) => v.path),
     });
-  };
+  }
 
   // Only prompt user if translation is actually needed
   let shouldTranslate = false;
@@ -46,28 +42,6 @@ export default async function saveAndTranslateDocument(input, options) {
 
   // Save documents in batches
   const batchSize = 3;
-  for (let i = 0; i < selectedDocs.length; i += batchSize) {
-    const batch = selectedDocs.slice(i, i + batchSize);
-
-    const savePromises = batch.map(async (doc) => {
-      try {
-        await saveDocument(doc);
-
-        // Record history for each document if feedback is provided
-        if (doc.feedback?.trim()) {
-          recordUpdate({
-            operation: "document_update",
-            feedback: doc.feedback.trim(),
-            documentPath: doc.path,
-          });
-        }
-      } catch (error) {
-        console.error(`❌ Failed to save document ${doc.path}:`, error.message);
-      }
-    });
-
-    await Promise.all(savePromises);
-  }
 
   // Return results if user chose to skip translation
   if (!shouldTranslate) {
@@ -77,30 +51,27 @@ export default async function saveAndTranslateDocument(input, options) {
   // Translate documents in batches
   const translateAgent = options.context.agents["translateMultilingual"];
 
-  for (let i = 0; i < selectedDocs.length; i += batchSize) {
-    const batch = selectedDocs.slice(i, i + batchSize);
-
-    const translatePromises = batch.map(async (doc) => {
+  await pMap(
+    selectedDocs,
+    async (doc) => {
       try {
         // Clear feedback to ensure translation is not affected by update feedback
         doc.feedback = "";
 
-        const result = await options.context.invoke(translateAgent, {
+        await options.context.invoke(translateAgent, {
           ...input, // context is required
           content: doc.content,
           translates: doc.translates,
           title: doc.title,
+          path: doc.path,
+          docsDir,
         });
-
-        // Save the translated content
-        await saveDocument(doc, result.translates, true);
       } catch (error) {
         console.error(`❌ Failed to translate document ${doc.path}:`, error.message);
       }
-    });
-
-    await Promise.all(translatePromises);
-  }
+    },
+    { concurrency: batchSize },
+  );
 
   return {};
 }

package/agents/update/update-document-detail.yaml

@@ -23,7 +23,7 @@ input_schema:
     locale:
       type: string
       description: User language, e.g. zh, en
-    datasources:
+    detailDataSource:
       type: string
       description: Context for document content
     glossary:
@@ -57,4 +57,5 @@ afs:
 keep_text_in_tool_uses: false
 skills:
   - ./document-tools/update-document-content.mjs
+  # - ./generate-diagram.yaml
 task_render_mode: collapse

package/agents/update/update-single-document.yaml

@@ -1,7 +1,7 @@
 type: team
 name: updateSingleDocument
 skills:
-  - ../utils/transform-detail-datasources.mjs
+  - ../utils/transform-detail-data-sources.mjs
   - ../update/user-review-document.mjs
 iterate_on: selectedDocs
 concurrency: 1

package/agents/update/user-review-document.mjs

@@ -105,6 +105,7 @@ async function showDocumentDetail(content, title) {
       renderer: new markedTerminal(),
     });
 
+    // FIXME: @zhanghan fix error "Could not find the language 'd2', did you forget to load/include a language module?"
     const renderedMarkdown = marked(content);
 
     // Restore original console.error
@@ -121,16 +122,15 @@ async function showDocumentDetail(content, title) {
   }
 }
 
-export default async function userReviewDocument(
-  { content, title, description, ...rest },
-  options,
-) {
+export default async function userReviewDocument({ content, description, ...rest }, options) {
   // Check if document content exists
   if (!content || typeof content !== "string" || content.trim().length === 0) {
     console.log("Please provide document content to review.");
     return { content };
   }
 
+  const title = rest.documentStructure?.find((x) => x.path === rest.path)?.title;
+
   // Print current document headings structure
   printDocumentHeadings(content, title || "Untitled Document");
 
@@ -190,7 +190,7 @@ export default async function userReviewDocument(
     feedbacks.push(feedback.trim());
 
     // Get the updateDocument agent
-    const updateAgent = options.context.agents["updateDocumentDetail"];
+    const updateAgent = options.context.agents["handleDocumentUpdate"];
     if (!updateAgent) {
       console.log(
         "We can't process your feedback right now. The document update feature is temporarily unavailable.",
@@ -213,6 +213,7 @@ export default async function userReviewDocument(
         originalContent: options.context.userContext.currentContent,
         feedback: feedback.trim(),
         userPreferences,
+        title,
       });
 
       // Check if feedback should be saved as user preference

package/agents/utils/choose-docs.mjs

@@ -18,6 +18,7 @@ export default async function chooseDocs(
     locale,
     reset = false,
     requiredFeedback = true,
+    title,
   },
   options,
 ) {
@@ -65,7 +66,7 @@ export default async function chooseDocs(
 
       // Let user select multiple files
       selectedFiles = await options.prompts.checkbox({
-        message: getActionText(isTranslate, "Select documents to {action}:"),
+        message: title || getActionText(isTranslate, "Select documents to {action}:"),
         source: (term) => {
           if (!term) return choices;
 

package/agents/utils/load-sources.mjs

@@ -4,11 +4,11 @@ import path from "node:path";
 import imageSize from "image-size";
 import {
   buildSourcesContent,
-  calculateFileStats,
   loadFilesFromPaths,
   readFileContents,
   getMimeType,
-  checkIsRemoteFile,
+  isRemoteFile,
+  calculateTokens,
 } from "../../utils/file-utils.mjs";
 import {
   getCurrentGitHead,
@@ -21,6 +21,7 @@ import {
   DEFAULT_INCLUDE_PATTERNS,
 } from "../../utils/constants/index.mjs";
 import { isOpenAPISpecFile } from "../../utils/openapi/index.mjs";
+import { loadDocumentStructure } from "../../utils/docs-finder-utils.mjs";
 
 export default async function loadSources(
   {
@@ -144,7 +145,7 @@ export default async function loadSources(
     files.map(async (file) => {
       const ext = path.extname(file).toLowerCase();
 
-      if (mediaExtensions.includes(ext) && !checkIsRemoteFile(file)) {
+      if (mediaExtensions.includes(ext) && !isRemoteFile(file)) {
         // This is a media file
         const relativePath = path.relative(docsDir, file);
         const fileName = path.basename(file);
@@ -195,13 +196,10 @@ export default async function loadSources(
   }
 
   // Read all source files using the utility function
-  let sourceFiles = await readFileContents(sourceFilesPaths, process.cwd());
-
-  // Count tokens and lines using utility function
-  const { totalTokens, totalLines } = calculateFileStats(sourceFiles);
-
-  // check if totalTokens is too large
-  const isLargeContext = totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD;
+  let sourceFiles = (await readFileContents(sourceFilesPaths, process.cwd())).map((i) => ({
+    ...i,
+    tokens: calculateTokens(`\n${i.sourceId}\n${i.content}`),
+  }));
 
   // filter OpenAPI doc should after check isLargeContext
   sourceFiles = sourceFiles.filter((file) => {
@@ -214,48 +212,32 @@ export default async function loadSources(
     return !isOpenAPI;
   });
 
-  const httpFileList = [];
+  const totalTokens = sourceFiles.reduce((sum, file) => sum + file.tokens, 0);
+  const totalLines = sourceFiles.reduce(
+    (sum, file) => sum + file.content.split("\n").filter(Boolean).length,
+    0,
+  );
+
+  const dataSources = splitSourcesToChunks(sourceFiles, INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD).map(
+    (i) => ({ dataSourceChunk: buildSourcesContent(i) }),
+  );
+
+  const remoteFileList = [];
 
   sourceFiles.forEach((file) => {
-    if (checkIsRemoteFile(file.sourceId)) {
-      httpFileList.push(file);
+    if (isRemoteFile(file.sourceId)) {
+      remoteFileList.push(file);
     }
   });
   if (options?.context?.userContext) {
-    options.context.userContext.httpFileList = httpFileList;
+    options.context.userContext.remoteFileList = remoteFileList;
   }
 
-  // Build allSources string using utility function
-  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
   // all files path
   const allFilesPaths = sourceFiles.map((x) => `- ${toRelativePath(x.sourceId)}`).join("\n");
 
   // Get the last documentation structure
-  let originalDocumentStructure;
-  if (outputDir) {
-    const documentStructurePath = path.join(outputDir, "structure-plan.json");
-    try {
-      const documentExecutionStructure = await readFile(documentStructurePath, "utf8");
-      if (documentExecutionStructure?.trim()) {
-        try {
-          // Validate that the content looks like JSON before parsing
-          const trimmedContent = documentExecutionStructure.trim();
-          if (trimmedContent.startsWith("{") || trimmedContent.startsWith("[")) {
-            originalDocumentStructure = JSON.parse(documentExecutionStructure);
-          } else {
-            console.warn(`structure-plan.json contains non-JSON content, skipping parse`);
-          }
-        } catch (err) {
-          console.error(`Failed to parse structure-plan.json: ${err.message}`);
-        }
-      }
-    } catch (err) {
-      if (err.code !== "ENOENT") {
-        console.warn(`Error reading structure-plan.json: ${err.message}`);
-      }
-      // The file does not exist or is not readable, originalDocumentStructure remains undefined
-    }
-  }
+  const originalDocumentStructure = await loadDocumentStructure(outputDir);
 
   // Get the last output result of the specified path
   let content;
@@ -308,7 +290,7 @@ export default async function loadSources(
   }
 
   return {
-    datasources: allSources,
+    dataSources,
     content,
     originalDocumentStructure,
     files,
@@ -316,7 +298,6 @@ export default async function loadSources(
     totalTokens,
     totalLines,
     mediaFiles,
-    isLargeContext,
     allFilesPaths,
   };
 }
@@ -364,8 +345,14 @@ loadSources.input_schema = {
 loadSources.output_schema = {
   type: "object",
   properties: {
-    datasources: {
-      type: "string",
+    dataSources: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          dataSourceChunk: { type: "string" },
+        },
+      },
     },
     files: {
       type: "array",
@@ -396,3 +383,33 @@ loadSources.output_schema = {
 };
 
 loadSources.task_render_mode = "hide";
+
+function splitSourcesToChunks(sources, maxTokens) {
+  const chunks = [];
+
+  let currentChunk = [];
+  let currentTokens = 0;
+
+  for (const source of sources) {
+    const sourceTokens = source.tokens;
+
+    if (currentTokens + sourceTokens > maxTokens) {
+      // Start a new chunk
+      if (currentChunk.length > 0) {
+        chunks.push(currentChunk);
+      }
+      currentChunk = [source];
+      currentTokens = sourceTokens;
+    } else {
+      // Add to current chunk
+      currentChunk.push(source);
+      currentTokens += sourceTokens;
+    }
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+
+  return chunks;
+}

package/agents/utils/{save-docs.mjs → post-generate.mjs}

@@ -1,4 +1,4 @@
-import { readdir, unlink, writeFile } from "node:fs/promises";
+import { readdir, unlink } from "node:fs/promises";
 import { join } from "node:path";
 import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
 import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
@@ -10,7 +10,7 @@ import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
  * @param {Array<string>} [params.translateLanguages] - Translation languages
  * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
  */
-export default async function saveDocs({
+export default async function postGenerate({
   documentExecutionStructure: documentStructure,
   docsDir,
   translateLanguages = [],
@@ -26,15 +26,6 @@ export default async function saveDocs({
     console.warn("Failed to save git HEAD:", err.message);
   }
 
-  // Generate _sidebar.md
-  try {
-    const sidebar = generateSidebar(documentStructure);
-    const sidebarPath = join(docsDir, "_sidebar.md");
-    await writeFile(sidebarPath, sidebar, "utf8");
-  } catch (err) {
-    console.error("Failed to save _sidebar.md:", err.message);
-  }
-
   // Clean up invalid .md files that are no longer in the documentation structure
   try {
     await cleanupInvalidFiles(documentStructure, docsDir, translateLanguages, locale);
@@ -140,43 +131,3 @@ async function cleanupInvalidFiles(documentStructure, docsDir, translateLanguage
 }
 
 // Generate sidebar content, support nested structure, and the order is consistent with documentStructure
-function generateSidebar(documentStructure) {
-  // Build tree structure
-  const root = {};
-  for (const { path, title, parentId } of documentStructure) {
-    const relPath = path.replace(/^\//, "");
-    const segments = relPath.split("/");
-    let node = root;
-    for (let i = 0; i < segments.length; i++) {
-      const seg = segments[i];
-      if (!node[seg])
-        node[seg] = {
-          __children: {},
-          __title: null,
-          __fullPath: segments.slice(0, i + 1).join("/"),
-          __parentId: parentId,
-        };
-      if (i === segments.length - 1) node[seg].__title = title;
-      node = node[seg].__children;
-    }
-  }
-  // Recursively generate sidebar text, the link path is the flattened file name
-  function walk(node, parentSegments = [], indent = "") {
-    let out = "";
-    for (const key of Object.keys(node)) {
-      const item = node[key];
-      const fullSegments = [...parentSegments, key];
-      const flatFile = `${fullSegments.join("-")}.md`;
-      if (item.__title) {
-        const realIndent = item.__parentId === null ? "" : indent;
-        out += `${realIndent}* [${item.__title}](/${flatFile})\n`;
-      }
-      const children = item.__children;
-      if (Object.keys(children).length > 0) {
-        out += walk(children, fullSegments, `${indent}  `);
-      }
-    }
-    return out;
-  }
-  return walk(root).replace(/\n+$/, "");
-}