@aigne/doc-smith 0.8.15-beta.1 → 0.8.15-beta.11

package/agents/update/check-document.mjs

@@ -2,7 +2,10 @@ import { access, readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { TeamAgent } from "@aigne/core";
+import fs from "fs-extra";
+import pMap from "p-map";
 
+import { getFileName } from "../../utils/utils.mjs";
 import checkDetailResult from "../utils/check-detail-result.mjs";
 
 // Get current script directory
@@ -18,13 +21,13 @@ export default async function checkDocument(
     modifiedFiles,
     forceRegenerate,
     locale,
+    translates,
     ...rest
   },
   options,
 ) {
   // Check if the detail file already exists
-  const flatName = path.replace(/^\//, "").replace(/\//g, "-");
-  const fileFullName = locale === "en" ? `${flatName}.md` : `${flatName}.${locale}.md`;
+  const fileFullName = getFileName(path, locale);
   const filePath = join(docsDir, fileFullName);
   let detailGenerated = true;
   let fileContent = null;
@@ -84,24 +87,42 @@ export default async function checkDocument(
       contentValidationFailed = true;
     }
   }
+  const languages = translates.map((x) => x.language);
+  const lackLanguages = new Set(languages);
+  const skills = [];
 
   // If file exists, sourceIds haven't changed, source files haven't changed, and content validation passes, no need to regenerate
   if (detailGenerated && !sourceIdsChanged && !contentValidationFailed && !forceRegenerate) {
-    return {
-      path,
-      docsDir,
-      ...rest,
-      detailGenerated: true,
-    };
+    await pMap(
+      languages,
+      async (x) => {
+        const languageFileName = getFileName(path, x);
+        const languageFilePath = join(docsDir, languageFileName);
+        if (await fs.exists(languageFilePath)) {
+          lackLanguages.delete(x);
+        }
+      },
+      { concurrency: 10 },
+    );
+    if (lackLanguages.size === 0) {
+      return {
+        path,
+        docsDir,
+        ...rest,
+        detailGenerated: true,
+      };
+    }
+    // translations during generation don't need feedback, content is satisfactory
+    rest.content = fileContent;
+  } else {
+    skills.push(options.context.agents["handleDocumentUpdate"]);
   }
 
+  skills.push(options.context.agents["translateMultilingual"]);
+
   const teamAgent = TeamAgent.from({
     name: "generateDocument",
-    skills: [
-      options.context.agents["handleDocumentUpdate"],
-      options.context.agents["translateMultilingual"],
-      options.context.agents["saveSingleDoc"],
-    ],
+    skills,
   });
   let openAPISpec = null;
 
@@ -119,6 +140,7 @@ export default async function checkDocument(
 
   const result = await options.context.invoke(teamAgent, {
     ...rest,
+    translates: translates.filter((x) => lackLanguages.has(x.language)),
     locale,
     docsDir,
     path,

package/agents/update/check-generate-diagram.mjs

@@ -0,0 +1,29 @@
+const placeholder = "DIAGRAM_PLACEHOLDER";
+
+export default async function checkGenerateDiagram(
+  { needDiagram, documentContent, locale },
+  options,
+) {
+  if (!needDiagram) {
+    return {};
+  }
+
+  const generateAgent = options.context?.agents?.["generateDiagram"];
+  let content = documentContent;
+
+  if (content.includes(placeholder)) {
+    try {
+      const { diagramSourceCode } = await options.context.invoke(generateAgent, {
+        documentContent,
+        locale,
+      });
+      content = content.replace(placeholder, diagramSourceCode);
+    } catch (error) {
+      // FIXME: @zhanghan should regenerate document without diagram
+      content = content.replace(placeholder, "");
+      console.log(`⚠️  Skip generate any diagram: ${error.message}`);
+    }
+  }
+
+  return { content };
+}

package/agents/update/generate-diagram.yaml

@@ -0,0 +1,29 @@
+type: team
+task_render_mode: collapse
+name: generateDiagram
+skills:
+  - ../generate/draw-diagram.yaml
+  - ../generate/wrap-diagram-code.mjs
+reflection:
+  reviewer: ../generate/check-diagram.mjs
+  is_approved: isValid
+  max_iterations: 5
+  return_last_on_max_iterations: false
+input_schema:
+  type: object
+  properties:
+    documentContent:
+      type: string
+      description: The **raw text content** of the current document. (**Note:** This is the original document and **does not include** any diagram source code.)
+    locale:
+      type: string
+      description: Language for diagram labels and text
+      default: en
+  required:
+    - documentContent
+output_schema:
+  type: object
+  properties:
+    diagramSourceCode:
+      type: string
+      description: The **diagram source code** generated from the input text.

package/agents/update/generate-document.yaml

@@ -16,7 +16,7 @@ input_schema:
     locale:
       type: string
       description: User language, such as zh, en
-    datasources:
+    detailDataSource:
       type: string
       description: Source data and context for document content generation
     targetAudience:
@@ -44,9 +44,21 @@ input_schema:
       description: Additional supplementary information
   required:
     - rules
-    - datasources
+    - detailDataSource
     - originalDocumentStructure
-output_key: content
+# output_key: content
+output_schema:
+  type: object
+  properties:
+    content:
+      type: string
+      description: Document content
+    needDiagram:
+      type: boolean
+      description: Does this document need to generate diagram?
+  required:
+    - content
+    - detailDataSource
 afs:
   modules:
     - module: system-fs
@@ -57,30 +69,5 @@ afs:
           Codebase of the project to be documented used as context for document generation,
           should search and read as needed while generating document content
 keep_text_in_tool_uses: false
-skills:
-  - type: team
-    task_render_mode: collapse
-    name: generateDiagram
-    skills:
-      - ../generate/draw-diagram.yaml
-      - ../generate/wrap-diagram-code.mjs
-    reflection:
-      reviewer: ../generate/check-diagram.mjs
-      is_approved: isValid
-      max_iterations: 5
-      return_last_on_max_iterations: false
-      custom_error_message: "MUST NOT generate any diagram: validation failed after max iterations."
-    input_schema:
-      type: object
-      properties:
-        documentContent:
-          type: string
-          description: The **raw text content** of the current document. (**Note:** This is the original document and **does not include** any diagram source code.)
-      required:
-        - documentContent
-    output_schema:
-      type: object
-      properties:
-        diagramSourceCode:
-          type: string
-          description: The **diagram source code** generated from the input text.
+# skills:
+#   - ./generate-diagram.yaml

package/agents/update/handle-document-update.yaml

@@ -2,7 +2,7 @@ type: team
 name: handleDocumentUpdate
 description: Update a document in a batch
 skills:
-  - ../utils/transform-detail-datasources.mjs
+  - ../utils/transform-detail-data-sources.mjs
   - type: team
     task_render_mode: collapse
     name: generateDocumentContent
@@ -23,6 +23,15 @@ skills:
       max_iterations: 5
       return_last_on_max_iterations: true
     task_title: Generate document for '{{ title }}'
+  - type: transform
+    jsonata: |
+      $merge([
+        $,
+        { "documentContent": content }
+      ])
+  - ./check-generate-diagram.mjs
+  # - ../generate/merge-diagram.yaml
+  - ../utils/save-doc.mjs
 input_schema:
   type: object
   properties:

package/agents/update/save-and-translate-document.mjs

@@ -1,3 +1,4 @@
+import pMap from "p-map";
 import { recordUpdate } from "../../utils/history-utils.mjs";
 
 export default async function saveAndTranslateDocument(input, options) {
@@ -7,20 +8,15 @@ export default async function saveAndTranslateDocument(input, options) {
     return {};
   }
 
-  // Saves a document with optional translation data
-  const saveDocument = async (doc, translates = null, isTranslate = false) => {
-    const saveAgent = options.context.agents["saveSingleDoc"];
-
-    return await options.context.invoke(saveAgent, {
-      path: doc.path,
-      content: doc.content,
-      docsDir: docsDir,
-      locale: locale,
-      translates: translates || doc.translates,
-      labels: doc.labels,
-      isTranslate: isTranslate,
+  // Record history if feedback is provided
+  const doc = selectedDocs[0];
+  if (doc.feedback?.trim()) {
+    recordUpdate({
+      operation: "document_update",
+      feedback: doc.feedback.trim(),
+      docPaths: selectedDocs.map((v) => v.path),
     });
-  };
+  }
 
   // Only prompt user if translation is actually needed
   let shouldTranslate = false;
@@ -46,28 +42,6 @@ export default async function saveAndTranslateDocument(input, options) {
 
   // Save documents in batches
   const batchSize = 3;
-  for (let i = 0; i < selectedDocs.length; i += batchSize) {
-    const batch = selectedDocs.slice(i, i + batchSize);
-
-    const savePromises = batch.map(async (doc) => {
-      try {
-        await saveDocument(doc);
-
-        // Record history for each document if feedback is provided
-        if (doc.feedback?.trim()) {
-          recordUpdate({
-            operation: "document_update",
-            feedback: doc.feedback.trim(),
-            documentPath: doc.path,
-          });
-        }
-      } catch (error) {
-        console.error(`❌ Failed to save document ${doc.path}:`, error.message);
-      }
-    });
-
-    await Promise.all(savePromises);
-  }
 
   // Return results if user chose to skip translation
   if (!shouldTranslate) {
@@ -77,30 +51,27 @@ export default async function saveAndTranslateDocument(input, options) {
   // Translate documents in batches
   const translateAgent = options.context.agents["translateMultilingual"];
 
-  for (let i = 0; i < selectedDocs.length; i += batchSize) {
-    const batch = selectedDocs.slice(i, i + batchSize);
-
-    const translatePromises = batch.map(async (doc) => {
+  await pMap(
+    selectedDocs,
+    async (doc) => {
       try {
         // Clear feedback to ensure translation is not affected by update feedback
         doc.feedback = "";
 
-        const result = await options.context.invoke(translateAgent, {
+        await options.context.invoke(translateAgent, {
           ...input, // context is required
           content: doc.content,
           translates: doc.translates,
           title: doc.title,
+          path: doc.path,
+          docsDir,
         });
-
-        // Save the translated content
-        await saveDocument(doc, result.translates, true);
       } catch (error) {
         console.error(`❌ Failed to translate document ${doc.path}:`, error.message);
       }
-    });
-
-    await Promise.all(translatePromises);
-  }
+    },
+    { concurrency: batchSize },
+  );
 
   return {};
 }

package/agents/update/update-document-detail.yaml

@@ -23,7 +23,7 @@ input_schema:
     locale:
       type: string
       description: User language, e.g. zh, en
-    datasources:
+    detailDataSource:
       type: string
       description: Context for document content
     glossary:
@@ -57,4 +57,5 @@ afs:
 keep_text_in_tool_uses: false
 skills:
   - ./document-tools/update-document-content.mjs
+  # - ./generate-diagram.yaml
 task_render_mode: collapse

package/agents/update/update-single-document.yaml

@@ -1,7 +1,7 @@
 type: team
 name: updateSingleDocument
 skills:
-  - ../utils/transform-detail-datasources.mjs
+  - ../utils/transform-detail-data-sources.mjs
   - ../update/user-review-document.mjs
 iterate_on: selectedDocs
 concurrency: 1

package/agents/update/user-review-document.mjs

@@ -105,6 +105,7 @@ async function showDocumentDetail(content, title) {
       renderer: new markedTerminal(),
     });
 
+    // FIXME: @zhanghan fix error "Could not find the language 'd2', did you forget to load/include a language module?"
     const renderedMarkdown = marked(content);
 
     // Restore original console.error
@@ -121,16 +122,15 @@ async function showDocumentDetail(content, title) {
   }
 }
 
-export default async function userReviewDocument(
-  { content, title, description, ...rest },
-  options,
-) {
+export default async function userReviewDocument({ content, description, ...rest }, options) {
   // Check if document content exists
   if (!content || typeof content !== "string" || content.trim().length === 0) {
     console.log("Please provide document content to review.");
     return { content };
   }
 
+  const title = rest.documentStructure?.find((x) => x.path === rest.path)?.title;
+
   // Print current document headings structure
   printDocumentHeadings(content, title || "Untitled Document");
 
@@ -190,7 +190,7 @@ export default async function userReviewDocument(
     feedbacks.push(feedback.trim());
 
     // Get the updateDocument agent
-    const updateAgent = options.context.agents["updateDocumentDetail"];
+    const updateAgent = options.context.agents["handleDocumentUpdate"];
     if (!updateAgent) {
       console.log(
         "We can't process your feedback right now. The document update feature is temporarily unavailable.",
@@ -213,6 +213,7 @@ export default async function userReviewDocument(
         originalContent: options.context.userContext.currentContent,
         feedback: feedback.trim(),
         userPreferences,
+        title,
       });
 
       // Check if feedback should be saved as user preference

package/agents/utils/choose-docs.mjs

@@ -6,6 +6,14 @@ import {
   getMainLanguageFiles,
   processSelectedFiles,
 } from "../../utils/docs-finder-utils.mjs";
+import { DOC_ACTION } from "../../utils/constants/index.mjs";
+
+function getFeedbackMessage(action) {
+  if (action === DOC_ACTION.translate) {
+    return "Any specific translation preferences or instructions? (Press Enter to skip):";
+  }
+  return "How would you like to improve this document? (Press Enter to skip)";
+}
 
 export default async function chooseDocs(
   {
@@ -18,11 +26,13 @@ export default async function chooseDocs(
     locale,
     reset = false,
     requiredFeedback = true,
+    action,
   },
   options,
 ) {
   let foundItems = [];
   let selectedFiles = [];
+  const docAction = action || (isTranslate ? DOC_ACTION.translate : DOC_ACTION.update);
 
   // If docs is empty or not provided, let user select multiple documents
   if (!docs || docs.length === 0) {
@@ -36,7 +46,9 @@ export default async function chooseDocs(
 
       if (mainLanguageFiles.length === 0) {
         throw new Error(
-          `No documents found in the docs directory. Please run ${chalk.yellow("`aigne doc generate`")} to generate the documents`,
+          `No documents found in the docs directory. You can generate them with ${chalk.yellow(
+            "`aigne doc generate`",
+          )}`,
         );
       }
 
@@ -65,7 +77,7 @@ export default async function chooseDocs(
 
       // Let user select multiple files
       selectedFiles = await options.prompts.checkbox({
-        message: getActionText(isTranslate, "Select documents to {action}:"),
+        message: getActionText("Select documents to {action}:", docAction),
         source: (term) => {
           if (!term) return choices;
 
@@ -87,7 +99,7 @@ export default async function chooseDocs(
       foundItems = await processSelectedFiles(selectedFiles, documentExecutionStructure, docsDir);
     } catch (error) {
       console.log(
-        getActionText(isTranslate, `\nFailed to select documents to {action}: ${error.message}`),
+        getActionText(`\nFailed to select documents to {action}: ${error.message}`, docAction),
       );
       process.exit(0);
     }
@@ -122,8 +134,7 @@ export default async function chooseDocs(
   // Prompt for feedback if not provided
   let userFeedback = feedback;
   if (!userFeedback && (requiredFeedback || foundItems?.length > 1)) {
-    const feedbackMessage =
-      "How should we improve this document? (Enter to skip, will auto-update from content sources):";
+    const feedbackMessage = getFeedbackMessage(docAction);
 
     userFeedback = await options.prompts.input({
       message: feedbackMessage,

package/agents/utils/find-item-by-path.mjs

@@ -1,3 +1,4 @@
+import { DOC_ACTION } from "../../utils/constants/index.mjs";
 import {
   fileNameToFlatPath,
   findItemByFlatName,
@@ -14,6 +15,7 @@ export default async function findItemByPath(
   let foundItem = null;
   let selectedFileContent = null;
   let docPath = doc;
+  const docAction = isTranslate ? DOC_ACTION.translate : DOC_ACTION.update;
 
   // If docPath is empty, let user select from available documents
   if (!docPath) {
@@ -31,7 +33,7 @@ export default async function findItemByPath(
 
       // Let user select a file
       const selectedFile = await options.prompts.search({
-        message: getActionText(isTranslate, "Select a document to {action}:"),
+        message: getActionText("Select a document to {action}:", docAction),
         source: async (input) => {
           if (!input || input.trim() === "") {
             return mainLanguageFiles.map((file) => ({
@@ -74,8 +76,8 @@ export default async function findItemByPath(
       console.debug(error?.message);
       throw new Error(
         getActionText(
-          isTranslate,
           "Please run 'aigne doc generate' first to generate documents, then select which document to {action}",
+          docAction,
         ),
       );
     }