@aigne/doc-smith 0.8.15-beta.10 → 0.8.15-beta.11

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [0.8.15-beta.11](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta.10...v0.8.15-beta.11) (2025-11-04)
+
+
+### Features
+
+* warn on invalid sources and tuning translate messages ([#241](https://github.com/AIGNE-io/aigne-doc-smith/issues/241)) ([bd786ca](https://github.com/AIGNE-io/aigne-doc-smith/commit/bd786cad8b0c6fa837511fdc2982c83b7f0095dd))
+
+
+### Bug Fixes
+
+* avoid useless diagram generate ([#245](https://github.com/AIGNE-io/aigne-doc-smith/issues/245)) ([de7600f](https://github.com/AIGNE-io/aigne-doc-smith/commit/de7600fb5839be9d1f0743adc34fd08c4c11729d))
+* iterate on chunk of datasource to generate structure ([#242](https://github.com/AIGNE-io/aigne-doc-smith/issues/242)) ([0b4db2a](https://github.com/AIGNE-io/aigne-doc-smith/commit/0b4db2a8cac77d6f4a1c197374b9dadbe078346b))
+
 ## [0.8.15-beta.10](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta.9...v0.8.15-beta.10) (2025-11-03)
 
 

package/agents/generate/user-review-document-structure.mjs

@@ -1,32 +1,9 @@
 import { getActiveRulesForScope } from "../../utils/preferences-utils.mjs";
 import { recordUpdate } from "../../utils/history-utils.mjs";
+import { buildDocumentTree } from "../../utils/docs-finder-utils.mjs";
 
 function formatDocumentStructure(structure) {
-  // Create a map of nodes for easy lookup
-  const nodeMap = new Map();
-  const rootNodes = [];
-
-  // First pass: create node map
-  structure.forEach((node) => {
-    nodeMap.set(node.path, {
-      ...node,
-      children: [],
-    });
-  });
-
-  // Build the tree structure
-  structure.forEach((node) => {
-    if (node.parentId) {
-      const parent = nodeMap.get(node.parentId);
-      if (parent) {
-        parent.children.push(nodeMap.get(node.path));
-      } else {
-        rootNodes.push(nodeMap.get(node.path));
-      }
-    } else {
-      rootNodes.push(nodeMap.get(node.path));
-    }
-  });
+  const { rootNodes } = buildDocumentTree(structure);
 
   function printNode(node, depth = 0) {
     const INDENT_SPACES = "  ";

package/agents/generate/utils/merge-document-structures.mjs

@@ -11,13 +11,13 @@ export default async function mergeDocumentStructures(input, options) {
 
   options.context.userContext.originalDocumentStructure ??= [];
 
-  const originalStructures = options.context.userContext.originalDocumentStructure;
+  const originalStructures = [...options.context.userContext.originalDocumentStructure];
 
   if (input.structures) {
     for (const item of input.structures) {
-      const old = originalStructures.find((s) => s.path === item.path);
-      if (old) {
-        Object.assign(old, item);
+      const index = originalStructures.findIndex((s) => s.path === item.path);
+      if (index !== -1) {
+        originalStructures[index] = item;
       } else {
         originalStructures.push(item);
       }

package/agents/init/check.mjs

@@ -6,7 +6,7 @@ export default async function checkNeedGenerate({ docsDir, locale, documentExecu
 
   if (mainLanguageFiles.length === 0) {
     console.log(
-      `No documents found in the docs directory. Please run ${chalk.yellow("`aigne doc generate`")} to generate the documents`,
+      `No documents found in the docs directory. You can generate them with ${chalk.yellow("`aigne doc generate`")}`,
     );
     process.exit(0);
   }

package/agents/update/check-generate-diagram.mjs

@@ -11,16 +11,19 @@ export default async function checkGenerateDiagram(
   const generateAgent = options.context?.agents?.["generateDiagram"];
   let content = documentContent;
 
-  try {
-    const { diagramSourceCode } = await options.context.invoke(generateAgent, {
-      documentContent,
-      locale,
-    });
-    content = content.replace(placeholder, diagramSourceCode);
-  } catch (error) {
-    // FIXME: @zhanghan should regenerate document without diagram
-    content = content.replace(placeholder, "");
-    console.log(`⚠️  Skip generate any diagram: ${error.message}`);
+  if (content.includes(placeholder)) {
+    try {
+      const { diagramSourceCode } = await options.context.invoke(generateAgent, {
+        documentContent,
+        locale,
+      });
+      content = content.replace(placeholder, diagramSourceCode);
+    } catch (error) {
+      // FIXME: @zhanghan should regenerate document without diagram
+      content = content.replace(placeholder, "");
+      console.log(`⚠️  Skip generate any diagram: ${error.message}`);
+    }
   }
+
   return { content };
 }

package/agents/utils/choose-docs.mjs

@@ -6,6 +6,14 @@ import {
   getMainLanguageFiles,
   processSelectedFiles,
 } from "../../utils/docs-finder-utils.mjs";
+import { DOC_ACTION } from "../../utils/constants/index.mjs";
+
+function getFeedbackMessage(action) {
+  if (action === DOC_ACTION.translate) {
+    return "Any specific translation preferences or instructions? (Press Enter to skip):";
+  }
+  return "How would you like to improve this document? (Press Enter to skip)";
+}
 
 export default async function chooseDocs(
   {
@@ -18,12 +26,13 @@ export default async function chooseDocs(
     locale,
     reset = false,
     requiredFeedback = true,
-    title,
+    action,
   },
   options,
 ) {
   let foundItems = [];
   let selectedFiles = [];
+  const docAction = action || (isTranslate ? DOC_ACTION.translate : DOC_ACTION.update);
 
   // If docs is empty or not provided, let user select multiple documents
   if (!docs || docs.length === 0) {
@@ -37,7 +46,9 @@ export default async function chooseDocs(
 
       if (mainLanguageFiles.length === 0) {
         throw new Error(
-          `No documents found in the docs directory. Please run ${chalk.yellow("`aigne doc generate`")} to generate the documents`,
+          `No documents found in the docs directory. You can generate them with ${chalk.yellow(
+            "`aigne doc generate`",
+          )}`,
         );
       }
 
@@ -66,7 +77,7 @@ export default async function chooseDocs(
 
       // Let user select multiple files
       selectedFiles = await options.prompts.checkbox({
-        message: title || getActionText(isTranslate, "Select documents to {action}:"),
+        message: getActionText("Select documents to {action}:", docAction),
         source: (term) => {
           if (!term) return choices;
 
@@ -88,7 +99,7 @@ export default async function chooseDocs(
       foundItems = await processSelectedFiles(selectedFiles, documentExecutionStructure, docsDir);
     } catch (error) {
       console.log(
-        getActionText(isTranslate, `\nFailed to select documents to {action}: ${error.message}`),
+        getActionText(`\nFailed to select documents to {action}: ${error.message}`, docAction),
       );
       process.exit(0);
     }
@@ -123,8 +134,7 @@ export default async function chooseDocs(
   // Prompt for feedback if not provided
   let userFeedback = feedback;
   if (!userFeedback && (requiredFeedback || foundItems?.length > 1)) {
-    const feedbackMessage =
-      "How should we improve this document? (Enter to skip, will auto-update from content sources):";
+    const feedbackMessage = getFeedbackMessage(docAction);
 
     userFeedback = await options.prompts.input({
       message: feedbackMessage,

package/agents/utils/find-item-by-path.mjs

@@ -1,3 +1,4 @@
+import { DOC_ACTION } from "../../utils/constants/index.mjs";
 import {
   fileNameToFlatPath,
   findItemByFlatName,
@@ -14,6 +15,7 @@ export default async function findItemByPath(
   let foundItem = null;
   let selectedFileContent = null;
   let docPath = doc;
+  const docAction = isTranslate ? DOC_ACTION.translate : DOC_ACTION.update;
 
   // If docPath is empty, let user select from available documents
   if (!docPath) {
@@ -31,7 +33,7 @@ export default async function findItemByPath(
 
       // Let user select a file
       const selectedFile = await options.prompts.search({
-        message: getActionText(isTranslate, "Select a document to {action}:"),
+        message: getActionText("Select a document to {action}:", docAction),
         source: async (input) => {
           if (!input || input.trim() === "") {
             return mainLanguageFiles.map((file) => ({
@@ -74,8 +76,8 @@ export default async function findItemByPath(
       console.debug(error?.message);
       throw new Error(
         getActionText(
-          isTranslate,
           "Please run 'aigne doc generate' first to generate documents, then select which document to {action}",
+          docAction,
         ),
       );
     }

package/agents/utils/load-sources.mjs

@@ -171,7 +171,7 @@ export default async function loadSources(
             if (dimensions.width < minImageWidth) {
               filteredImageCount++;
               console.log(
-                `Filtered image: ${fileName} (${dimensions.width}x${dimensions.height}px < ${minImageWidth}px minimum)`,
+                `Ignored image: ${fileName} (${dimensions.width}x${dimensions.height}px < ${minImageWidth}px minimum)`,
               );
               return;
             }

package/agents/utils/save-sidebar.mjs

@@ -1,5 +1,6 @@
 import { join } from "node:path";
 import fs from "fs-extra";
+import { buildDocumentTree } from "../../utils/docs-finder-utils.mjs";
 
 export default async function saveSidebar({ documentStructure, docsDir }) {
   // Generate _sidebar.md
@@ -16,44 +17,22 @@ export default async function saveSidebar({ documentStructure, docsDir }) {
 }
 
 // Recursively generate sidebar text, the link path is the flattened file name
-function walk(node, parentSegments = [], indent = "") {
+function walk(nodes, indent = "") {
   let out = "";
-  for (const key of Object.keys(node)) {
-    const item = node[key];
-    const fullSegments = [...parentSegments, key];
-    const flatFile = `${fullSegments.join("-")}.md`;
-    if (item.__title) {
-      const realIndent = item.__parentId === null ? "" : indent;
-      out += `${realIndent}* [${item.__title}](/${flatFile})\n`;
-    }
-    const children = item.__children;
-    if (Object.keys(children).length > 0) {
-      out += walk(children, fullSegments, `${indent}  `);
+  for (const node of nodes) {
+    const relPath = node.path.replace(/^\//, "");
+    const flatFile = `${relPath.split("/").join("-")}.md`;
+    const realIndent = node.parentId === null ? "" : indent;
+    out += `${realIndent}* [${node.title}](/${flatFile})\n`;
+
+    if (node.children && node.children.length > 0) {
+      out += walk(node.children, `${indent}  `);
     }
   }
   return out;
 }
 
 function generateSidebar(documentStructure) {
-  // Build tree structure
-  const root = {};
-  for (const { path, title, parentId } of documentStructure) {
-    const relPath = path.replace(/^\//, "");
-    const segments = relPath.split("/");
-    let node = root;
-    for (let i = 0; i < segments.length; i++) {
-      const seg = segments[i];
-      if (!node[seg])
-        node[seg] = {
-          __children: {},
-          __title: null,
-          __fullPath: segments.slice(0, i + 1).join("/"),
-          __parentId: parentId,
-        };
-      if (i === segments.length - 1) node[seg].__title = title;
-      node = node[seg].__children;
-    }
-  }
-
-  return walk(root).replace(/\n+$/, "");
+  const { rootNodes } = buildDocumentTree(documentStructure);
+  return walk(rootNodes).replace(/\n+$/, "");
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/doc-smith",
-  "version": "0.8.15-beta.10",
+  "version": "0.8.15-beta.11",
   "description": "AI-driven documentation generation tool built on the AIGNE Framework",
   "publishConfig": {
     "access": "public"
@@ -48,6 +48,7 @@
     "marked": "^15.0.12",
     "marked-terminal": "^7.3.0",
     "mermaid": "^11.9.0",
+    "minimatch": "^10.1.1",
     "open": "^10.2.0",
     "p-limit": "^7.1.1",
     "p-map": "^7.0.3",

package/prompts/structure/generate/user-prompt.md

@@ -1,7 +1,7 @@
 <data_sources>
 Following are the partial or complete data sources provided by the user to help you design the document structure. Use these data sources to inform your structural planning.
 
-{{ dataSources }}
+{{ dataSourceChunk }}
 
 
 NOTICE: There are additional data source contents not displayed. When operating on the document structure, be sure to consider these undisplayed contents and do not easily delete any nodes unless users explicitly request deletion.
@@ -41,17 +41,22 @@ NOTICE: There are additional data source contents not displayed. When operating
 </openapi>
 {% endif %}
 
-
-<last_document_structure>
+<document_info>
 projectName: |
   {{projectName}}
+{% if projectDesc %}
 projectDesc: |
   {{projectDesc}}
+{% endif %}
+</document_info>
+
+<last_document_structure>
 
 {% if originalDocumentStructure %}
 {{ originalDocumentStructure | yaml.stringify }}
 {% elseif userContext.originalDocumentStructure %}
 {{ userContext.originalDocumentStructure | yaml.stringify }}
+{% else %}
 No previous document structure provided. generate a new structure based on the data sources!
 {% endif %}
 

package/prompts/structure/review/structure-review-system.md

@@ -5,8 +5,10 @@ You are a **Documentation Structure Refiner** with the analytical mindset of an
 <document_info>
 projectName: |
   {{projectName}}
+{% if projectDesc %}
 projectDesc: |
   {{projectDesc}}
+{% endif %}
 </document_info>
 
 <document_structure>

package/prompts/structure/update/user-prompt.md

@@ -7,7 +7,7 @@
 </file_list>
 
 <data_sources>
-{{ dataSources }}
+{{ dataSourceChunk }}
 </data_sources>
 
 

package/prompts/translate/translate-document.md

@@ -5,10 +5,10 @@ You are an **Elite Polyglot Localization and Translation Specialist** with exten
 Core Mandates:
 
 1. Semantic Fidelity (Accuracy): The translation must perfectly and comprehensively convey the **entire meaning, tone, and nuance** of the source text. **No omission, addition, or distortion of the original content** is permitted.
-2. Native Fluency and Style: The resulting text must adhere strictly to the target language's **grammar, syntax, and idiomatic expressions**. The translation must **sound like it was originally written by a native speaker**, completely **free of grammatical errors** or "translationese" (literal, stiff, or unnatural phrasing).
+2. Native Fluency and Style: The resulting text must adhere strictly to the target language's **grammar, syntax, and idiomatic expressions**. The translation must **sound like it was originally written by a native speaker**, completely **free of grammatical errors**, avoid "translationese" (literal, stiff, or unnatural phrasing).
 3. Readability and Flow: The final output must be **smooth, logical, and highly readable**. Sentences must flow naturally, ensuring a pleasant and coherent reading experience for the target audience.
 4. Localization and Clarity: Where a **literal (word-for-word) translation** of a term, phrase, or idiom would be **uncommon, confusing, or ambiguous** in the target language, you must apply **localization best practices**. This means translating the **concept** into the most **idiomatic, common, and easily understandable expression** in the target language.
-5. Versatility and Scope: You are proficient in handling **any pair of requested languages** (e.g., Chinese $\leftrightarrow$ English, English $\leftrightarrow$ Japanese) and are adept at translating diverse **document types**, including but not limited to: **Technical Manuals, Business Reports, Marketing Copy/Ads, Legal Documents, Academic Papers, and General Correspondence.**
+5. Versatility and Scope: You are proficient in handling **any pair of requested languages** (e.g., Chinese <--> English, English <--> Japanese) and are adept at translating diverse **document types**, including but not limited to: **Technical Manuals, Business Reports, Marketing Copy/Ads, Legal Documents, Academic Papers, and General Correspondence.**
 
 </role_and_goal>
 

package/utils/constants/index.mjs

@@ -549,3 +549,9 @@ export const DOC_SMITH_DIR = ".aigne/doc-smith";
 export const TMP_DIR = ".tmp";
 export const TMP_DOCS_DIR = "docs";
 export const TMP_ASSETS_DIR = "assets";
+
+export const DOC_ACTION = {
+  translate: "translate",
+  update: "update",
+  clear: "clear",
+};

package/utils/docs-finder-utils.mjs

@@ -4,12 +4,11 @@ import { pathExists } from "./file-utils.mjs";
 
 /**
  * Get action-specific text based on isTranslate flag
- * @param {boolean} isTranslate - Whether this is a translation action
  * @param {string} baseText - Base text template with {action} placeholder
+ * @param {string} action - doc action type
  * @returns {string} Text with action replaced
  */
-export function getActionText(isTranslate, baseText) {
-  const action = isTranslate ? "translate" : "update";
+export function getActionText(baseText, action) {
   return baseText.replace("{action}", action);
 }
 
@@ -324,3 +323,38 @@ export async function loadDocumentStructure(outputDir) {
     return null;
   }
 }
+
+/**
+ * Build a tree structure from a flat document structure array using parentId
+ * @param {Array} documentStructure - Flat array of document structure items with path and parentId
+ * @returns {Object} Object containing rootNodes (array of root nodes) and nodeMap (Map for lookups)
+ */
+export function buildDocumentTree(documentStructure) {
+  // Create a map of nodes for easy lookup
+  const nodeMap = new Map();
+  const rootNodes = [];
+
+  // First pass: create node map
+  documentStructure.forEach((node) => {
+    nodeMap.set(node.path, {
+      ...node,
+      children: [],
+    });
+  });
+
+  // Build the tree structure using parentId
+  documentStructure.forEach((node) => {
+    if (node.parentId) {
+      const parent = nodeMap.get(node.parentId);
+      if (parent) {
+        parent.children.push(nodeMap.get(node.path));
+      } else {
+        rootNodes.push(nodeMap.get(node.path));
+      }
+    } else {
+      rootNodes.push(nodeMap.get(node.path));
+    }
+  });
+
+  return { rootNodes, nodeMap };
+}

package/utils/file-utils.mjs

@@ -13,6 +13,7 @@ import { debug } from "./debug.mjs";
 import { isGlobPattern } from "./utils.mjs";
 import { uploadFiles } from "./upload-files.mjs";
 import { extractApi } from "./extract-api.mjs";
+import { minimatch } from "minimatch";
 
 /**
  * Check if a directory is inside a git repository using git command
@@ -859,3 +860,99 @@ export async function downloadAndUploadImage(imageUrl, docsDir, appUrl, accessTo
     return { url: imageUrl, downloadFinalPath: null };
   }
 }
+
+/**
+ * Extract the path prefix from a glob pattern until the first glob character
+ */
+export function getPathPrefix(pattern) {
+  const segments = pattern.split("/");
+  const result = [];
+
+  for (const segment of segments) {
+    if (isGlobPattern(segment)) {
+      break;
+    }
+    result.push(segment);
+  }
+
+  return result.join("/") || ".";
+}
+
+/**
+ * Check if a dir matches any exclude pattern
+ */
+export function isDirExcluded(dir, excludePatterns) {
+  if (!dir || typeof dir !== "string") {
+    return false;
+  }
+
+  let normalizedDir = dir.replace(/\\/g, "/").replace(/^\.\/+/, "");
+  normalizedDir = normalizedDir.endsWith("/") ? normalizedDir : `${normalizedDir}/`;
+
+  for (const excludePattern of excludePatterns) {
+    if (minimatch(normalizedDir, excludePattern, { dot: true })) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Return source paths that would be excluded by exclude patterns (files are skipped, directories use minimatch, glob patterns use path prefix heuristic)
+ */
+export async function findInvalidSourcePaths(sourcePaths, excludePatterns) {
+  if (!Array.isArray(sourcePaths) || sourcePaths.length === 0) {
+    return [];
+  }
+
+  if (!Array.isArray(excludePatterns) || excludePatterns.length === 0) {
+    return [];
+  }
+
+  const invalidPaths = [];
+
+  for (const sourcePath of sourcePaths) {
+    if (typeof sourcePath !== "string" || !sourcePath) {
+      continue;
+    }
+
+    // Skip paths starting with "!" (exclusion patterns)
+    if (sourcePath.startsWith("!")) {
+      continue;
+    }
+
+    // Skip remote URLs
+    if (isRemoteFile(sourcePath)) {
+      continue;
+    }
+
+    // Check glob pattern: use heuristic algorithm
+    if (isGlobPattern(sourcePath)) {
+      const representativePath = getPathPrefix(sourcePath);
+      if (isDirExcluded(representativePath, excludePatterns)) {
+        invalidPaths.push(sourcePath);
+      }
+      continue;
+    }
+
+    try {
+      const stats = await stat(sourcePath);
+      // Skip file
+      if (stats.isFile()) {
+        continue;
+      }
+      // Check dir with minimatch
+      if (stats.isDirectory()) {
+        if (isDirExcluded(sourcePath, excludePatterns)) {
+          invalidPaths.push(sourcePath);
+        }
+      }
+    } catch {
+      // Path doesn't exist
+      invalidPaths.push(sourcePath);
+    }
+  }
+
+  return invalidPaths;
+}

package/utils/load-config.mjs

@@ -1,7 +1,10 @@
 import fs from "node:fs/promises";
+import chalk from "chalk";
 import path from "node:path";
 import { parse } from "yaml";
 import { processConfigFields, resolveFileReferences } from "./utils.mjs";
+import { DEFAULT_EXCLUDE_PATTERNS } from "./constants/index.mjs";
+import { findInvalidSourcePaths, toDisplayPath } from "./file-utils.mjs";
 
 export default async function loadConfig({ config, appUrl }) {
   const configPath = path.isAbsolute(config) ? config : path.join(process.cwd(), config);
@@ -30,6 +33,22 @@ export default async function loadConfig({ config, appUrl }) {
     // Parse new configuration fields and convert keys to actual content
     const processedConfig = await processConfigFields(parsedConfig);
 
+    // Validate sourcePaths against exclude patterns
+    const sourcesPath = processedConfig.sourcesPath || parsedConfig.sourcesPath;
+    if (sourcesPath) {
+      const excludePatterns = [
+        ...DEFAULT_EXCLUDE_PATTERNS,
+        ...(processedConfig.excludePatterns || parsedConfig.excludePatterns || []),
+      ];
+
+      const invalidPaths = await findInvalidSourcePaths(sourcesPath, excludePatterns);
+      if (invalidPaths.length > 0) {
+        console.warn(
+          `⚠️  Some source paths have been excluded and will not be processed:\n${invalidPaths.map((p) => `  - ${chalk.yellow(p)}`).join("\n")}\n💡 Tip: You can remove these paths in ${toDisplayPath(configPath)}\n`,
+        );
+      }
+    }
+
     return {
       lastGitHead: parsedConfig.lastGitHead || "",
       ...parsedConfig,