@aigne/doc-smith 0.8.15-beta → 0.8.15-beta.10

package/agents/utils/load-sources.mjs

@@ -1,12 +1,14 @@
 import { readFile } from "node:fs/promises";
+import { statSync } from "node:fs";
 import path from "node:path";
 import imageSize from "image-size";
 import {
   buildSourcesContent,
-  calculateFileStats,
   loadFilesFromPaths,
   readFileContents,
   getMimeType,
+  isRemoteFile,
+  calculateTokens,
 } from "../../utils/file-utils.mjs";
 import {
   getCurrentGitHead,
@@ -18,28 +20,64 @@ import {
   DEFAULT_EXCLUDE_PATTERNS,
   DEFAULT_INCLUDE_PATTERNS,
 } from "../../utils/constants/index.mjs";
-
-export default async function loadSources({
-  sources = [],
-  sourcesPath = [],
-  includePatterns,
-  excludePatterns,
-  outputDir,
-  docsDir,
-  "doc-path": docPath,
-  boardId,
-  useDefaultPatterns = true,
-  lastGitHead,
-  reset = false,
-  media,
-} = {}) {
+import { isOpenAPISpecFile } from "../../utils/openapi/index.mjs";
+import { loadDocumentStructure } from "../../utils/docs-finder-utils.mjs";
+
+export default async function loadSources(
+  {
+    sources = [],
+    sourcesPath = [],
+    includePatterns,
+    excludePatterns,
+    outputDir,
+    docsDir,
+    "doc-path": docPath,
+    boardId,
+    useDefaultPatterns = true,
+    lastGitHead,
+    reset = false,
+    media,
+  } = {},
+  options,
+) {
   let files = Array.isArray(sources) ? [...sources] : [];
   const { minImageWidth } = media || { minImageWidth: 800 };
 
   if (sourcesPath) {
-    const allFiles = await loadFilesFromPaths(sourcesPath, {
+    const sourcesPathList = Array.isArray(sourcesPath) ? sourcesPath : [sourcesPath];
+    const pickSourcesPath = [];
+    const omitSourcesPath = [];
+    sourcesPathList.forEach((x) => {
+      if (typeof x !== "string" || !x) {
+        return;
+      }
+      if (x.startsWith("!")) {
+        omitSourcesPath.push(x.substring(1));
+      } else {
+        pickSourcesPath.push(x);
+      }
+    });
+
+    const customExcludePatterns = omitSourcesPath
+      .map((x) => {
+        try {
+          const stats = statSync(x);
+          if (stats.isFile()) {
+            return x;
+          }
+          if (stats.isDirectory()) {
+            return `${x}/**`;
+          }
+          return null;
+        } catch (error) {
+          console.warn(`Failed to stat path ${x}: ${error.message}`);
+          return null;
+        }
+      })
+      .filter(Boolean);
+    const allFiles = await loadFilesFromPaths(pickSourcesPath, {
       includePatterns,
-      excludePatterns,
+      excludePatterns: [...new Set([...(excludePatterns || []), ...customExcludePatterns])],
       useDefaultPatterns,
       defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
       defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
@@ -50,9 +88,6 @@ export default async function loadSources({
 
   files = [...new Set(files)];
 
-  // all files path
-  const allFilesPaths = files.map((file) => `- ${toRelativePath(file)}`).join("\n");
-
   // Define media file extensions
   const mediaExtensions = [
     ".jpg",
@@ -110,7 +145,7 @@ export default async function loadSources({
     files.map(async (file) => {
       const ext = path.extname(file).toLowerCase();
 
-      if (mediaExtensions.includes(ext)) {
+      if (mediaExtensions.includes(ext) && !isRemoteFile(file)) {
         // This is a media file
         const relativePath = path.relative(docsDir, file);
         const fileName = path.basename(file);
@@ -161,44 +196,49 @@ export default async function loadSources({
   }
 
   // Read all source files using the utility function
-  const sourceFiles = await readFileContents(sourceFilesPaths, process.cwd());
+  let sourceFiles = (await readFileContents(sourceFilesPaths, process.cwd())).map((i) => ({
+    ...i,
+    tokens: calculateTokens(`\n${i.sourceId}\n${i.content}`),
+  }));
+
+  // filter OpenAPI doc should after check isLargeContext
+  sourceFiles = sourceFiles.filter((file) => {
+    if (options?.context?.userContext.openAPISpec) return true;
+
+    const isOpenAPI = isOpenAPISpecFile(file.content);
+    if (isOpenAPI && options?.context?.userContext) {
+      options.context.userContext.openAPISpec = file;
+    }
+    return !isOpenAPI;
+  });
 
-  // Count tokens and lines using utility function
-  const { totalTokens, totalLines } = calculateFileStats(sourceFiles);
+  const totalTokens = sourceFiles.reduce((sum, file) => sum + file.tokens, 0);
+  const totalLines = sourceFiles.reduce(
+    (sum, file) => sum + file.content.split("\n").filter(Boolean).length,
+    0,
+  );
 
-  // check if totalTokens is too large
-  const isLargeContext = totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD;
+  const dataSources = splitSourcesToChunks(sourceFiles, INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD).map(
+    (i) => ({ dataSourceChunk: buildSourcesContent(i) }),
+  );
 
-  // Build allSources string using utility function
-  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
+  const remoteFileList = [];
 
-  // Get the last documentation structure
-  let originalDocumentStructure;
-  if (outputDir) {
-    const documentStructurePath = path.join(outputDir, "structure-plan.json");
-    try {
-      const documentExecutionStructure = await readFile(documentStructurePath, "utf8");
-      if (documentExecutionStructure?.trim()) {
-        try {
-          // Validate that the content looks like JSON before parsing
-          const trimmedContent = documentExecutionStructure.trim();
-          if (trimmedContent.startsWith("{") || trimmedContent.startsWith("[")) {
-            originalDocumentStructure = JSON.parse(documentExecutionStructure);
-          } else {
-            console.warn(`structure-plan.json contains non-JSON content, skipping parse`);
-          }
-        } catch (err) {
-          console.error(`Failed to parse structure-plan.json: ${err.message}`);
-        }
-      }
-    } catch (err) {
-      if (err.code !== "ENOENT") {
-        console.warn(`Error reading structure-plan.json: ${err.message}`);
-      }
-      // The file does not exist or is not readable, originalDocumentStructure remains undefined
+  sourceFiles.forEach((file) => {
+    if (isRemoteFile(file.sourceId)) {
+      remoteFileList.push(file);
     }
+  });
+  if (options?.context?.userContext) {
+    options.context.userContext.remoteFileList = remoteFileList;
   }
 
+  // all files path
+  const allFilesPaths = sourceFiles.map((x) => `- ${toRelativePath(x.sourceId)}`).join("\n");
+
+  // Get the last documentation structure
+  const originalDocumentStructure = await loadDocumentStructure(outputDir);
+
   // Get the last output result of the specified path
   let content;
   if (docPath && !reset && docsDir) {
@@ -250,7 +290,7 @@ export default async function loadSources({
   }
 
   return {
-    datasources: allSources,
+    dataSources,
     content,
     originalDocumentStructure,
     files,
@@ -258,7 +298,6 @@ export default async function loadSources({
     totalTokens,
     totalLines,
     mediaFiles,
-    isLargeContext,
     allFilesPaths,
   };
 }
@@ -306,8 +345,14 @@ loadSources.input_schema = {
 loadSources.output_schema = {
   type: "object",
   properties: {
-    datasources: {
-      type: "string",
+    dataSources: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          dataSourceChunk: { type: "string" },
+        },
+      },
     },
     files: {
       type: "array",
@@ -338,3 +383,33 @@ loadSources.output_schema = {
 };
 
 loadSources.task_render_mode = "hide";
+
+function splitSourcesToChunks(sources, maxTokens) {
+  const chunks = [];
+
+  let currentChunk = [];
+  let currentTokens = 0;
+
+  for (const source of sources) {
+    const sourceTokens = source.tokens;
+
+    if (currentTokens + sourceTokens > maxTokens) {
+      // Start a new chunk
+      if (currentChunk.length > 0) {
+        chunks.push(currentChunk);
+      }
+      currentChunk = [source];
+      currentTokens = sourceTokens;
+    } else {
+      // Add to current chunk
+      currentChunk.push(source);
+      currentTokens += sourceTokens;
+    }
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+
+  return chunks;
+}

package/agents/utils/{save-docs.mjs → post-generate.mjs}

@@ -1,4 +1,4 @@
-import { readdir, unlink, writeFile } from "node:fs/promises";
+import { readdir, unlink } from "node:fs/promises";
 import { join } from "node:path";
 import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
 import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
@@ -10,7 +10,7 @@ import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
  * @param {Array<string>} [params.translateLanguages] - Translation languages
  * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
  */
-export default async function saveDocs({
+export default async function postGenerate({
   documentExecutionStructure: documentStructure,
   docsDir,
   translateLanguages = [],
@@ -26,15 +26,6 @@ export default async function saveDocs({
     console.warn("Failed to save git HEAD:", err.message);
   }
 
-  // Generate _sidebar.md
-  try {
-    const sidebar = generateSidebar(documentStructure);
-    const sidebarPath = join(docsDir, "_sidebar.md");
-    await writeFile(sidebarPath, sidebar, "utf8");
-  } catch (err) {
-    console.error("Failed to save _sidebar.md:", err.message);
-  }
-
   // Clean up invalid .md files that are no longer in the documentation structure
   try {
     await cleanupInvalidFiles(documentStructure, docsDir, translateLanguages, locale);
@@ -140,43 +131,3 @@ async function cleanupInvalidFiles(documentStructure, docsDir, translateLanguage
 }
 
 // Generate sidebar content, support nested structure, and the order is consistent with documentStructure
-function generateSidebar(documentStructure) {
-  // Build tree structure
-  const root = {};
-  for (const { path, title, parentId } of documentStructure) {
-    const relPath = path.replace(/^\//, "");
-    const segments = relPath.split("/");
-    let node = root;
-    for (let i = 0; i < segments.length; i++) {
-      const seg = segments[i];
-      if (!node[seg])
-        node[seg] = {
-          __children: {},
-          __title: null,
-          __fullPath: segments.slice(0, i + 1).join("/"),
-          __parentId: parentId,
-        };
-      if (i === segments.length - 1) node[seg].__title = title;
-      node = node[seg].__children;
-    }
-  }
-  // Recursively generate sidebar text, the link path is the flattened file name
-  function walk(node, parentSegments = [], indent = "") {
-    let out = "";
-    for (const key of Object.keys(node)) {
-      const item = node[key];
-      const fullSegments = [...parentSegments, key];
-      const flatFile = `${fullSegments.join("-")}.md`;
-      if (item.__title) {
-        const realIndent = item.__parentId === null ? "" : indent;
-        out += `${realIndent}* [${item.__title}](/${flatFile})\n`;
-      }
-      const children = item.__children;
-      if (Object.keys(children).length > 0) {
-        out += walk(children, fullSegments, `${indent}  `);
-      }
-    }
-    return out;
-  }
-  return walk(root).replace(/\n+$/, "");
-}

package/agents/utils/save-doc-translation.mjs

@@ -0,0 +1,27 @@
+import { saveDocTranslation as _saveDocTranslation } from "../../utils/utils.mjs";
+
+export default async function saveDocTranslation({
+  path,
+  docsDir,
+  translation,
+  language,
+  labels,
+  isShowMessage = false,
+}) {
+  await _saveDocTranslation({
+    path,
+    docsDir,
+    language,
+    translation,
+    labels,
+  });
+
+  if (isShowMessage) {
+    const message = `✅ Translation completed successfully.`;
+    return { message };
+  }
+
+  return {};
+}
+
+saveDocTranslation.task_render_mode = "hide";

package/agents/utils/{save-single-doc.mjs → save-doc.mjs}

@@ -1,24 +1,22 @@
 import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
-import { saveDocWithTranslations } from "../../utils/utils.mjs";
+import { saveDoc as _saveDoc } from "../../utils/utils.mjs";
 
-export default async function saveSingleDoc({
+export default async function saveDoc({
   path,
   content,
   docsDir,
-  translates,
   labels,
   locale,
-  isTranslate = false,
+  feedback,
   isShowMessage = false,
+  ...rest
 }) {
-  const _results = await saveDocWithTranslations({
+  await _saveDoc({
     path,
     content,
     docsDir,
-    translates,
     labels,
     locale,
-    isTranslate,
   });
 
   if (isShowMessage) {
@@ -29,13 +27,20 @@ export default async function saveSingleDoc({
       console.warn("Failed to shutdown mermaid worker pool:", error.message);
     }
 
-    const message = isTranslate
-      ? `✅ Translation completed successfully`
-      : `✅ Document updated successfully`;
+    const message = `✅ Document updated successfully.`;
     return { message };
   }
 
-  return {};
+  return {
+    path,
+    content,
+    docsDir,
+    labels,
+    locale,
+    feedback,
+    isShowMessage,
+    ...rest,
+  };
 }
 
-saveSingleDoc.task_render_mode = "hide";
+saveDoc.task_render_mode = "hide";

package/agents/utils/save-sidebar.mjs

@@ -0,0 +1,59 @@
+import { join } from "node:path";
+import fs from "fs-extra";
+
+export default async function saveSidebar({ documentStructure, docsDir }) {
+  // Generate _sidebar.md
+  try {
+    const sidebar = generateSidebar(documentStructure);
+    const sidebarPath = join(docsDir, "_sidebar.md");
+
+    await fs.ensureDir(docsDir);
+    await fs.writeFile(sidebarPath, sidebar, "utf8");
+  } catch (err) {
+    console.error("Failed to save _sidebar.md:", err.message);
+  }
+  return {};
+}
+
+// Recursively generate sidebar text, the link path is the flattened file name
+function walk(node, parentSegments = [], indent = "") {
+  let out = "";
+  for (const key of Object.keys(node)) {
+    const item = node[key];
+    const fullSegments = [...parentSegments, key];
+    const flatFile = `${fullSegments.join("-")}.md`;
+    if (item.__title) {
+      const realIndent = item.__parentId === null ? "" : indent;
+      out += `${realIndent}* [${item.__title}](/${flatFile})\n`;
+    }
+    const children = item.__children;
+    if (Object.keys(children).length > 0) {
+      out += walk(children, fullSegments, `${indent}  `);
+    }
+  }
+  return out;
+}
+
+function generateSidebar(documentStructure) {
+  // Build tree structure
+  const root = {};
+  for (const { path, title, parentId } of documentStructure) {
+    const relPath = path.replace(/^\//, "");
+    const segments = relPath.split("/");
+    let node = root;
+    for (let i = 0; i < segments.length; i++) {
+      const seg = segments[i];
+      if (!node[seg])
+        node[seg] = {
+          __children: {},
+          __title: null,
+          __fullPath: segments.slice(0, i + 1).join("/"),
+          __parentId: parentId,
+        };
+      if (i === segments.length - 1) node[seg].__title = title;
+      node = node[seg].__children;
+    }
+  }
+
+  return walk(root).replace(/\n+$/, "");
+}

package/agents/utils/transform-detail-data-sources.mjs

@@ -0,0 +1,45 @@
+import fs from "node:fs";
+import { isRemoteFile } from "../../utils/file-utils.mjs";
+import { normalizePath, toRelativePath } from "../../utils/utils.mjs";
+
+export default function transformDetailDataSource({ sourceIds }, options = {}) {
+  // Read file content for each sourceId, ignoring failures
+  let openAPISpec;
+  const remoteFileList = options?.context?.userContext?.remoteFileList || [];
+  const contents = (sourceIds || [])
+    .filter((id) => {
+      const openApiSourceId = options?.context?.userContext?.openAPISpec?.sourceId;
+      if (openApiSourceId !== undefined && openApiSourceId === id) {
+        openAPISpec = options.context.userContext.openAPISpec;
+        return false;
+      }
+      return true;
+    })
+    .map((id) => {
+      try {
+        if (isRemoteFile(id)) {
+          const findFile = remoteFileList.find((f) => f.sourceId === id);
+          if (findFile) {
+            return `// sourceId: ${id}\n${findFile.content}\n`;
+          }
+          return null;
+        }
+
+        const normalizedId = normalizePath(id);
+        const content = fs.readFileSync(normalizedId, "utf8");
+        const relativeId = toRelativePath(id);
+        return `// sourceId: ${relativeId}\n${content}\n`;
+      } catch {
+        // Ignore files that cannot be read
+        return null;
+      }
+    })
+    .filter(Boolean);
+
+  return {
+    detailDataSource: contents.join(""),
+    openAPISpec,
+  };
+}
+
+transformDetailDataSource.task_render_mode = "hide";

package/aigne.yaml

@@ -1,8 +1,9 @@
 #!/usr/bin/env aigne
 
-chat_model:
-  provider: google
-  name: gemini-2.5-pro
+model:
+  model: aignehub/gemini-2.5-pro # reasoning_effort 128-32768
+  # https://github.com/AIGNE-io/aigne-framework/blob/main/models/gemini/src/gemini-chat-model.ts#L115
+  reasoning_effort: 502
   # name: gemini-2.5-flash
   temperature: 0.8
 agents:
@@ -11,14 +12,13 @@ agents:
 
   # Documentation Structure Generation
   - ./agents/generate/generate-structure.yaml
-  - ./agents/generate/generate-structure-without-tools.yaml
   - ./agents/generate/update-document-structure.yaml
   - ./agents/generate/check-need-generate-structure.mjs
   - ./agents/generate/refine-document-structure.yaml
   - ./agents/generate/check-document-structure.yaml
   - ./agents/generate/user-review-document-structure.mjs
   - ./agents/generate/index.yaml
-  
+
   # Documentation Structure Tools
   - ./agents/generate/document-structure-tools/add-document.mjs
   - ./agents/generate/document-structure-tools/delete-document.mjs
@@ -47,6 +47,7 @@ agents:
 
   # Publishing
   - ./agents/publish/publish-docs.mjs
+  - ./agents/publish/translate-meta.mjs
   - ./agents/publish/index.yaml
 
   # Media
@@ -65,9 +66,11 @@ agents:
 
   # Utilities
   - ./agents/utils/load-sources.mjs
-  - ./agents/utils/save-docs.mjs
-  - ./agents/utils/transform-detail-datasources.mjs
-  - ./agents/utils/save-single-doc.mjs
+  - ./agents/utils/post-generate.mjs
+  - ./agents/utils/save-sidebar.mjs
+  - ./agents/utils/transform-detail-data-sources.mjs
+  - ./agents/utils/save-doc.mjs
+  - ./agents/utils/save-doc-translation.mjs
   - ./agents/utils/save-output.mjs
   - ./agents/utils/format-document-structure.mjs
   - ./agents/utils/find-item-by-path.mjs
@@ -92,6 +95,11 @@ agents:
   - ./agents/evaluate/document-structure.yaml
   - ./agents/evaluate/document.yaml
   - ./agents/evaluate/code-snippet.mjs
+
+  # Diagram
+  - ./agents/generate/merge-diagram.yaml
+  - ./agents/update/generate-diagram.yaml
+  - ./agents/update/check-generate-diagram.mjs
 cli:
   chat: ./agents/chat/index.yaml
   agents:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/doc-smith",
-  "version": "0.8.15-beta",
+  "version": "0.8.15-beta.10",
   "description": "AI-driven documentation generation tool built on the AIGNE Framework",
   "publishConfig": {
     "access": "public"
@@ -56,6 +56,7 @@
     "remark-lint": "^10.0.1",
     "remark-parse": "^11.0.0",
     "terminal-link": "^4.0.0",
+    "typescript": "^5.9.3",
     "ufo": "^1.6.1",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",

package/prompts/common/document/content-rules-core.md

@@ -2,12 +2,12 @@ Target Audience: {{targetAudience}}
 
 Content Generation Rules:
 
-- Use only information from DataSources, never fabricate or supplement content not present in the sources
+- Use only information from `<detail_data_source>`, never fabricate or supplement content not present in the sources
 - Combine the current {{nodeName}} title and description to create a well-structured content plan that is rich, organized, and engaging
 - Content style must match the target audience
-- Clearly differentiate content from other {{nodeName}} items in the documentStructure to avoid duplication and highlight this {{nodeName}}'s unique value
+- Clearly differentiate content from other {{nodeName}} items in the `<document_structure>` to avoid duplication and highlight this {{nodeName}}'s unique value
 {% if enforceInfoCompleteness %}
-- If DataSources lack sufficient information, return an error message requesting users to provide additional content. Ensure page content is sufficiently rich, don't hesitate to ask users for supplementary information
+- If `<detail_data_source>` lack sufficient information, return an error message requesting users to provide additional content. Ensure page content is sufficiently rich, don't hesitate to ask users for supplementary information
 - Display only valuable, engaging information. If information is insufficient, prompt users to provide more details
 {% endif %}
 - Output complete information including all content planned for the {{nodeName}}
@@ -15,6 +15,6 @@ Content Generation Rules:
 - Maintain a strict, sequential heading hierarchy; no skipping (e.g., no jump from level-1 to level-3).
 - Format markdown output with proper line breaks and spacing for easy reading
 - For list data with many items, prioritize using markdown table for cleaner, more readable presentation
-- Do not mention 'DataSources' in output, your content is for user consumption, and users are unaware of DataSources
-- Do not include file paths from Data Sources in output as they are meaningless to users
-- Avoid phrases like 'current {{nodeName}}'
+- Do not mention `<detail_data_source>` in output, your content is for user consumption, and users are unaware of detailDataSource
+- Do not include file paths from `<data_sources>` in output as they are meaningless to users
+- Avoid phrases like 'current {{nodeName}}'

package/prompts/common/document/media-file-list-usage-rules.md

@@ -0,0 +1,12 @@
+<media_file_list_usage_rules>
+
+**Usage Workflow**
+1. Read the `<media_file_list>` data and take note of each file's `path` and `description` as references.
+2. Combine those descriptions with the current document's content to decide which images should be used and where they should be inserted.
+3. Confirm that every inserted image path comes from `<media_file_list>`. If a path is missing from that list, replace it with one that is included.
+
+**Usage Requirements**
+- Insert images with Markdown syntax: `![Descriptive alt text](<path-from-media_file_list>)`.
+- Never invent, reinterpret, fabricate, normalize, or rewrite any media file path under any circumstances.
+
+</media_file_list_usage_rules>

package/prompts/common/document/openapi-usage-rules.md

@@ -0,0 +1,36 @@
+{% if openAPISpec %}
+<openapi_usage_rules>
+
+**Goal:** Use the provided OpenAPI (Swagger) specification, align it with the current page objective, and leverage it to refine this document.
+
+**OpenAPI File Content:** 
+<openapi_doc>
+
+{{ openAPISpec }}
+
+</openapi_doc>
+
+---
+
+### **Documentation Requirements and Constraints**
+
+1.  **Extract the core content:**
+    * Organize the document by functional modules.
+    * For each path item, include the following elements:
+        * HTTP method and path.
+        * Concise summary.
+        * Detailed description.
+        * Request parameters: name, location (`in`), type, required flag, description.
+        * Request body: describe its structure when present.
+        * Responses: at least the key status codes (e.g., 200, 201, 400, 500) and their schemas.
+
+2.  **Mandatory API description constraints (deduplication rule):**
+    * **Ensure that throughout the document (including preface, overview, etc.), any introduction to the project APIs appears only within this OpenAPI-generated "API reference" section.**
+    * **Never** repeat or expand the interface list elsewhere in the document (for example, "Quick Start" or "Architecture Overview" sections).
+
+---
+
+**Expected output format:** A concise, clear, and easy-to-scan Markdown document.
+
+</openapi_usage_rules>
+{% endif %}