@aigne/doc-smith 0.8.15-beta → 0.8.15-beta.1

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.8.15-beta.1](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.15-beta...v0.8.15-beta.1) (2025-10-23)
+
+
+### Features
+
+* support openapi datasource, generate better api docs ([#212](https://github.com/AIGNE-io/aigne-doc-smith/issues/212)) ([6a683f1](https://github.com/AIGNE-io/aigne-doc-smith/commit/6a683f117aa94f265383be6e1b3b957803004032))
+
 ## [0.8.15-beta](https://github.com/AIGNE-io/aigne-doc-smith/compare/v0.8.14...v0.8.15-beta) (2025-10-21)
 
 

package/agents/update/check-document.mjs

@@ -2,6 +2,7 @@ import { access, readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { TeamAgent } from "@aigne/core";
+
 import checkDetailResult from "../utils/check-detail-result.mjs";
 
 // Get current script directory
@@ -102,6 +103,19 @@ export default async function checkDocument(
       options.context.agents["saveSingleDoc"],
     ],
   });
+  let openAPISpec = null;
+
+  if (options.context?.userContext?.openAPISpec?.sourceId) {
+    const matchingDocument = originalDocumentStructure.find((item) => {
+      if (item.path === path) {
+        return item.sourceIds.find((x) => x === options.context.userContext.openAPISpec.sourceId);
+      }
+      return false;
+    });
+    if (matchingDocument) {
+      openAPISpec = options.context.userContext.openAPISpec;
+    }
+  }
 
   const result = await options.context.invoke(teamAgent, {
     ...rest,
@@ -112,6 +126,7 @@ export default async function checkDocument(
     originalDocumentStructure,
     documentStructure,
     detailFeedback: contentValidationFailed ? validationResult.detailFeedback : "",
+    openAPISpec,
   });
 
   return {

package/agents/utils/choose-docs.mjs

@@ -50,7 +50,12 @@ export default async function chooseDocs(
         });
 
         // Use title if available, otherwise fall back to filename
-        const displayName = docItem?.title || file;
+        let displayName = docItem?.title;
+        if (displayName) {
+          displayName = `${displayName} (${file})`;
+        } else {
+          displayName = file;
+        }
 
         return {
           name: displayName,

package/agents/utils/load-sources.mjs

@@ -1,4 +1,5 @@
 import { readFile } from "node:fs/promises";
+import { statSync } from "node:fs";
 import path from "node:path";
 import imageSize from "image-size";
 import {
@@ -7,6 +8,7 @@ import {
   loadFilesFromPaths,
   readFileContents,
   getMimeType,
+  checkIsRemoteFile,
 } from "../../utils/file-utils.mjs";
 import {
   getCurrentGitHead,
@@ -18,28 +20,63 @@ import {
   DEFAULT_EXCLUDE_PATTERNS,
   DEFAULT_INCLUDE_PATTERNS,
 } from "../../utils/constants/index.mjs";
-
-export default async function loadSources({
-  sources = [],
-  sourcesPath = [],
-  includePatterns,
-  excludePatterns,
-  outputDir,
-  docsDir,
-  "doc-path": docPath,
-  boardId,
-  useDefaultPatterns = true,
-  lastGitHead,
-  reset = false,
-  media,
-} = {}) {
+import { isOpenAPISpecFile } from "../../utils/openapi/index.mjs";
+
+export default async function loadSources(
+  {
+    sources = [],
+    sourcesPath = [],
+    includePatterns,
+    excludePatterns,
+    outputDir,
+    docsDir,
+    "doc-path": docPath,
+    boardId,
+    useDefaultPatterns = true,
+    lastGitHead,
+    reset = false,
+    media,
+  } = {},
+  options,
+) {
   let files = Array.isArray(sources) ? [...sources] : [];
   const { minImageWidth } = media || { minImageWidth: 800 };
 
   if (sourcesPath) {
-    const allFiles = await loadFilesFromPaths(sourcesPath, {
+    const sourcesPathList = Array.isArray(sourcesPath) ? sourcesPath : [sourcesPath];
+    const pickSourcesPath = [];
+    const omitSourcesPath = [];
+    sourcesPathList.forEach((x) => {
+      if (typeof x !== "string" || !x) {
+        return;
+      }
+      if (x.startsWith("!")) {
+        omitSourcesPath.push(x.substring(1));
+      } else {
+        pickSourcesPath.push(x);
+      }
+    });
+
+    const customExcludePatterns = omitSourcesPath
+      .map((x) => {
+        try {
+          const stats = statSync(x);
+          if (stats.isFile()) {
+            return x;
+          }
+          if (stats.isDirectory()) {
+            return `${x}/**`;
+          }
+          return null;
+        } catch (error) {
+          console.warn(`Failed to stat path ${x}: ${error.message}`);
+          return null;
+        }
+      })
+      .filter(Boolean);
+    const allFiles = await loadFilesFromPaths(pickSourcesPath, {
       includePatterns,
-      excludePatterns,
+      excludePatterns: [...new Set([...(excludePatterns || []), ...customExcludePatterns])],
       useDefaultPatterns,
       defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
       defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
@@ -50,9 +87,6 @@ export default async function loadSources({
 
   files = [...new Set(files)];
 
-  // all files path
-  const allFilesPaths = files.map((file) => `- ${toRelativePath(file)}`).join("\n");
-
   // Define media file extensions
   const mediaExtensions = [
     ".jpg",
@@ -110,7 +144,7 @@ export default async function loadSources({
     files.map(async (file) => {
       const ext = path.extname(file).toLowerCase();
 
-      if (mediaExtensions.includes(ext)) {
+      if (mediaExtensions.includes(ext) && !checkIsRemoteFile(file)) {
         // This is a media file
         const relativePath = path.relative(docsDir, file);
         const fileName = path.basename(file);
@@ -161,7 +195,7 @@ export default async function loadSources({
   }
 
   // Read all source files using the utility function
-  const sourceFiles = await readFileContents(sourceFilesPaths, process.cwd());
+  let sourceFiles = await readFileContents(sourceFilesPaths, process.cwd());
 
   // Count tokens and lines using utility function
   const { totalTokens, totalLines } = calculateFileStats(sourceFiles);
@@ -169,8 +203,32 @@ export default async function loadSources({
   // check if totalTokens is too large
   const isLargeContext = totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD;
 
+  // filter OpenAPI doc should after check isLargeContext
+  sourceFiles = sourceFiles.filter((file) => {
+    if (options?.context?.userContext.openAPISpec) return true;
+
+    const isOpenAPI = isOpenAPISpecFile(file.content);
+    if (isOpenAPI && options?.context?.userContext) {
+      options.context.userContext.openAPISpec = file;
+    }
+    return !isOpenAPI;
+  });
+
+  const httpFileList = [];
+
+  sourceFiles.forEach((file) => {
+    if (checkIsRemoteFile(file.sourceId)) {
+      httpFileList.push(file);
+    }
+  });
+  if (options?.context?.userContext) {
+    options.context.userContext.httpFileList = httpFileList;
+  }
+
   // Build allSources string using utility function
   const allSources = buildSourcesContent(sourceFiles, isLargeContext);
+  // all files path
+  const allFilesPaths = sourceFiles.map((x) => `- ${toRelativePath(x.sourceId)}`).join("\n");
 
   // Get the last documentation structure
   let originalDocumentStructure;

package/agents/utils/transform-detail-datasources.mjs

@@ -1,11 +1,30 @@
 import fs from "node:fs";
 import { normalizePath, toRelativePath } from "../../utils/utils.mjs";
+import { checkIsRemoteFile } from "../../utils/file-utils.mjs";
 
-export default function transformDetailDatasources({ sourceIds }) {
+export default function transformDetailDatasources({ sourceIds }, options = {}) {
   // Read file content for each sourceId, ignoring failures
+  let openAPISpec;
+  const httpFileList = options?.context?.userContext?.httpFileList || [];
   const contents = (sourceIds || [])
+    .filter((id) => {
+      const openApiSourceId = options?.context?.userContext?.openAPISpec?.sourceId;
+      if (openApiSourceId !== undefined && openApiSourceId === id) {
+        openAPISpec = options.context.userContext.openAPISpec;
+        return false;
+      }
+      return true;
+    })
     .map((id) => {
       try {
+        if (checkIsRemoteFile(id)) {
+          const findFile = httpFileList.find((f) => f.sourceId === id);
+          if (findFile) {
+            return `// sourceId: ${id}\n${findFile.content}\n`;
+          }
+          return null;
+        }
+
         const normalizedId = normalizePath(id);
         const content = fs.readFileSync(normalizedId, "utf8");
         const relativeId = toRelativePath(id);
@@ -17,7 +36,10 @@ export default function transformDetailDatasources({ sourceIds }) {
     })
     .filter(Boolean);
 
-  return { detailDataSources: contents.join("") };
+  return {
+    detailDataSources: contents.join(""),
+    openAPISpec,
+  };
 }
 
 transformDetailDatasources.task_render_mode = "hide";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/doc-smith",
-  "version": "0.8.15-beta",
+  "version": "0.8.15-beta.1",
   "description": "AI-driven documentation generation tool built on the AIGNE Framework",
   "publishConfig": {
     "access": "public"

package/prompts/detail/d2-diagram/system-prompt.md

@@ -913,6 +913,28 @@ Ensure that the shape names used in connections are accurate and match the actua
 - **Good Practice:**
   ```d2
   shape: sequence_diagram
+  User: { 
+    shape: c4-person 
+  }
+
+  App: {
+    label: "Your Application"
+    shape: rectangle
+
+    ResumeSubscription: {
+      label: "ResumeSubscription Component"
+    }
+  }
+
+  Payment-API: {
+    label: "Payment Backend API"
+    shape: rectangle
+  }
+
+  DID-Wallet: {
+    label: "DID Wallet"
+    icon: "https://www.arcblock.io/image-bin/uploads/37198ddc4a0b9e91e5c1c821ab895a34.svg"
+  }
 
   User -> App.ResumeSubscription: "1. Triggers resume action"
 
@@ -922,12 +944,16 @@ Ensure that the shape names used in connections are accurate and match the actua
   App.ResumeSubscription.t1 -> User: "4. Display confirmation dialog"
   User -> App.ResumeSubscription.t1: "5. Clicks 'Confirm'"
 
-  App.ResumeSubscription.t1 -> DID-Wallet: "6a. Open 're-stake' session"
-  User -> DID-Wallet: "7a. Approve in wallet"
-  DID-Wallet -> App.ResumeSubscription.t1: "8a. Send success callback"
+  "If Re-Staking is Required": {
+    App.ResumeSubscription.t1 -> DID-Wallet: "6a. Open 're-stake' session"
+    User -> DID-Wallet: "7a. Approve in wallet"
+    DID-Wallet -> App.ResumeSubscription.t1: "8a. Send success callback"
+  }
 
-  App.ResumeSubscription.t1 -> Payment-API: "6b. Call recover endpoint\n(PUT /recover)"
-  Payment-API -> App.ResumeSubscription.t1: "7b. Return success"
+  "If No Staking is Required": {
+    App.ResumeSubscription.t1 -> Payment-API: "6b. Call recover endpoint\n(PUT /recover)"
+    Payment-API -> App.ResumeSubscription.t1: "7b. Return success"
+  }
 
   App.ResumeSubscription.t1 -> Payment-API: "9. Fetch updated subscription details"
   Payment-API -> App.ResumeSubscription.t1: "10. Return latest subscription"
@@ -1086,12 +1112,12 @@ Ensure that the shape names used in connections are accurate and match the actua
   Blocklet-Service -> Application.Auth-Middleware: "4. Return permissions"
   Application.Auth-Middleware -> Application.Auth-Middleware: "5. Evaluate all rules"
 
-  "If Authorized" {
+  "If Authorized": {
     Application.Auth-Middleware -> Application.Protected-Route: "6a. next()"
     Application.Protected-Route -> Client: "7a. 200 OK Response"
   }
 
-  "If Forbidden" {
+  "If Forbidden": {
     Application.Auth-Middleware -> Client: "6b. 403 Forbidden Response"
   }
   ```

package/prompts/detail/generate/user-prompt.md

@@ -27,6 +27,43 @@
 
 </datasources>
 
+{% if openAPISpec %}
+<openapi>
+
+**Goal:** Use the provided OpenAPI (Swagger) specification, align it with the current page objective, and leverage it to refine this document.
+
+**OpenAPI File Content:** 
+<openapi_doc>
+
+{{ openAPISpec }}
+
+</openapi_doc>
+
+---
+
+### **Documentation Requirements and Constraints**
+
+1.  **Extract the core content:**
+    * Organize the document by functional modules.
+    * For each path item, include the following elements:
+        * HTTP method and path.
+        * Concise summary.
+        * Detailed description.
+        * Request parameters: name, location (`in`), type, required flag, description.
+        * Request body: describe its structure when present.
+        * Responses: at least the key status codes (e.g., 200, 201, 400, 500) and their schemas.
+
+2.  **Mandatory API description constraints (deduplication rule):**
+    * **Ensure that throughout the document (including preface, overview, etc.), any introduction to the project APIs appears only within this OpenAPI-generated "API reference" section.**
+    * **Never** repeat or expand the interface list elsewhere in the document (for example, "Quick Start" or "Architecture Overview" sections).
+
+---
+
+**Expected output format:** A concise, clear, and easy-to-scan Markdown document.
+
+</openapi>
+{% endif %}
+
 
 {% include "./detail-example.md" %}
 

package/prompts/structure/generate/user-prompt.md

@@ -12,6 +12,40 @@
 {{ datasources }}
 </datasources>
 
+{% if userContext.openAPISpec %}
+<openapi>
+
+**Goal:** Use the provided OpenAPI (Swagger) specification to design how the OpenAPI content and the overall document should be structured together.
+
+**OpenAPI File Content:** 
+<openapi_doc>
+
+{{ userContext.openAPISpec }}
+
+</openapi_doc>
+
+---
+
+### **Documentation Requirements and Constraints**
+
+1.  **Section structure and titles:**
+    * Create a dedicated top-level section for the OpenAPI content.
+    * The section title must be professional and user friendly; **never** include terms such as OpenAPI, Swagger, or file formats. Recommended titles include **"API Interface Reference"** or **"Interface Reference"**.
+
+2.  **Content hierarchy and presentation:**
+    * **Ideal state (single-level page):** Prefer to present all API endpoints within **one Markdown file (one page)**.
+    * **Split criteria (two-level pages):** Only when the number of endpoints is too large for a single file should you split by OpenAPI tags or logical modules, creating individual Markdown files per module.
+    * **Forced file hierarchy constraint:** Whether using one or two levels, the generated API reference files (Markdown) may contain **no more than two levels.**
+        * **Example (two-level structure):** `/api-reference.md` (index) -> `/api/user.md`, `/api/order.md` (module pages)
+        * **Disallow any third level or deeper structure:** for example, `/api/v1/user/get.md`.
+
+3.  **Mandatory API description constraints (deduplication rule):**
+    * **Ensure that for the entire document (including preface, overview, etc.), any introduction to the project APIs appears only within this OpenAPI-generated "API reference" section.**
+    * **Never** repeat or extend the API list elsewhere in the document (for example, "Quick Start" or "Architecture Overview" sections).
+
+</openapi>
+{% endif %}
+
 
 {% if originalDocumentStructure %}
 <last_document_structure>

package/utils/file-utils.mjs

@@ -8,6 +8,8 @@ import { isBinaryFile } from "isbinaryfile";
 import { encode } from "gpt-tokenizer";
 import { fileTypeFromBuffer } from "file-type";
 import { gunzipSync } from "node:zlib";
+
+import { debug } from "./debug.mjs";
 import { isGlobPattern } from "./utils.mjs";
 import { INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD } from "./constants/index.mjs";
 import { uploadFiles } from "./upload-files.mjs";
@@ -284,6 +286,11 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
         continue;
       }
 
+      if (checkIsRemoteFile(dir)) {
+        allFiles.push(dir);
+        continue;
+      }
+
       // First try to access as a file or directory
       const stats = await stat(dir);
 
@@ -313,7 +320,13 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
             : [];
 
           finalIncludePatterns = [...defaultIncludePatterns, ...userInclude];
-          finalExcludePatterns = [...defaultExcludePatterns, ...userExclude];
+          finalExcludePatterns = [
+            ...defaultExcludePatterns,
+            ...userExclude.map((x) => {
+              const prefix = `${dir}/`;
+              return x.startsWith(prefix) ? x.slice(prefix.length) : x;
+            }),
+          ];
         } else {
           // Use only user patterns
           if (includePatterns) {
@@ -374,6 +387,10 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
  * @returns {Promise<boolean>} True if file appears to be a text file
  */
 async function isTextFile(filePath) {
+  if (checkIsRemoteFile(filePath)) {
+    return checkIsHttpTextFile(filePath);
+  }
+
   try {
     const isBinary = await isBinaryFile(filePath);
     return !isBinary;
@@ -383,6 +400,53 @@ async function isTextFile(filePath) {
   }
 }
 
+export function checkIsRemoteFile(filepath) {
+  if (filepath.startsWith("http://") || filepath.startsWith("https://")) {
+    return true;
+  }
+  return false;
+}
+
+export async function checkIsHttpTextFile(fileUrl) {
+  try {
+    const res = await fetch(fileUrl, {
+      method: "HEAD",
+    });
+    const contentType = res.headers.get("content-type") || "";
+    const textMimeTypes = [
+      "application/json",
+      "application/ld+json",
+      "application/graphql+json",
+      "application/xml",
+      "application/xhtml+xml",
+      "application/javascript",
+      "application/ecmascript",
+      "application/x-www-form-urlencoded",
+      "application/rss+xml",
+      "application/atom+xml",
+    ];
+    if (contentType.startsWith("text/") || textMimeTypes.includes(contentType)) {
+      return true;
+    }
+    return false;
+  } catch (error) {
+    debug(`Failed to check HTTP file content type: ${fileUrl} - ${error.message}`);
+    return null;
+  }
+}
+
+export async function getHttpFileContent(file) {
+  if (!file) return null;
+  try {
+    const res = await fetch(file);
+    const text = await res.text();
+    return text;
+  } catch (error) {
+    debug(`Failed to fetch HTTP file content: ${file} - ${error.message}`);
+    return null;
+  }
+}
+
 /**
  * Read and parse file contents from an array of file paths
  * @param {string[]} files - Array of file paths to read
@@ -405,12 +469,24 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
       }
 
       try {
-        const content = await readFile(file, "utf8");
-        const relativePath = path.relative(baseDir, file);
-        return {
-          sourceId: relativePath,
-          content,
-        };
+        if (checkIsRemoteFile(file)) {
+          const content = await getHttpFileContent(file);
+          if (content) {
+            return {
+              sourceId: file,
+              content,
+            };
+          }
+
+          return null;
+        } else {
+          const content = await readFile(file, "utf8");
+          const relativePath = path.relative(baseDir, file);
+          return {
+            sourceId: relativePath,
+            content,
+          };
+        }
       } catch (error) {
         // If reading as text fails (e.g., binary file), skip it
         console.warn(`Failed to read file as text: ${file} - ${error.message}`);

package/utils/openapi/index.mjs

@@ -0,0 +1,24 @@
+import { parse } from "yaml";
+
+export function isOpenAPISpecFile(content) {
+  const trimmedContent = content.trim();
+  try {
+    const parsed = parse(trimmedContent, {
+      logLevel: "silent",
+    });
+    if (parsed.openapi || parsed.swagger) {
+      return true;
+    }
+  } catch {
+    //
+  }
+  try {
+    const parsed = JSON.parse(trimmedContent);
+    if (parsed.openapi || parsed.swagger) {
+      return true;
+    }
+  } catch {
+    //
+  }
+  return false;
+}