@aigne/doc-smith 0.0.1

package/agents/publish-docs.mjs

@@ -0,0 +1,212 @@
+import { join } from "node:path";
+import { joinURL } from "ufo";
+import open from "open";
+import { publishDocs as publishDocsFn } from "@aigne/publish-docs";
+import { createConnect } from "@aigne/cli/utils/load-aigne.js";
+import { existsSync, mkdirSync } from "node:fs";
+import { readFile, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { parse, stringify } from "yaml";
+import { execSync } from "node:child_process";
+import { basename } from "node:path";
+
+const WELLKNOWN_SERVICE_PATH_PREFIX = "/.well-known/service";
+
+/**
+ * Get project name from git repository or current directory
+ * @returns {string} - The project name
+ */
+function getProjectName() {
+  // Check if we're in a git repository
+  try {
+    const gitRemote = execSync("git remote get-url origin", {
+      encoding: "utf8",
+      stdio: ["pipe", "pipe", "ignore"],
+    }).trim();
+
+    // Extract repository name from git remote URL
+    const repoName = gitRemote.split("/").pop().replace(".git", "");
+    return repoName;
+  } catch (error) {
+    // Not in git repository or no origin remote, use current directory name
+    return basename(process.cwd());
+  }
+}
+
+/**
+ * Get access token from environment, config file, or prompt user for authorization
+ * @param {string} appUrl - The application URL
+ * @returns {Promise<string>} - The access token
+ */
+async function getAccessToken(appUrl) {
+  const DOC_SMITH_ENV_FILE = join(
+    homedir(),
+    ".aigne",
+    "doc-smith-connected.yaml"
+  );
+  const { hostname } = new URL(appUrl);
+
+  let accessToken = process.env.DOC_DISCUSS_KIT_ACCESS_TOKEN;
+
+  // Check if access token exists in environment or config file
+  if (!accessToken) {
+    try {
+      if (existsSync(DOC_SMITH_ENV_FILE)) {
+        const data = await readFile(DOC_SMITH_ENV_FILE, "utf8");
+        if (data.includes("DOC_DISCUSS_KIT_ACCESS_TOKEN")) {
+          const envs = parse(data);
+          if (envs[hostname] && envs[hostname].DOC_DISCUSS_KIT_ACCESS_TOKEN) {
+            accessToken = envs[hostname].DOC_DISCUSS_KIT_ACCESS_TOKEN;
+          }
+        }
+      }
+    } catch (error) {
+      console.warn("Failed to read config file:", error.message);
+    }
+  }
+
+  // If still no access token, prompt user to authorize
+  if (!accessToken) {
+    const DISCUSS_KIT_URL = appUrl;
+    const connectUrl = joinURL(
+      new URL(DISCUSS_KIT_URL).origin,
+      WELLKNOWN_SERVICE_PATH_PREFIX
+    );
+
+    try {
+      const result = await createConnect({
+        connectUrl: connectUrl,
+        connectAction: "gen-simple-access-key",
+        source: `@aigne/cli doc-smith connect to Discuss Kit`,
+        closeOnSuccess: true,
+        openPage: (pageUrl) => open(pageUrl),
+      });
+
+      accessToken = result.accessKeySecret;
+      process.env.DOC_DISCUSS_KIT_ACCESS_TOKEN = accessToken;
+
+      // Save the access token to config file
+      const aigneDir = join(homedir(), ".aigne");
+      if (!existsSync(aigneDir)) {
+        mkdirSync(aigneDir, { recursive: true });
+      }
+
+      const existingConfig = existsSync(DOC_SMITH_ENV_FILE)
+        ? parse(await readFile(DOC_SMITH_ENV_FILE, "utf8"))
+        : {};
+
+      await writeFile(
+        DOC_SMITH_ENV_FILE,
+        stringify({
+          ...existingConfig,
+          [hostname]: {
+            DOC_DISCUSS_KIT_ACCESS_TOKEN: accessToken,
+            DOC_DISCUSS_KIT_URL: DISCUSS_KIT_URL,
+          },
+        })
+      );
+    } catch (error) {
+      console.error("Failed to get access token:", error);
+      throw new Error(
+        "Failed to obtain access token. Please check your network connection and try again later."
+      );
+    }
+  }
+
+  return accessToken;
+}
+
+/**
+ * Save boardId to input.yaml file if it was auto-created
+ * @param {string} boardId - The original boardId (may be empty)
+ * @param {string} newBoardId - The boardId returned from publishDocsFn
+ */
+async function saveBoardIdToInput(boardId, newBoardId) {
+  // Only save if boardId was auto-created
+  if (!boardId && newBoardId) {
+    try {
+      const docSmithDir = join(process.cwd(), "doc-smith");
+      if (!existsSync(docSmithDir)) {
+        mkdirSync(docSmithDir, { recursive: true });
+      }
+
+      const inputFilePath = join(docSmithDir, "input.yaml");
+      let fileContent = "";
+
+      // Read existing file content if it exists
+      if (existsSync(inputFilePath)) {
+        fileContent = await readFile(inputFilePath, "utf8");
+      }
+
+      // Check if boardId already exists in the file
+      const boardIdRegex = /^boardId:\s*.*$/m;
+      const newBoardIdLine = `boardId: ${newBoardId}`;
+
+      if (boardIdRegex.test(fileContent)) {
+        // Replace existing boardId line
+        fileContent = fileContent.replace(boardIdRegex, newBoardIdLine);
+      } else {
+        // Add boardId to the end of file
+        if (fileContent && !fileContent.endsWith("\n")) {
+          fileContent += "\n";
+        }
+        fileContent += newBoardIdLine + "\n";
+      }
+
+      await writeFile(inputFilePath, fileContent);
+      console.log(`Board ID saved to: ${inputFilePath}`);
+    } catch (error) {
+      console.warn("Failed to save board ID to input.yaml:", error.message);
+    }
+  }
+}
+
+export default async function publish({ docsDir, appUrl, boardId }) {
+  const accessToken = await getAccessToken(appUrl);
+
+  process.env.DOC_ROOT_DIR = docsDir;
+
+  const sidebarPath = join(docsDir, "_sidebar.md");
+
+  const boardName = boardId ? "" : getProjectName();
+
+  const { success, boardId: newBoardId } = await publishDocsFn({
+    sidebarPath,
+    accessToken,
+    appUrl,
+    boardId,
+    // If boardId is empty, use project name as boardName and auto create board
+    boardName,
+    autoCreateBoard: !boardId,
+  });
+
+  // Save boardId to input.yaml if it was auto-created
+  await saveBoardIdToInput(boardId, newBoardId);
+
+  return {
+    publishResult: {
+      success,
+    },
+  };
+}
+
+publish.input_schema = {
+  type: "object",
+  properties: {
+    docsDir: {
+      type: "string",
+      description: "The directory of the docs",
+    },
+    appUrl: {
+      type: "string",
+      description: "The url of the app",
+      default:
+        // "https://bbqawfllzdt3pahkdsrsone6p3wpxcwp62vlabtawfu.did.abtnet.io",
+        "https://www.staging.arcblock.io",
+    },
+    boardId: {
+      type: "string",
+      description: "The id of the board",
+    },
+  },
+};

package/agents/reflective-structure-planner.yaml

@@ -0,0 +1,10 @@
+type: team
+name: reflective-structure-planner
+description: A team of agents that plan the structure of the documentation.
+skills:
+  - structure-planning.yaml
+reflection:
+  reviewer: check-structure-planning-result.yaml
+  is_approved: isValid
+  max_iterations: 3
+  return_last_on_max_iterations: true

package/agents/save-docs.mjs

@@ -0,0 +1,153 @@
+import { writeFile, readdir, unlink } from "node:fs/promises";
+import { join } from "node:path";
+
+/**
+ * @param {Object} params
+ * @param {Array<{path: string, content: string, title: string}>} params.structurePlan
+ * @param {string} params.docsDir
+ * @param {Array<string>} [params.translateLanguages] - Translation languages
+ * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
+ */
+export default async function saveDocs({
+  structurePlanResult: structurePlan,
+  docsDir,
+  translateLanguages = [],
+}) {
+  const results = [];
+
+  // Generate _sidebar.md
+  try {
+    const sidebar = generateSidebar(structurePlan);
+    const sidebarPath = join(docsDir, "_sidebar.md");
+    await writeFile(sidebarPath, sidebar, "utf8");
+    results.push({ path: sidebarPath, success: true });
+  } catch (err) {
+    results.push({ path: "_sidebar.md", success: false, error: err.message });
+  }
+
+  // Clean up invalid .md files that are no longer in the structure plan
+  try {
+    const cleanupResults = await cleanupInvalidFiles(
+      structurePlan,
+      docsDir,
+      translateLanguages
+    );
+    results.push(...cleanupResults);
+  } catch (err) {
+    results.push({ path: "cleanup", success: false, error: err.message });
+  }
+
+  return { saveDocsResult: results };
+}
+
+/**
+ * Clean up .md files that are no longer in the structure plan
+ * @param {Array<{path: string, title: string}>} structurePlan
+ * @param {string} docsDir
+ * @param {Array<string>} translateLanguages
+ * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
+ */
+async function cleanupInvalidFiles(structurePlan, docsDir, translateLanguages) {
+  const results = [];
+
+  try {
+    // Get all .md files in docsDir
+    const files = await readdir(docsDir);
+    const mdFiles = files.filter((file) => file.endsWith(".md"));
+
+    // Generate expected file names from structure plan
+    const expectedFiles = new Set();
+
+    // Add main document files
+    for (const { path } of structurePlan) {
+      const flatName = path.replace(/^\//, "").replace(/\//g, "-");
+      const fileName = `${flatName}.md`;
+      expectedFiles.add(fileName);
+
+      // Add translation files for each language
+      for (const lang of translateLanguages) {
+        const translateFileName = `${flatName}.${lang}.md`;
+        expectedFiles.add(translateFileName);
+      }
+    }
+
+    // Find files to delete (files that are not in expectedFiles and not _sidebar.md)
+    const filesToDelete = mdFiles.filter(
+      (file) => !expectedFiles.has(file) && file !== "_sidebar.md"
+    );
+
+    // Delete invalid files
+    for (const file of filesToDelete) {
+      try {
+        const filePath = join(docsDir, file);
+        await unlink(filePath);
+        results.push({
+          path: filePath,
+          success: true,
+          message: `Deleted invalid file: ${file}`,
+        });
+      } catch (err) {
+        results.push({
+          path: file,
+          success: false,
+          error: `Failed to delete ${file}: ${err.message}`,
+        });
+      }
+    }
+
+    if (filesToDelete.length > 0) {
+      console.log(
+        `Cleaned up ${filesToDelete.length} invalid .md files from ${docsDir}`
+      );
+    }
+  } catch (err) {
+    // If docsDir doesn't exist or can't be read, that's okay
+    if (err.code !== "ENOENT") {
+      throw err;
+    }
+  }
+
+  return results;
+}
+
+// Generate sidebar content, support nested structure, and the order is consistent with structurePlan
+function generateSidebar(structurePlan) {
+  // Build tree structure
+  const root = {};
+  for (const { path, title, parentId } of structurePlan) {
+    const relPath = path.replace(/^\//, "");
+    const segments = relPath.split("/");
+    let node = root;
+    for (let i = 0; i < segments.length; i++) {
+      const seg = segments[i];
+      if (!node[seg])
+        node[seg] = {
+          __children: {},
+          __title: null,
+          __fullPath: segments.slice(0, i + 1).join("/"),
+          __parentId: parentId,
+        };
+      if (i === segments.length - 1) node[seg].__title = title;
+      node = node[seg].__children;
+    }
+  }
+  // Recursively generate sidebar text, the link path is the flattened file name
+  function walk(node, parentSegments = [], indent = "") {
+    let out = "";
+    for (const key of Object.keys(node)) {
+      const item = node[key];
+      const fullSegments = [...parentSegments, key];
+      const flatFile = fullSegments.join("-") + ".md";
+      if (item.__title) {
+        const realIndent = item.__parentId === null ? "" : indent;
+        out += `${realIndent}* [${item.__title}](/${flatFile})\n`;
+      }
+      const children = item.__children;
+      if (Object.keys(children).length > 0) {
+        out += walk(children, fullSegments, `${indent}  `);
+      }
+    }
+    return out;
+  }
+  return walk(root).replace(/\n+$/, "");
+}

package/agents/save-output.mjs

@@ -0,0 +1,25 @@
+import { promises as fs } from "node:fs";
+import { join } from "node:path";
+
+export default async function saveOutput({ savePath, fileName, saveKey, ...rest }) {
+  console.log("saveOutput", savePath, fileName, saveKey, rest);
+  if (!(saveKey in rest)) {
+    console.warn(`saveKey "${saveKey}" not found in input, skip saving.`);
+    return {
+      saveOutputStatus: false,
+      saveOutputPath: null,
+    };
+  }
+
+  const value = rest[saveKey];
+  const content =
+    typeof value === "object" && value !== null ? JSON.stringify(value, null, 2) : String(value);
+  await fs.mkdir(savePath, { recursive: true });
+  const filePath = join(savePath, fileName);
+  await fs.writeFile(filePath, content, "utf8");
+
+  return {
+    saveOutputStatus: true,
+    saveOutputPath: filePath,
+  };
+}

package/agents/save-single-doc.mjs

@@ -0,0 +1,18 @@
+import { saveDocWithTranslations } from "../utils/utils.mjs";
+
+export default async function saveSingleDoc({
+  path,
+  content,
+  docsDir,
+  translates,
+  labels,
+}) {
+  const results = await saveDocWithTranslations({
+    path,
+    content,
+    docsDir,
+    translates,
+    labels,
+  });
+  return { saveSingleDocResult: results };
+}

package/agents/schema/structure-plan-result.yaml

@@ -0,0 +1,32 @@
+type: array
+items:
+  type: object
+  properties:
+    title:
+      type: string
+    description:
+      type: string
+    path:
+      type: string
+      description: 路径，以 RUL 的格式返回，不能为空, 必须以 / 开头，不需要包含语言层级，比如 /zh/about 直接返回 /about
+    parentId:
+      type:
+        - string
+        - "null"
+    translates:
+      type: array
+      items:
+        type: object
+        properties:
+          language:
+            type: string
+            description: 语言代码，比如 zh, en, ja 等
+    sourceIds:
+      type: array
+      items:
+        type: string
+      description: 关联的 dataSources 中的 sourceId，用于后续的翻译和内容生成，必须来自 datasources 中的 sourceId，不能有虚假的 id, 不能为空
+  required:
+    - title
+    - description
+    - path

package/agents/schema/structure-plan.yaml

@@ -0,0 +1,26 @@
+type: array
+items:
+  type: object
+  properties:
+    title:
+      type: string
+    description:
+      type: string
+    path:
+      type: string
+      description: 路径，以 RUL 的格式返回，不能为空, 必须以 / 开头，不需要包含语言层级，比如 /zh/about 直接返回 /about
+    parentId:
+      type:
+        - string
+        - "null"
+      description: 父节点 path，如果为 null 则表示是顶级节点
+    sourceIds:
+      type: array
+      description: 关联的 dataSources 中的 sourceId，用于后续的翻译和内容生成，必须来自 datasources 中的 sourceId，不能有虚假的 id, **不能为空**
+      items:
+        type: string
+  required:
+    - title
+    - description
+    - path
+    - sourceIds

package/agents/structure-planning.yaml

@@ -0,0 +1,49 @@
+name: structure-planning
+description: 通用结构规划生成器，支持网站、文档、书籍、演示文稿等多种场景
+instructions:
+  url: ../prompts/structure-planning.md
+input_schema:
+  type: object
+  properties:
+    rules:
+      type: string
+      description: 用户的结构规划的要求
+    locale:
+      type: string
+      description: 用户语言，如 zh、en
+    datasources:
+      type: string
+      description: 结构规划的上下文，用于辅助结构规划
+    targetAudience:
+      type: string
+      description: 结构规划的目标受众
+    nodeName:
+      type: string
+      description: 结构规划的节点名称
+    glossary:
+      type: string
+      description: 术语表
+    structurePlanFeedback:
+      type: string
+      description: 结构规划的反馈
+    docsType:
+      type: string
+      description: 文档类型，支持：general、getting-started、reference、faq
+      default: general
+  required:
+    - rules
+    - datasources
+output_schema:
+  type: object
+  properties:
+    structurePlan: ./schema/structure-plan.yaml
+    structurePlanTree:
+      type: string
+      description: |
+        结构规划的树形结构，用于展示结构规划的层级关系，每一级有不同的缩进，方便用户直观查看结构规划的层级关系,参考格式：
+        ```
+        - 首页
+          - 产品
+            - 产品详情
+              - 产品介绍
+        ```

package/agents/transform-detail-datasources.mjs

@@ -0,0 +1,14 @@
+export default function transformDetailDatasources({
+  sourceIds,
+  datasourcesList,
+}) {
+  // Build a map for fast lookup
+  const dsMap = Object.fromEntries(
+    (datasourcesList || []).map((ds) => [ds.sourceId, ds.content])
+  );
+  // Collect formatted contents in order
+  const contents = (sourceIds || [])
+    .filter((id) => dsMap[id])
+    .map((id) => `// sourceId: ${id}\n${dsMap[id]}\n`);
+  return { detailDataSources: contents.join("") };
+}

package/agents/translate.yaml

@@ -0,0 +1,28 @@
+name: translate
+description: Translate the wiki article to another language
+instructions:
+  url: ../prompts/translator.md
+input_schema:
+  type: object
+  properties:
+    language:
+      type: string
+      description: Language to translate the article into (e.g., 'zh_CN' for Chinese)
+    content:
+      type: string
+      description: Content to translate
+    glossary:
+      type: string
+      description: 术语表
+  required:
+    - language
+    - content
+output_schema:
+  type: object
+  properties:
+    translation:
+      type: string
+      description: Translation of the content
+    language:
+      type: string
+      description: Language of the translation

package/aigne.yaml

@@ -0,0 +1,28 @@
+#!/usr/bin/env aigne
+
+chat_model:
+  provider: gemini
+  name: gemini-2.5-pro-preview-05-06
+  # name: gemini-2.5-flash-preview-05-20
+  temperature: 0.8
+agents:
+  - ./agents/docs-generator.yaml
+  - ./agents/structure-planning.yaml
+  - ./agents/batch-docs-detail-generator.yaml
+  - ./agents/load-sources.mjs
+  - ./agents/save-docs.mjs
+  - ./agents/translate.yaml
+  - ./agents/detail-generator-and-translate.yaml
+  - ./agents/check-detail-generated.mjs
+  - ./agents/transform-detail-datasources.mjs
+  - ./agents/batch-translate.yaml
+  - ./agents/save-single-doc.mjs
+  - ./agents/save-output.mjs
+  - ./agents/check-structure-planning.mjs
+  - ./agents/content-detail-generator.yaml
+  - ./agents/reflective-structure-planner.yaml
+  - ./agents/check-structure-planning-result.yaml
+  - ./agents/input-generator.mjs
+  - ./agents/detail-regenerator.yaml
+  - ./agents/publish-docs.mjs
+  - ./agents/format-structure-plan.mjs

package/biome.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "https://biomejs.dev/schemas/2.1.2/schema.json",
+  "vcs": {
+    "enabled": true,
+    "clientKind": "git",
+    "useIgnoreFile": true
+  },
+  "files": {
+    "ignoreUnknown": true,
+    "experimentalScannerIgnores": ["node_modules"]
+  },
+  "formatter": {
+    "enabled": true,
+    "indentStyle": "space",
+    "lineWidth": 100
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true,
+      "correctness": {
+        "useImportExtensions": "error",
+        "noUnusedVariables": "error",
+        "noUnusedImports": "error",
+        "noUnusedFunctionParameters": "error",
+        "noUnusedPrivateClassMembers": "error"
+      },
+      "suspicious": {
+        "noConfusingVoidType": "off",
+        "noExplicitAny": "off"
+      },
+      "complexity": {
+        "noForEach": "off",
+        "useLiteralKeys": "off"
+      }
+    }
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "double"
+    }
+  },
+  "assist": {
+    "enabled": true,
+    "actions": {
+      "source": {
+        "organizeImports": "on"
+      }
+    }
+  }
+}

package/docs-mcp/aigne.yaml

@@ -0,0 +1,8 @@
+chat_model:
+  provider: gemini
+  name: gemini-2.5-pro-preview-05-06
+  # name: gemini-2.5-flash-preview-05-20
+  temperature: 0.8
+agents:
+  - get-docs-structure.mjs
+  - get-docs-detail.mjs