@aigne/doc-smith 0.8.12-beta.2 → 0.8.12-beta.4

package/agents/publish/publish-docs.mjs

@@ -1,12 +1,13 @@
 import { basename, join } from "node:path";
 
 import { publishDocs as publishDocsFn } from "@aigne/publish-docs";
+import { BrokerClient } from "@blocklet/payment-broker-client/node";
 import chalk from "chalk";
 import fs from "fs-extra";
 
-import { getAccessToken } from "../../utils/auth-utils.mjs";
+import { getAccessToken, getOfficialAccessToken } from "../../utils/auth-utils.mjs";
 import {
-  DEFAULT_APP_URL,
+  CLOUD_SERVICE_URL_PROD,
   DISCUSS_KIT_STORE_URL,
   DOC_SMITH_DIR,
   TMP_DIR,
@@ -16,96 +17,112 @@ import { beforePublishHook, ensureTmpDir } from "../../utils/d2-utils.mjs";
 import { deploy } from "../../utils/deploy.mjs";
 import { getGithubRepoUrl, loadConfigFromFile, saveValueToConfig } from "../../utils/utils.mjs";
 
+const BASE_URL = process.env.DOC_SMITH_BASE_URL || CLOUD_SERVICE_URL_PROD;
+
 export default async function publishDocs(
   { docsDir: rawDocsDir, appUrl, boardId, projectName, projectDesc, projectLogo },
   options,
 ) {
-  // move work dir to tmp-dir
-  await ensureTmpDir();
-
-  const docsDir = join(DOC_SMITH_DIR, TMP_DIR, TMP_DOCS_DIR);
-  await fs.rm(docsDir, { recursive: true, force: true });
-  await fs.mkdir(docsDir, {
-    recursive: true,
-  });
-  await fs.cp(rawDocsDir, docsDir, { recursive: true });
-
-  // ----------------- trigger beforePublishHook -----------------------------
-  await beforePublishHook({ docsDir });
-
-  // ----------------- main publish process flow -----------------------------
-  // Check if DOC_DISCUSS_KIT_URL is set in environment variables
-  const envAppUrl = process.env.DOC_DISCUSS_KIT_URL;
-  const useEnvAppUrl = !!envAppUrl;
-
-  // Use environment variable if available, otherwise use the provided appUrl
-  if (useEnvAppUrl) {
-    appUrl = envAppUrl;
-  }
+  let message;
 
-  // Check if appUrl is default and not saved in config (only when not using env variable)
-  const config = await loadConfigFromFile();
-  const isDefaultAppUrl = appUrl === DEFAULT_APP_URL;
-  const hasAppUrlInConfig = config?.appUrl;
-
-  let token = "";
-
-  if (!useEnvAppUrl && isDefaultAppUrl && !hasAppUrlInConfig) {
-    const hasCachedCheckoutId = !!config?.checkoutId;
-    const choice = await options.prompts.select({
-      message: "Select platform to publish your documents:",
-      choices: [
-        {
-          name: `${chalk.blue("DocSmith Cloud (docsmith.aigne.io)")} – ${chalk.green("Free")} hosting. Your documents will be publicly accessible. Best for open-source projects or community sharing.`,
-          value: "default",
-        },
-        {
-          name: `${chalk.blue("Your existing website")} - Integrate and publish directly on your current site (setup required)`,
-          value: "custom",
-        },
-        ...(hasCachedCheckoutId
-          ? [
-              {
-                name: `${chalk.yellow("Resume previous website setup")} - ${chalk.green("Already paid.")} Continue where you left off. Your payment has already been processed.`,
-                value: "new-instance-continue",
-              },
-            ]
-          : []),
-        {
-          name: `${chalk.blue("New website")} - ${chalk.yellow("Paid service.")} We'll help you set up a brand-new website with custom domain and hosting. Great if you want a professional presence.`,
-          value: "new-instance",
-        },
-      ],
+  try {
+    // move work dir to tmp-dir
+    await ensureTmpDir();
+
+    const docsDir = join(DOC_SMITH_DIR, TMP_DIR, TMP_DOCS_DIR);
+    await fs.rm(docsDir, { recursive: true, force: true });
+    await fs.mkdir(docsDir, {
+      recursive: true,
     });
+    await fs.cp(rawDocsDir, docsDir, { recursive: true });
+
+    // ----------------- trigger beforePublishHook -----------------------------
+    await beforePublishHook({ docsDir });
+
+    // ----------------- main publish process flow -----------------------------
+    // Check if DOC_DISCUSS_KIT_URL is set in environment variables
+    const envAppUrl = process.env.DOC_DISCUSS_KIT_URL;
+    const useEnvAppUrl = !!envAppUrl;
+
+    // Use environment variable if available, otherwise use the provided appUrl
+    if (useEnvAppUrl) {
+      appUrl = envAppUrl;
+    }
+
+    // Check if appUrl is default and not saved in config (only when not using env variable)
+    const config = await loadConfigFromFile();
+    const isDefaultAppUrl = appUrl === CLOUD_SERVICE_URL_PROD;
+    const hasAppUrlInConfig = config?.appUrl;
+
+    let token = "";
+
+    if (!useEnvAppUrl && isDefaultAppUrl && !hasAppUrlInConfig) {
+      const authToken = await getOfficialAccessToken(BASE_URL);
+
+      if (!authToken) {
+        throw new Error("Failed to get official access token");
+      }
 
-    if (choice === "custom") {
-      console.log(
-        `${chalk.bold("\n💡 Tips")}\n\n` +
-          `Start here to run your own website:\n${chalk.cyan(DISCUSS_KIT_STORE_URL)}\n`,
-      );
-      const userInput = await options.prompts.input({
-        message: "Please enter your website URL:",
-        validate: (input) => {
-          try {
-            // Check if input contains protocol, if not, prepend https://
-            const urlWithProtocol = input.includes("://") ? input : `https://${input}`;
-            new URL(urlWithProtocol);
-            return true;
-          } catch {
-            return "Please enter a valid URL";
-          }
-        },
+      const client = new BrokerClient({ baseUrl: BASE_URL, authToken });
+
+      const { sessionId, paymentLink } = await client.checkCacheSession({
+        needShortUrl: true,
+        sessionId: config?.checkoutId,
+      });
+
+      const choice = await options.prompts.select({
+        message: "Select platform to publish your documents:",
+        choices: [
+          {
+            name: `${chalk.blue("DocSmith Cloud (docsmith.aigne.io)")} – ${chalk.green("Free")} hosting. Your documents will be publicly accessible. Best for open-source projects or community sharing.`,
+            value: "default",
+          },
+          {
+            name: `${chalk.blue("Your existing website")} - Integrate and publish directly on your current site (setup required)`,
+            value: "custom",
+          },
+          ...(sessionId
+            ? [
+                {
+                  name: `${chalk.yellow("Resume previous website setup")} - ${chalk.green("Already paid.")} Continue where you left off. Your payment has already been processed.`,
+                  value: "new-instance-continue",
+                },
+              ]
+            : []),
+          {
+            name: `${chalk.blue("New website")} - ${chalk.yellow("Paid service.")} We'll help you set up a brand-new website with custom domain and hosting. Great if you want a professional presence.`,
+            value: "new-instance",
+          },
+        ],
       });
-      // Ensure appUrl has protocol
-      appUrl = userInput.includes("://") ? userInput : `https://${userInput}`;
-    } else if (["new-instance", "new-instance-continue"].includes(choice)) {
-      // Deploy a new Discuss Kit service
-      try {
+
+      if (choice === "custom") {
+        console.log(
+          `${chalk.bold("\n💡 Tips")}\n\n` +
+            `Start here to run your own website:\n${chalk.cyan(DISCUSS_KIT_STORE_URL)}\n`,
+        );
+        const userInput = await options.prompts.input({
+          message: "Please enter your website URL:",
+          validate: (input) => {
+            try {
+              // Check if input contains protocol, if not, prepend https://
+              const urlWithProtocol = input.includes("://") ? input : `https://${input}`;
+              new URL(urlWithProtocol);
+              return true;
+            } catch {
+              return "Please enter a valid URL";
+            }
+          },
+        });
+        // Ensure appUrl has protocol
+        appUrl = userInput.includes("://") ? userInput : `https://${userInput}`;
+      } else if (["new-instance", "new-instance-continue"].includes(choice)) {
+        // Deploy a new Discuss Kit service
         let id = "";
         let paymentUrl = "";
         if (choice === "new-instance-continue") {
-          id = config?.checkoutId;
-          paymentUrl = config?.paymentUrl;
+          id = sessionId;
+          paymentUrl = paymentLink;
           console.log(`\nResuming your previous website setup...`);
         } else {
           console.log(`\nCreating new website for your documentation...`);
@@ -114,42 +131,35 @@ export default async function publishDocs(
 
         appUrl = homeUrl;
         token = ltToken;
-      } catch (error) {
-        const errorMsg = error?.message || "Unknown error occurred";
-        return { message: `${chalk.red("❌ Failed to publish to website:")} ${errorMsg}` };
       }
     }
-  }
-
-  console.log(`\nPublishing docs to ${chalk.cyan(appUrl)}\n`);
 
-  const accessToken = await getAccessToken(appUrl, token);
+    console.log(`\nPublishing docs to ${chalk.cyan(appUrl)}\n`);
 
-  process.env.DOC_ROOT_DIR = docsDir;
+    const accessToken = await getAccessToken(appUrl, token);
 
-  const sidebarPath = join(docsDir, "_sidebar.md");
+    process.env.DOC_ROOT_DIR = docsDir;
 
-  // Get project info from config
-  const projectInfo = {
-    name: projectName || config?.projectName || basename(process.cwd()),
-    description: projectDesc || config?.projectDesc || "",
-    icon: projectLogo || config?.projectLogo || "",
-  };
+    const sidebarPath = join(docsDir, "_sidebar.md");
 
-  // Construct boardMeta object
-  const boardMeta = {
-    category: config?.documentPurpose || [],
-    githubRepoUrl: getGithubRepoUrl(),
-    commitSha: config?.lastGitHead || "",
-    languages: [
-      ...(config?.locale ? [config.locale] : []),
-      ...(config?.translateLanguages || []),
-    ].filter((lang, index, arr) => arr.indexOf(lang) === index), // Remove duplicates
-  };
+    // Get project info from config
+    const projectInfo = {
+      name: projectName || config?.projectName || basename(process.cwd()),
+      description: projectDesc || config?.projectDesc || "",
+      icon: projectLogo || config?.projectLogo || "",
+    };
 
-  let message;
+    // Construct boardMeta object
+    const boardMeta = {
+      category: config?.documentPurpose || [],
+      githubRepoUrl: getGithubRepoUrl(),
+      commitSha: config?.lastGitHead || "",
+      languages: [
+        ...(config?.locale ? [config.locale] : []),
+        ...(config?.translateLanguages || []),
+      ].filter((lang, index, arr) => arr.indexOf(lang) === index), // Remove duplicates
+    };
 
-  try {
     const {
       success,
       boardId: newBoardId,
@@ -184,16 +194,25 @@ export default async function publishDocs(
     } else {
       // If the error is 401 or 403, it means the access token is invalid
       if (error?.includes("401") || error?.includes("403")) {
-        message = `❌ Publishing failed: you don’t have valid authorization.\n   Run ${chalk.cyan("aigne doc clear")} to reset it, then publish again.`;
+        message = `❌ Publishing failed: you don't have valid authorization.\n   Run ${chalk.cyan("aigne doc clear")} to reset it, then publish again.`;
       }
     }
+
+    // clean up tmp work dir
+    await fs.rm(docsDir, { recursive: true, force: true });
   } catch (error) {
     message = `❌ Failed to publish docs: ${error.message}`;
+
+    // clean up tmp work dir in case of error
+    try {
+      const docsDir = join(DOC_SMITH_DIR, TMP_DIR, TMP_DOCS_DIR);
+      await fs.rm(docsDir, { recursive: true, force: true });
+    } catch {
+      // ignore cleanup errors
+    }
   }
-  await saveValueToConfig("checkoutId", "", "Checkout ID for document deployment service");
 
-  // clean up tmp work dir
-  await fs.rm(docsDir, { recursive: true, force: true });
+  await saveValueToConfig("checkoutId", "", "Checkout ID for document deployment service");
   return message ? { message } : {};
 }
 
@@ -207,7 +226,7 @@ publishDocs.input_schema = {
     appUrl: {
       type: "string",
       description: "The url of the app",
-      default: DEFAULT_APP_URL,
+      default: CLOUD_SERVICE_URL_PROD,
     },
     boardId: {
       type: "string",

package/agents/translate/index.yaml

@@ -37,7 +37,7 @@ skills:
         default_input:
           isTranslate: true
     iterate_on: selectedDocs
-    concurrency: 3
+    concurrency: 10
   - url: ../utils/check-feedback-refiner.mjs
     default_input:
       stage: translation_refine

package/agents/update/batch-generate-document.yaml

@@ -15,5 +15,5 @@ input_schema:
       items: { type: string }
       description: Array of modified files since last generation
 iterate_on: documentExecutionStructure
-concurrency: 3
+concurrency: 10
 mode: sequential

package/agents/update/batch-update-document.yaml

@@ -4,4 +4,4 @@ name: batchUpdateDocument
 skills:
   - ../update/handle-document-update.yaml
 iterate_on: selectedDocs
-concurrency: 5
+concurrency: 10

package/agents/update/check-document.mjs

@@ -2,7 +2,6 @@ import { access, readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { TeamAgent } from "@aigne/core";
-import { hasSourceFilesChanged } from "../../utils/utils.mjs";
 import checkDetailResult from "../utils/check-detail-result.mjs";
 
 // Get current script directory
@@ -17,13 +16,14 @@ export default async function checkDocument(
     documentStructure,
     modifiedFiles,
     forceRegenerate,
+    locale,
     ...rest
   },
   options,
 ) {
   // Check if the detail file already exists
   const flatName = path.replace(/^\//, "").replace(/\//g, "-");
-  const fileFullName = `${flatName}.md`;
+  const fileFullName = locale === "en" ? `${flatName}.md` : `${flatName}.${locale}.md`;
   const filePath = join(docsDir, fileFullName);
   let detailGenerated = true;
   let fileContent = null;
@@ -69,16 +69,6 @@ export default async function checkDocument(
     }
   }
 
-  // Check if source files have changed since last generation
-  let sourceFilesChanged = false;
-  if (sourceIds && sourceIds.length > 0 && modifiedFiles) {
-    sourceFilesChanged = hasSourceFilesChanged(sourceIds, modifiedFiles);
-
-    if (sourceFilesChanged) {
-      console.log(`Source files changed for ${path}, will regenerate`);
-    }
-  }
-
   // If file exists, check content validation
   let contentValidationFailed = false;
   let validationResult = {};
@@ -95,13 +85,7 @@ export default async function checkDocument(
   }
 
   // If file exists, sourceIds haven't changed, source files haven't changed, and content validation passes, no need to regenerate
-  if (
-    detailGenerated &&
-    !sourceIdsChanged &&
-    !sourceFilesChanged &&
-    !contentValidationFailed &&
-    !forceRegenerate
-  ) {
+  if (detailGenerated && !sourceIdsChanged && !contentValidationFailed && !forceRegenerate) {
     return {
       path,
       docsDir,

package/agents/update/user-review-document.mjs

@@ -175,9 +175,10 @@ export default async function userReviewDocument(
     const feedback = await options.prompts.input({
       message:
         "How would you like to improve this document?\n" +
-        "  • Add, modify, or remove content\n" +
-        "  • Improve clarity, accuracy, or completeness\n" +
-        "  • Adjust tone, style, or technical level\n\n" +
+        "Examples:\n" +
+        "  • Add troubleshooting section for common errors\n" +
+        "  • Simplify the explanation for beginners\n" +
+        "  • Remove the outdated information about version 1.0\n\n" +
         "  Your feedback:",
     });
 

package/agents/utils/load-sources.mjs

@@ -1,15 +1,21 @@
-import { readFile, stat } from "node:fs/promises";
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import {
-  DEFAULT_EXCLUDE_PATTERNS,
-  DEFAULT_INCLUDE_PATTERNS,
-} from "../../utils/constants/index.mjs";
-import { getFilesWithGlob, loadGitignore } from "../../utils/file-utils.mjs";
+  buildSourcesContent,
+  calculateFileStats,
+  loadFilesFromPaths,
+  readFileContents,
+} from "../../utils/file-utils.mjs";
 import {
   getCurrentGitHead,
   getModifiedFilesBetweenCommits,
-  isGlobPattern,
+  toRelativePath,
 } from "../../utils/utils.mjs";
+import {
+  INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD,
+  DEFAULT_EXCLUDE_PATTERNS,
+  DEFAULT_INCLUDE_PATTERNS,
+} from "../../utils/constants/index.mjs";
 
 export default async function loadSources({
   sources = [],
@@ -27,103 +33,22 @@ export default async function loadSources({
   let files = Array.isArray(sources) ? [...sources] : [];
 
   if (sourcesPath) {
-    const paths = Array.isArray(sourcesPath) ? sourcesPath : [sourcesPath];
-    let allFiles = [];
-
-    for (const dir of paths) {
-      try {
-        if (typeof dir !== "string") {
-          console.warn(`Invalid source path: ${dir}`);
-          continue;
-        }
-
-        // First try to access as a file or directory
-        const stats = await stat(dir);
-
-        if (stats.isFile()) {
-          // If it's a file, add it directly without filtering
-          allFiles.push(dir);
-        } else if (stats.isDirectory()) {
-          // If it's a directory, use the existing glob logic
-          // Load .gitignore for this directory
-          const gitignorePatterns = await loadGitignore(dir);
-
-          // Prepare patterns
-          let finalIncludePatterns = null;
-          let finalExcludePatterns = null;
-
-          if (useDefaultPatterns) {
-            // Merge with default patterns
-            const userInclude = includePatterns
-              ? Array.isArray(includePatterns)
-                ? includePatterns
-                : [includePatterns]
-              : [];
-            const userExclude = excludePatterns
-              ? Array.isArray(excludePatterns)
-                ? excludePatterns
-                : [excludePatterns]
-              : [];
-
-            finalIncludePatterns = [...DEFAULT_INCLUDE_PATTERNS, ...userInclude];
-            finalExcludePatterns = [...DEFAULT_EXCLUDE_PATTERNS, ...userExclude];
-          } else {
-            // Use only user patterns
-            if (includePatterns) {
-              finalIncludePatterns = Array.isArray(includePatterns)
-                ? includePatterns
-                : [includePatterns];
-            }
-            if (excludePatterns) {
-              finalExcludePatterns = Array.isArray(excludePatterns)
-                ? excludePatterns
-                : [excludePatterns];
-            }
-          }
-
-          // Get files using glob
-          const filesInDir = await getFilesWithGlob(
-            dir,
-            finalIncludePatterns,
-            finalExcludePatterns,
-            gitignorePatterns,
-          );
-          allFiles = allFiles.concat(filesInDir);
-        }
-      } catch (err) {
-        if (err.code === "ENOENT") {
-          // Path doesn't exist as file or directory, try as glob pattern
-          try {
-            // Check if it looks like a glob pattern
-            const isGlobPatternResult = isGlobPattern(dir);
-
-            if (isGlobPatternResult) {
-              // Use glob to find matching files from current working directory
-              const { glob } = await import("glob");
-              const matchedFiles = await glob(dir, {
-                absolute: true,
-                nodir: true, // Only files, not directories
-                dot: false, // Don't include hidden files
-              });
-
-              if (matchedFiles.length > 0) {
-                allFiles = allFiles.concat(matchedFiles);
-              }
-            }
-          } catch (globErr) {
-            console.warn(`Failed to process glob pattern "${dir}": ${globErr.message}`);
-          }
-        } else {
-          throw err;
-        }
-      }
-    }
+    const allFiles = await loadFilesFromPaths(sourcesPath, {
+      includePatterns,
+      excludePatterns,
+      useDefaultPatterns,
+      defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
+      defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
+    });
 
     files = files.concat(allFiles);
   }
 
   files = [...new Set(files)];
 
+  // all files path
+  const allFilesPaths = files.map((file) => `- ${toRelativePath(file)}`).join("\n");
+
   // Define media file extensions
   const mediaExtensions = [
     ".jpg",
@@ -142,38 +67,40 @@ export default async function loadSources({
   ];
 
   // Separate source files from media files
-  const sourceFiles = [];
+  const sourceFilesPaths = [];
   const mediaFiles = [];
-  let allSources = "";
 
-  await Promise.all(
-    files.map(async (file) => {
-      const ext = path.extname(file).toLowerCase();
+  for (const file of files) {
+    const ext = path.extname(file).toLowerCase();
+
+    if (mediaExtensions.includes(ext)) {
+      // This is a media file
+      const relativePath = path.relative(docsDir, file);
+      const fileName = path.basename(file);
+      const description = path.parse(fileName).name;
+
+      mediaFiles.push({
+        name: fileName,
+        path: relativePath,
+        description,
+      });
+    } else {
+      // This is a source file
+      sourceFilesPaths.push(file);
+    }
+  }
+
+  // Read all source files using the utility function
+  const sourceFiles = await readFileContents(sourceFilesPaths, process.cwd());
 
-      if (mediaExtensions.includes(ext)) {
-        // This is a media file
-        const relativePath = path.relative(docsDir, file);
-        const fileName = path.basename(file);
-        const description = path.parse(fileName).name;
+  // Count tokens and lines using utility function
+  const { totalTokens, totalLines } = calculateFileStats(sourceFiles);
 
-        mediaFiles.push({
-          name: fileName,
-          path: relativePath,
-          description,
-        });
-      } else {
-        // This is a source file
-        const content = await readFile(file, "utf8");
-        const relativePath = path.relative(process.cwd(), file);
-        allSources += `// sourceId: ${relativePath}\n${content}\n`;
+  // check if totalTokens is too large
+  const isLargeContext = totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD;
 
-        sourceFiles.push({
-          sourceId: relativePath,
-          content,
-        });
-      }
-    }),
-  );
+  // Build allSources string using utility function
+  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
 
   // Get the last documentation structure
   let originalDocumentStructure;
@@ -283,31 +210,17 @@ export default async function loadSources({
     assetsContent += "```\n";
   }
 
-  // Count words and lines in allSources
-  let totalWords = 0;
-  let totalLines = 0;
-
-  for (const source of Object.values(allSources)) {
-    if (typeof source === "string") {
-      // Count English words (simple regex for words containing a-zA-Z)
-      const words = source.match(/[a-zA-Z]+/g) || [];
-      totalWords += words.length;
-
-      // Count lines (excluding empty lines)
-      totalLines += source.split("\n").filter((line) => line.trim() !== "").length;
-    }
-  }
-
   return {
-    datasourcesList: sourceFiles,
     datasources: allSources,
     content,
     originalDocumentStructure,
     files,
     modifiedFiles,
-    totalWords,
+    totalTokens,
     totalLines,
     assetsContent,
+    isLargeContext,
+    allFilesPaths,
   };
 }
 
@@ -357,16 +270,6 @@ loadSources.output_schema = {
     datasources: {
       type: "string",
     },
-    datasourcesList: {
-      type: "array",
-      items: {
-        type: "object",
-        properties: {
-          sourceId: { type: "string" },
-          content: { type: "string" },
-        },
-      },
-    },
     files: {
       type: "array",
       items: { type: "string" },