@aigne/doc-smith 0.8.15-beta.1 → 0.8.15-beta.11

package/utils/history-utils.mjs

@@ -99,7 +99,7 @@ function recordUpdateGit({ feedback }) {
 /**
  * Records an update in the YAML file.
  */
-function recordUpdateYaml({ operation, feedback, documentPath = null }) {
+function recordUpdateYaml({ operation, feedback, docPaths = null }) {
   try {
     const docSmithDir = join(process.cwd(), DOC_SMITH_DIR);
     if (!existsSync(docSmithDir)) {
@@ -125,9 +125,9 @@ function recordUpdateYaml({ operation, feedback, documentPath = null }) {
       feedback,
     };
 
-    // Add document path if provided
-    if (documentPath) {
-      entry.documentPath = documentPath;
+    // Add document paths if provided
+    if (Array.isArray(docPaths) && docPaths.length > 0) {
+      entry.docPaths = docPaths;
     }
 
     // Add to beginning (newest first)
@@ -153,14 +153,14 @@ function recordUpdateYaml({ operation, feedback, documentPath = null }) {
  * @param {Object} params
  * @param {string} params.operation - The type of operation (e.g., 'document_update', 'structure_update', 'translation_update').
  * @param {string} params.feedback - The user's feedback text.
- * @param {string} params.documentPath - The document path.
+ * @param {string[]} [params.docPaths] - Document path list for updates.
  */
-export function recordUpdate({ operation, feedback, documentPath = null }) {
+export function recordUpdate({ operation, feedback, docPaths = null }) {
   // Skip if no feedback
   if (!feedback?.trim()) return;
 
   // Always record in YAML
-  recordUpdateYaml({ operation, feedback, documentPath });
+  recordUpdateYaml({ operation, feedback, docPaths });
 
   // Also record in git if git is available and not in a git repository
   if (isGitAvailable() && !isInGitRepository(process.cwd())) {
@@ -183,7 +183,19 @@ export function getHistory() {
 
   try {
     const content = readFileSync(historyPath, "utf8");
-    return parse(content) || { entries: [] };
+    const parsed = parse(content) || { entries: [] };
+    const entries = Array.isArray(parsed.entries) ? parsed.entries : [];
+
+    const normalized = entries.map((entry) => {
+      if (!entry) return entry;
+      // Normalize legacy entries: documentPath -> docPaths: [documentPath]
+      if (!entry.docPaths && entry.documentPath) {
+        return { ...entry, docPaths: [entry.documentPath] };
+      }
+      return entry;
+    });
+
+    return { ...parsed, entries: normalized };
   } catch (error) {
     console.warn("Could not read the history:", error.message);
     return { entries: [] };

package/utils/load-config.mjs

@@ -1,7 +1,10 @@
 import fs from "node:fs/promises";
+import chalk from "chalk";
 import path from "node:path";
 import { parse } from "yaml";
 import { processConfigFields, resolveFileReferences } from "./utils.mjs";
+import { DEFAULT_EXCLUDE_PATTERNS } from "./constants/index.mjs";
+import { findInvalidSourcePaths, toDisplayPath } from "./file-utils.mjs";
 
 export default async function loadConfig({ config, appUrl }) {
   const configPath = path.isAbsolute(config) ? config : path.join(process.cwd(), config);
@@ -28,7 +31,23 @@ export default async function loadConfig({ config, appUrl }) {
     }
 
     // Parse new configuration fields and convert keys to actual content
-    const processedConfig = processConfigFields(parsedConfig);
+    const processedConfig = await processConfigFields(parsedConfig);
+
+    // Validate sourcePaths against exclude patterns
+    const sourcesPath = processedConfig.sourcesPath || parsedConfig.sourcesPath;
+    if (sourcesPath) {
+      const excludePatterns = [
+        ...DEFAULT_EXCLUDE_PATTERNS,
+        ...(processedConfig.excludePatterns || parsedConfig.excludePatterns || []),
+      ];
+
+      const invalidPaths = await findInvalidSourcePaths(sourcesPath, excludePatterns);
+      if (invalidPaths.length > 0) {
+        console.warn(
+          `⚠️  Some source paths have been excluded and will not be processed:\n${invalidPaths.map((p) => `  - ${chalk.yellow(p)}`).join("\n")}\n💡 Tip: You can remove these paths in ${toDisplayPath(configPath)}\n`,
+        );
+      }
+    }
 
     return {
       lastGitHead: parsedConfig.lastGitHead || "",

package/utils/markdown-checker.mjs

@@ -3,9 +3,12 @@ import path from "node:path";
 import remarkGfm from "remark-gfm";
 import remarkLint from "remark-lint";
 import remarkParse from "remark-parse";
+import { isRelative } from "ufo";
 import { unified } from "unified";
 import { visit } from "unist-util-visit";
 import { VFile } from "vfile";
+
+import { isRemoteFile, isRemoteFileAvailable } from "./file-utils.mjs";
 import { validateMermaidSyntax } from "./mermaid-validator.mjs";
 
 /**
@@ -232,6 +235,34 @@ function checkLocalImages(markdown, source, errorMessages, markdownFilePath, bas
   }
 }
 
+async function checkRemoteImages(markdown, source, errorMessages) {
+  const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
+  let match;
+
+  while (true) {
+    match = imageRegex.exec(markdown);
+    if (match === null) break;
+    const imagePath = match[2].trim();
+    const altText = match[1];
+
+    if (isRelative(imagePath)) continue;
+    if (imagePath.startsWith("/")) continue;
+
+    // Skip data URLs
+    if (/^data:/.test(imagePath)) continue;
+
+    if (isRemoteFile(imagePath)) {
+      const isAvailable = await isRemoteFileAvailable(imagePath);
+      if (isAvailable) continue;
+      else {
+        errorMessages.push(
+          `Found invalid remote image in ${source}: ![${altText}](${imagePath}) - only valid media resources can be used`,
+        );
+      }
+    }
+  }
+}
+
 /**
  * Check content structure and formatting issues
  * @param {string} markdown - The markdown content
@@ -370,7 +401,10 @@ export async function checkMarkdown(markdown, source = "content", options = {})
     // 2. Check local images existence
     checkLocalImages(markdown, source, errorMessages, filePath, baseDir);
 
-    // 3. Check content structure and formatting issues
+    // 3. Check remote images existence
+    await checkRemoteImages(markdown, source, errorMessages);
+
+    // 4. Check content structure and formatting issues
     checkContentStructure(markdown, source, errorMessages);
 
     // Check mermaid code blocks and other custom validations

package/utils/utils.mjs

@@ -18,6 +18,7 @@ import {
   SUPPORTED_LANGUAGES,
   TARGET_AUDIENCES,
 } from "./constants/index.mjs";
+import { isRemoteFile, getRemoteFileContent } from "./file-utils.mjs";
 
 /**
  * Normalize path to absolute path for consistent comparison
@@ -47,16 +48,6 @@ export function isGlobPattern(pattern) {
   return /[*?[\]]|(\*\*)/.test(pattern);
 }
 
-/**
- * Check if a string is an HTTP/HTTPS URL
- * @param {string} url - The string to check
- * @returns {boolean} - True if the string starts with http:// or https://
- */
-export function isHttp(url) {
-  if (typeof url !== "string") return false;
-  return url.startsWith("http://") || url.startsWith("https://");
-}
-
 export function processContent({ content }) {
   // Match markdown regular links [text](link), exclude images ![text](link)
   return content.replace(/(?<!!)\[([^\]]+)\]\(([^)]+)\)/g, (match, text, link) => {
@@ -81,77 +72,72 @@ export function processContent({ content }) {
   });
 }
 
+// Helper function to generate filename based on language
+export function getFileName(docPath, language) {
+  // Flatten path: remove leading /, replace all / with -
+  const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
+  const isEnglish = language === "en";
+  return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
+}
+
 /**
- * Save a single document and its translations to files
+ * Save a single document to files
  * @param {Object} params
  * @param {string} params.path - Relative path (without extension)
  * @param {string} params.content - Main document content
  * @param {string} params.docsDir - Root directory
  * @param {string} params.locale - Main content language (e.g., 'en', 'zh', 'fr')
- * @param {Array<{language: string, translation: string}>} [params.translates] - Translation content
  * @param {Array<string>} [params.labels] - Document labels for front matter
- * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
+ * @returns {Promise<{ path: string, success: boolean, error?: string }>}
  */
-export async function saveDocWithTranslations({
-  path: docPath,
-  content,
-  docsDir,
-  locale,
-  translates = [],
-  labels,
-  isTranslate = false,
-}) {
-  const results = [];
+export async function saveDoc({ path: docPath, content, docsDir, locale, labels }) {
   try {
-    // Flatten path: remove leading /, replace all / with -
-    const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
     await fs.mkdir(docsDir, { recursive: true });
+    const mainFileName = getFileName(docPath, locale);
+    const mainFilePath = path.join(docsDir, mainFileName);
 
-    // Helper function to generate filename based on language
-    const getFileName = (language) => {
-      const isEnglish = language === "en";
-      return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
-    };
-
-    // Save main content with appropriate filename based on locale (skip if isTranslate is true)
-    if (!isTranslate) {
-      const mainFileName = getFileName(locale);
-      const mainFilePath = path.join(docsDir, mainFileName);
-
-      // Add labels front matter if labels are provided
-      let finalContent = processContent({ content });
+    // Add labels front matter if labels are provided
+    let finalContent = processContent({ content });
 
-      if (labels && labels.length > 0) {
-        const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
-        finalContent = frontMatter + finalContent;
-      }
-
-      await fs.writeFile(mainFilePath, finalContent, "utf8");
-      results.push({ path: mainFilePath, success: true });
+    if (labels && labels.length > 0) {
+      const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
+      finalContent = frontMatter + finalContent;
     }
 
-    // Process all translations
-    for (const translate of translates) {
-      const translateFileName = getFileName(translate.language);
-      const translatePath = path.join(docsDir, translateFileName);
+    await fs.writeFile(mainFilePath, finalContent, "utf8");
+    return { path: mainFilePath, success: true };
+  } catch (err) {
+    return { path: docPath, success: false, error: err.message };
+  }
+}
 
-      // Add labels front matter to translation content if labels are provided
-      let finalTranslationContent = processContent({
-        content: translate.translation,
-      });
+export async function saveDocTranslation({
+  path: docPath,
+  docsDir,
+  translation,
+  language,
+  labels,
+}) {
+  try {
+    await fs.mkdir(docsDir, { recursive: true });
+    const translateFileName = getFileName(docPath, language);
+    const translatePath = path.join(docsDir, translateFileName);
 
-      if (labels && labels.length > 0) {
-        const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
-        finalTranslationContent = frontMatter + finalTranslationContent;
-      }
+    // Add labels front matter to translation content if labels are provided
+    let finalTranslationContent = processContent({
+      content: translation,
+    });
 
-      await fs.writeFile(translatePath, finalTranslationContent, "utf8");
-      results.push({ path: translatePath, success: true });
+    if (labels && labels.length > 0) {
+      const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
+      finalTranslationContent = frontMatter + finalTranslationContent;
     }
+
+    await fs.writeFile(translatePath, finalTranslationContent, "utf8");
+    return { path: translatePath, success: true };
   } catch (err) {
-    results.push({ path: docPath, success: false, error: err.message });
+    return { path: docPath, success: false, error: err.message };
   }
-  return results;
 }
 
 /**
@@ -963,7 +949,7 @@ export function processTargetAudience(targetAudienceTypes, existingTargetAudienc
  * @param {Object} config - Parsed configuration
  * @returns {Object} Processed configuration with content fields
  */
-export function processConfigFields(config) {
+export async function processConfigFields(config) {
   const processed = {};
   const allRulesContent = [];
 
@@ -995,7 +981,15 @@ export function processConfigFields(config) {
     if (typeof config.rules === "string") {
       const existingRules = config.rules.trim();
       if (existingRules) {
-        allRulesContent.push(existingRules);
+        // load rules from remote url
+        if (isRemoteFile(existingRules)) {
+          const remoteFileContent = await getRemoteFileContent(existingRules);
+          if (remoteFileContent) {
+            allRulesContent.push(remoteFileContent);
+          }
+        } else {
+          allRulesContent.push(existingRules);
+        }
       }
     } else if (Array.isArray(config.rules)) {
       // Handle array of rules - join them with newlines
@@ -1054,6 +1048,12 @@ export function processConfigFields(config) {
     }
   }
 
+  if (config.glossary) {
+    if (isRemoteFile(config.glossary)) {
+      processed.glossary = await getRemoteFileContent(config.glossary);
+    }
+  }
+
   // Detect and handle conflicts in user selections
   const conflicts = detectResolvableConflicts(config);
   if (conflicts.length > 0) {
@@ -1097,8 +1097,10 @@ export function processConfigFields(config) {
  * @returns {Promise<any>} - The processed configuration with file content loaded in place of references.
  */
 export async function resolveFileReferences(obj, basePath = process.cwd()) {
-  if (typeof obj === "string" && obj.startsWith("@")) {
-    return await loadFileContent(obj.slice(1), basePath);
+  if (typeof obj === "string") {
+    if (obj.startsWith("@")) {
+      return await loadFileContent(obj.slice(1), basePath);
+    }
   }
 
   if (Array.isArray(obj)) {

package/agents/generate/document-structure-tools/generate-sub-structure.mjs

@@ -1,131 +0,0 @@
-import {
-  buildSourcesContent,
-  calculateFileStats,
-  loadFilesFromPaths,
-  readFileContents,
-} from "../../../utils/file-utils.mjs";
-import {
-  INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD,
-  DEFAULT_EXCLUDE_PATTERNS,
-  DEFAULT_INCLUDE_PATTERNS,
-} from "../../../utils/constants/index.mjs";
-import { toRelativePath } from "../../../utils/utils.mjs";
-
-export default async function generateSubStructure(
-  {
-    parentDocument,
-    subSourcePaths,
-    includePatterns,
-    excludePatterns,
-    useDefaultPatterns = true,
-    ...rest
-  },
-  options,
-) {
-  const sourcePaths = subSourcePaths?.map((item) => item.path);
-  if (!sourcePaths || sourcePaths.length === 0) {
-    return {
-      subStructure: [],
-    };
-  }
-
-  let files = await loadFilesFromPaths(sourcePaths, {
-    includePatterns,
-    excludePatterns,
-    useDefaultPatterns,
-    defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
-    defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
-  });
-  files = [...new Set(files)];
-
-  // all files path
-  const allFilesPaths = files.map((file) => `- ${toRelativePath(file)}`).join("\n");
-
-  // Read all source files using the utility function
-  const sourceFiles = await readFileContents(files, process.cwd());
-
-  // Count tokens and lines using utility function
-  const { totalTokens } = calculateFileStats(sourceFiles);
-
-  // check if totalTokens is too large
-  let isLargeContext = false;
-  if (totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD) {
-    isLargeContext = true;
-  }
-
-  // Build allSources string using utility function
-  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
-
-  // Performance optimization:
-  // Using both structured output and Tool with Gemini model causes redundant calls
-  // Only use Tool when context is very large
-  const generateStructureAgent = isLargeContext
-    ? options.context.agents["generateStructure"]
-    : options.context.agents["generateStructureWithoutTools"];
-  const result = await options.context.invoke(generateStructureAgent, {
-    ...rest,
-    isSubStructure: true,
-    parentDocument,
-    datasources: allSources,
-    allFilesPaths,
-    isLargeContext,
-    files,
-    totalTokens,
-  });
-
-  return {
-    subStructure: result.documentStructure || [],
-    message: `Generated a sub structure for '${parentDocument.path}' successfully. Please merge all sub-structures to output the complete document structure.`,
-  };
-}
-
-generateSubStructure.description = `
-Generates a sub-structure. 
-Handles large file sets by splitting them into smaller sub-document structures when the context size exceeds limits. This approach ensures more focused and complete documentation generation.
-`;
-
-generateSubStructure.inputSchema = {
-  type: "object",
-  properties: {
-    parentDocument: {
-      type: "object",
-      description: "The parent node to generate a sub structure for",
-      properties: {
-        title: { type: "string", description: "The title of the parent node" },
-        description: { type: "string", description: "The description of the parent node" },
-        path: {
-          type: "string",
-          description:
-            "The path of the parent node, Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about ",
-        },
-        parentId: { type: "string", description: "The parent ID of the parent node" },
-        sourceIds: { type: "array", description: "The source IDs of the parent node" },
-      },
-    },
-    subSourcePaths: {
-      type: "array",
-      description: "The source paths of the sub structure",
-      items: {
-        type: "object",
-        properties: {
-          path: { type: "string", description: "The source path of the sub structure" },
-          reason: { type: "string", description: "The reason for selecting the source path" },
-        },
-        required: ["path", "reason"],
-      },
-    },
-  },
-};
-
-generateSubStructure.outputSchema = {
-  type: "object",
-  properties: {
-    subStructure: {
-      type: "array",
-      description:
-        "The sub structure of the parent node, need merge all sub-structures and output the complete document structure.",
-    },
-    message: { type: "string", description: "The message of the sub structure" },
-  },
-  required: ["subStructure"],
-};

package/agents/generate/generate-structure-without-tools.yaml

@@ -1,65 +0,0 @@
-name: generateStructureWithoutTools
-description: Generate the structure and organization of your documentation
-instructions:
-  - role: system
-    url: ../../prompts/structure/generate/system-prompt.md
-  - role: user
-    url: ../../prompts/structure/generate/user-prompt.md
-task_render_mode: collapse
-task_title: Generate the structure of the documentation
-input_schema:
-  type: object
-  properties:
-    rules:
-      type: string
-      description: Your specific requirements for documentation structure
-    locale:
-      type: string
-      description: Primary language for documentation (e.g., zh, en, ja)
-    datasources:
-      type: string
-      description: Project content and context to help generate documentation structure
-    targetAudience:
-      type: string
-      description: Target audience for the documentation
-    nodeName:
-      type: string
-      description: Specific section or page name to focus on
-    glossary:
-      type: string
-      description: Glossary for consistent terminology
-    feedback:
-      type: string
-      description: Tell us how to improve the documentation structure
-    userPreferences:
-      type: string
-      description: Your saved preferences for structure and documentation style
-    docsType:
-      type: string
-      description: "Documentation type (options: general, getting-started, reference, faq)"
-      default: general
-  required:
-    - rules
-    - datasources
-output_schema:
-  type: object
-  properties:
-    projectName:
-      type: string
-      description: Project name identified from your content sources
-    projectDesc:
-      type: string
-      description: Brief project description generated from content analysis (under 50 words)
-    documentStructure: ../schema/document-structure.yaml
-    documentStructureTree:
-      type: string
-      description: |
-        Visual tree structure showing documentation hierarchy with indented levels for easy review:
-        ```
-        - Home
-          - Getting Started
-            - Installation
-              - Requirements
-        ```
-  required:
-    - documentStructure

package/prompts/common/document/media-handling-rules.md

@@ -1,9 +0,0 @@
-<media_handling_rules>
-Media resource usage rules:
-
-- When DataSources contain media resource files, incorporate them appropriately in the generated content
-- Media resources are provided in markdown format, example: ![Resource description](https://xxxx)
-- Display images in markdown format within generated results
-- Based on resource descriptions, place images strategically in contextually relevant positions to enhance the presentation
-- To ensure correct media resource paths, **only use media resources provided in media_list or remote URL media resources**
-</media_handling_rules>