@aigne/doc-smith 0.9.8-alpha.3 → 0.9.8-alpha.5

package/agents/utils/load-all-document-content.mjs

@@ -1,30 +0,0 @@
-import pMap from "p-map";
-import { findItemByPath } from "../../utils/docs-finder-utils.mjs";
-
-/**
- * Loads all document content from the file system based on the provided document structure.
- *
- * @async
- * @function loadAllDocumentContent
- * @param {Object} params - The parameters object
- * @param {string} params.docsDir - The root directory path where documents are located
- * @param {Array<Object>} params.documentStructure - The document structure array containing items with path information
- * @param {string} params.documentStructure[].path - The file path of each document item
- * @returns {Promise<Array>} returns.allDocumentContentList - An array of document content items loaded from the file system
- * @example
- * const result = await loadAllDocumentContent({
- *   docsDir: './docs',
- *   documentStructure: [{ path: 'readme.md' }, { path: 'guide.md' }]
- * });
- */
-export default async function loadAllDocumentContent({ docsDir, documentStructure = [] }) {
-  const allDocumentContentList = await pMap(documentStructure, async (item) => {
-    const itemResult = await findItemByPath(documentStructure, item.path, null, docsDir);
-    return itemResult;
-  });
-  return {
-    allDocumentContentList,
-  };
-}
-
-loadAllDocumentContent.taskRenderMode = "hide";

package/agents/utils/load-document-all-content.mjs

@@ -1,84 +0,0 @@
-import { readdirSync } from "node:fs";
-import { findItemByPath, readFileContent } from "../../utils/docs-finder-utils.mjs";
-
-/**
- * Loads a document's content along with all its translations from the docs directory.
- *
- * This function finds a document by its path in the documentation structure, then searches
- * for all translation files in the docs directory that match the document's naming pattern.
- * Translation files are identified by the pattern: {flatName}.{language-code}.md
- *
- * @param {Object} params - The parameters object
- * @param {string} params.path - The document path to find in the structure
- * @param {string} params.docsDir - The directory containing document files and translations
- * @param {Object} params.documentStructure - The documentation structure object to search in
- * @returns {Promise<Object>} An object containing the document data with translations
- * @throws {Error} Throws an error if the document path is not found in the structure
- */
-export default async function loadDocumentAllContent({ path, docsDir, documentStructure }) {
-  // Find the document item by path in the documentation structure
-  const result = await findItemByPath(documentStructure, path, null, docsDir);
-
-  if (!result) {
-    throw new Error(`Document with path "${path}" not found in documentStructure`);
-  }
-
-  // Convert path to flat filename format (remove leading slash, replace slashes with dashes)
-  // e.g., "/api/users" becomes "api-users"
-  const flatName = result.path.replace(/^\//, "").replace(/\//g, "-");
-
-  // Arrays to store translation data in different formats
-  const translations = [];
-  const translationsString = [];
-
-  try {
-    // Read all files in the docs directory
-    const files = readdirSync(docsDir);
-
-    // Filter files to find translation files matching the pattern:
-    // - Starts with the flat name
-    // - Ends with .md
-    // - Is not the main document file (flatName.md)
-    // - Matches language pattern: .{language-code}.md (e.g., .en.md, .zh-CN.md)
-    const translationFiles = files.filter(
-      (file) =>
-        file.startsWith(`${flatName}.`) &&
-        file.endsWith(".md") &&
-        file !== `${flatName}.md` &&
-        file.match(/\.\w+(-\w+)?\.md$/),
-    );
-
-    // Process each translation file
-    for (const file of translationFiles) {
-      const content = await readFileContent(docsDir, file);
-      if (content) {
-        // Extract language code from filename (e.g., "en" from "doc.en.md" or "zh-CN" from "doc.zh-CN.md")
-        const langMatch = file.match(/\.(\w+(-\w+)?)\.md$/);
-        if (langMatch) {
-          const language = langMatch[1];
-          // Store translation in structured format
-          translations.push({
-            language,
-            translation: content,
-          });
-
-          // Store translation in XML-like string format for prompt templates
-          translationsString.push(`<${language}>\n${content}\n</${language}>`);
-        }
-      }
-    }
-  } catch (error) {
-    console.warn(`⚠️  Could not read translation files from ${docsDir}:`, error.message);
-  }
-
-  // Return the original document result enhanced with translation data
-  return {
-    ...result,
-    // FIXME: @zhanghan use anthoer way to evaluate translationQuality
-    // translates: translations, // Array of translation objects with language and content
-    translationsString: translationsString.join("\n\n"), // Combined translations as formatted string
-  };
-}
-
-// Hide this function from task rendering in the UI
-loadDocumentAllContent.taskRenderMode = "hide";

package/agents/utils/load-sources.mjs

@@ -1,405 +0,0 @@
-import { statSync } from "node:fs";
-import { readFile } from "node:fs/promises";
-import path from "node:path";
-import imageSize from "image-size";
-import {
-  DEFAULT_EXCLUDE_PATTERNS,
-  DEFAULT_INCLUDE_PATTERNS,
-  INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD,
-} from "../../utils/constants/index.mjs";
-import { loadDocumentStructure } from "../../utils/docs-finder-utils.mjs";
-import {
-  buildSourcesContent,
-  calculateTokens,
-  getMimeType,
-  isRemoteFile,
-  loadFilesFromPaths,
-  readFileContents,
-} from "../../utils/file-utils.mjs";
-import { isOpenAPISpecFile } from "../../utils/openapi/index.mjs";
-import {
-  getCurrentGitHead,
-  getModifiedFilesBetweenCommits,
-  toRelativePath,
-} from "../../utils/utils.mjs";
-
-const imageExts = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".heic", ".heif"];
-const videoExts = [
-  ".mp4",
-  ".mpeg",
-  ".mpg",
-  ".mov",
-  ".avi",
-  ".flv",
-  ".mkv",
-  ".webm",
-  ".wmv",
-  ".m4v",
-  ".3gpp",
-];
-
-export default async function loadSources(
-  {
-    sources = [],
-    sourcesPath = [],
-    includePatterns,
-    excludePatterns,
-    outputDir,
-    docsDir,
-    "doc-path": docPath,
-    boardId,
-    useDefaultPatterns = true,
-    lastGitHead,
-    reset = false,
-    media,
-  } = {},
-  options,
-) {
-  let files = Array.isArray(sources) ? [...sources] : [];
-  const { minImageWidth } = media || { minImageWidth: 800 };
-
-  if (sourcesPath) {
-    const sourcesPathList = Array.isArray(sourcesPath) ? sourcesPath : [sourcesPath];
-    const pickSourcesPath = [];
-    const omitSourcesPath = [];
-    sourcesPathList.forEach((x) => {
-      if (typeof x !== "string" || !x) {
-        return;
-      }
-      if (x.startsWith("!")) {
-        omitSourcesPath.push(x.substring(1));
-      } else {
-        pickSourcesPath.push(x);
-      }
-    });
-
-    const customExcludePatterns = omitSourcesPath
-      .map((x) => {
-        try {
-          const stats = statSync(x);
-          if (stats.isFile()) {
-            return x;
-          }
-          if (stats.isDirectory()) {
-            return `${x}/**`;
-          }
-          return null;
-        } catch (error) {
-          console.warn(`Failed to stat path ${x}: ${error.message}`);
-          return null;
-        }
-      })
-      .filter(Boolean);
-    const allFiles = await loadFilesFromPaths(pickSourcesPath, {
-      includePatterns,
-      excludePatterns: [
-        ...new Set([
-          ...(excludePatterns || []),
-          ...customExcludePatterns,
-          ...videoExts.map((x) => `**/*${x}`),
-        ]),
-      ],
-      useDefaultPatterns,
-      defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
-      defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
-    });
-
-    files = files.concat(allFiles);
-  }
-
-  files = [...new Set(files)];
-
-  // Define media file extensions
-  const mediaExtensions = [
-    ...imageExts,
-    // ignore video temporary
-    // ...videoExts
-  ];
-
-  // Separate source files from media files
-  const sourceFilesPaths = [];
-  const mediaFiles = [];
-
-  // Helper function to determine file type from extension
-  const getFileType = (filePath) => {
-    const ext = path.extname(filePath).toLowerCase();
-
-    if (imageExts.includes(ext)) return "image";
-    if (videoExts.includes(ext)) return "video";
-    return "media";
-  };
-
-  let filteredImageCount = 0;
-
-  await Promise.all(
-    files.map(async (file) => {
-      const ext = path.extname(file).toLowerCase();
-
-      if (mediaExtensions.includes(ext) && !isRemoteFile(file)) {
-        // This is a media file
-        const relativePath = path.relative(docsDir, file);
-        const fileName = path.basename(file);
-        const description = path.parse(fileName).name;
-
-        const mediaItem = {
-          name: fileName,
-          path: relativePath,
-          type: getFileType(relativePath),
-          description,
-          mimeType: getMimeType(file),
-        };
-
-        // For image files, get dimensions and filter by width
-        if (mediaItem.type === "image") {
-          try {
-            const buffer = await readFile(file);
-            const dimensions = imageSize(buffer);
-            mediaItem.width = dimensions.width;
-            mediaItem.height = dimensions.height;
-
-            // Filter out images with width less than minImageWidth, but not SVG images
-            if (dimensions.width < minImageWidth && mediaItem.mimeType !== "image/svg+xml") {
-              filteredImageCount++;
-              console.log(
-                `Ignored image: ${fileName} (${dimensions.width}x${dimensions.height}px < ${minImageWidth}px minimum)`,
-              );
-              return;
-            }
-          } catch (err) {
-            console.warn(`⚠️  Failed to get dimensions for ${fileName}: ${err.message}`);
-          }
-        }
-
-        mediaFiles.push(mediaItem);
-      } else {
-        // This is a source file
-        sourceFilesPaths.push(file);
-      }
-    }),
-  );
-
-  // Log summary of filtered images
-  if (filteredImageCount > 0) {
-    console.log(
-      `\nTotal ${filteredImageCount} low-resolution image(s) filtered for better documentation quality (minimum width: ${minImageWidth}px)\n`,
-    );
-  }
-
-  // Read all source files using the utility function
-  let sourceFiles = (await readFileContents(sourceFilesPaths, process.cwd())).map((i) => ({
-    ...i,
-    tokens: calculateTokens(`\n${i.sourceId}\n${i.content}`),
-  }));
-
-  // filter OpenAPI doc should after check isLargeContext
-  sourceFiles = sourceFiles.filter((file) => {
-    if (options?.context?.userContext.openAPISpec) return true;
-
-    const isOpenAPI = isOpenAPISpecFile(file.content);
-    if (isOpenAPI && options?.context?.userContext) {
-      options.context.userContext.openAPISpec = file;
-    }
-    return !isOpenAPI;
-  });
-
-  const totalTokens = sourceFiles.reduce((sum, file) => sum + file.tokens, 0);
-  const totalLines = sourceFiles.reduce(
-    (sum, file) => sum + file.content.split("\n").filter(Boolean).length,
-    0,
-  );
-
-  const dataSources = splitSourcesToChunks(sourceFiles, INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD).map(
-    (i) => ({ dataSourceChunk: buildSourcesContent(i) }),
-  );
-
-  const remoteFileList = [];
-
-  sourceFiles.forEach((file) => {
-    if (isRemoteFile(file.sourceId)) {
-      remoteFileList.push(file);
-    }
-  });
-  if (options?.context?.userContext) {
-    options.context.userContext.remoteFileList = remoteFileList;
-  }
-
-  // all files path
-  const allFilesPaths = sourceFiles.map((x) => `- ${toRelativePath(x.sourceId)}`).join("\n");
-
-  // Get the last documentation structure
-  const originalDocumentStructure = await loadDocumentStructure(outputDir);
-
-  // Get the last output result of the specified path
-  let content;
-  if (docPath && !reset && docsDir) {
-    // First try direct path matching (original format)
-    const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
-    const fileFullName = `${flatName}.md`;
-    let filePath = path.join(docsDir, fileFullName);
-
-    try {
-      content = await readFile(filePath, "utf8");
-    } catch (err) {
-      if (err.code !== "ENOENT") {
-        console.warn(`Error reading document file ${filePath}: ${err.message}`);
-      }
-
-      // If not found and boardId is provided, try boardId-flattenedPath format
-      if (boardId && docPath.startsWith(`${boardId}-`)) {
-        // Extract the flattened path part after boardId-
-        const flattenedPath = docPath.substring(boardId.length + 1);
-        const boardIdFileFullName = `${flattenedPath}.md`;
-        filePath = path.join(docsDir, boardIdFileFullName);
-
-        try {
-          content = await readFile(filePath, "utf8");
-        } catch (boardIdErr) {
-          if (boardIdErr.code !== "ENOENT") {
-            console.warn(`Error reading document file ${filePath}: ${boardIdErr.message}`);
-          }
-          // The file does not exist, content remains undefined
-        }
-      }
-    }
-  }
-
-  // Get git change detection data
-  let modifiedFiles = [];
-  let currentGitHead = null;
-
-  if (lastGitHead) {
-    try {
-      currentGitHead = getCurrentGitHead();
-      if (currentGitHead && currentGitHead !== lastGitHead) {
-        modifiedFiles = getModifiedFilesBetweenCommits(lastGitHead, currentGitHead);
-        console.log(`Detected ${modifiedFiles.length} modified files since last generation`);
-      }
-    } catch (error) {
-      console.warn("Failed to detect git changes:", error.message);
-    }
-  }
-
-  return {
-    dataSources,
-    content,
-    originalDocumentStructure,
-    files,
-    modifiedFiles,
-    totalTokens,
-    totalLines,
-    mediaFiles,
-    allFilesPaths,
-  };
-}
-
-loadSources.input_schema = {
-  type: "object",
-  properties: {
-    sources: {
-      type: "array",
-      items: { type: "string" },
-      description: "Array of paths to the sources files",
-    },
-    sourcesPath: {
-      anyOf: [{ type: "string" }, { type: "array", items: { type: "string" } }],
-      description: "Directory or directories to recursively read files from",
-    },
-    includePatterns: {
-      anyOf: [{ type: "string" }, { type: "array", items: { type: "string" } }],
-      description: "Glob patterns to filter files by path or filename. If not set, include all.",
-    },
-    excludePatterns: {
-      anyOf: [{ type: "string" }, { type: "array", items: { type: "string" } }],
-      description: "Glob patterns to exclude files by path or filename. If not set, exclude none.",
-    },
-    useDefaultPatterns: {
-      type: "boolean",
-      description: "Whether to use default include/exclude patterns. Defaults to true.",
-    },
-    "doc-path": {
-      type: "string",
-      description: "The document path to load content for",
-    },
-    boardId: {
-      type: "string",
-      description: "The board ID for boardId-flattenedPath format matching",
-    },
-    lastGitHead: {
-      type: "string",
-      description: "The git HEAD from last generation for change detection",
-    },
-  },
-  required: [],
-};
-
-loadSources.output_schema = {
-  type: "object",
-  properties: {
-    dataSources: {
-      type: "array",
-      items: {
-        type: "object",
-        properties: {
-          dataSourceChunk: { type: "string" },
-        },
-      },
-    },
-    files: {
-      type: "array",
-      items: { type: "string" },
-      description: "Array of file paths that were loaded",
-    },
-    modifiedFiles: {
-      type: "array",
-      items: { type: "string" },
-      description: "Array of modified files since last generation",
-    },
-    mediaFiles: {
-      type: "array",
-      items: {
-        type: "object",
-        properties: {
-          name: { type: "string" },
-          path: { type: "string" },
-          type: { type: "string" },
-          width: { type: "number" },
-          height: { type: "number" },
-          mimeType: { type: "string" },
-        },
-      },
-      description: "Array of media file objects (images/videos)",
-    },
-  },
-};
-
-loadSources.task_render_mode = "hide";
-
-function splitSourcesToChunks(sources, maxTokens) {
-  const chunks = [];
-
-  let currentChunk = [];
-  let currentTokens = 0;
-
-  for (const source of sources) {
-    const sourceTokens = source.tokens;
-
-    if (currentTokens + sourceTokens > maxTokens) {
-      // Start a new chunk
-      if (currentChunk.length > 0) {
-        chunks.push(currentChunk);
-      }
-      currentChunk = [source];
-      currentTokens = sourceTokens;
-    } else {
-      // Add to current chunk
-      currentChunk.push(source);
-      currentTokens += sourceTokens;
-    }
-  }
-
-  if (currentChunk.length > 0) {
-    chunks.push(currentChunk);
-  }
-
-  return chunks;
-}

package/agents/utils/map-reasoning-effort-level.mjs

@@ -1,15 +0,0 @@
-import {
-  DEFAULT_REASONING_EFFORT_LEVEL,
-  DEFAULT_REASONING_EFFORT_VALUE,
-  REASONING_EFFORT_LEVELS,
-} from "../../utils/constants/index.mjs";
-
-export default function mapReasoningEffortLevel({ level }, options) {
-  const g =
-    REASONING_EFFORT_LEVELS[level] || REASONING_EFFORT_LEVELS[DEFAULT_REASONING_EFFORT_LEVEL];
-
-  return {
-    reasoningEffort:
-      g[options.context.userContext.thinkingEffort] ?? DEFAULT_REASONING_EFFORT_VALUE,
-  };
-}

package/agents/utils/post-generate.mjs

@@ -1,144 +0,0 @@
-import { readdir, unlink } from "node:fs/promises";
-import { join } from "node:path";
-import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
-import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
-
-/**
- * @param {Object} params
- * @param {Array<{path: string, content: string, title: string}>} params.documentStructure
- * @param {string} params.docsDir
- * @param {Array<string>} [params.translateLanguages] - Translation languages
- * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
- */
-export default async function postGenerate({
-  documentStructure,
-  docsDir,
-  translateLanguages = [],
-  locale,
-  projectInfoMessage,
-  isChat,
-  feedback,
-}) {
-  const _results = [];
-  // Save current git HEAD to config.yaml for change detection
-  try {
-    const gitHead = getCurrentGitHead();
-    await saveGitHeadToConfig(gitHead);
-  } catch (err) {
-    console.warn("Failed to save git HEAD:", err.message);
-  }
-
-  // Clean up invalid .md files that are no longer in the documentation structure
-  try {
-    await cleanupInvalidFiles(documentStructure, docsDir, translateLanguages, locale);
-  } catch (err) {
-    console.error("Failed to cleanup invalid .md files:", err.message);
-  }
-
-  let message = `
-✅ Documentation generated successfully! (\`${documentStructure.length}\` documents → \`${docsDir}\`)
-${projectInfoMessage || ""} `;
-
-  if (!isChat) {
-    message += `
-🚀 Next: Make your documentation live and generate a shareable link, run: \`aigne doc publish\`
-💡 Optional: Update specific document (\`aigne doc update\`) or refine documentation structure (\`aigne doc create\`)
-    `;
-  }
-
-  if (isChat && feedback) {
-    message = `User feedback: ${feedback}
-The documentation structure and relevant documents have been updated according to the user's feedback. All feedback has been fully processed.`;
-  }
-
-  // Shutdown mermaid worker pool to ensure clean exit
-  try {
-    await shutdownMermaidWorkerPool();
-  } catch (error) {
-    console.warn("Failed to shutdown mermaid worker pool:", error.message);
-  }
-
-  return {
-    message,
-  };
-}
-
-/**
- * Generate filename based on flatName and language
- * @param {string} flatName - Flattened path name
- * @param {string} language - Language code
- * @returns {string} - Generated filename
- */
-function generateFileName(flatName, language) {
-  const isEnglish = language === "en";
-  return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
-}
-
-/**
- * Clean up .md files that are no longer in the documentation structure
- * @param {Array<{path: string, title: string}>} documentStructure
- * @param {string} docsDir
- * @param {Array<string>} translateLanguages
- * @param {string} locale - Main language locale (e.g., 'en', 'zh', 'fr')
- * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
- */
-async function cleanupInvalidFiles(documentStructure, docsDir, translateLanguages, locale) {
-  const results = [];
-
-  try {
-    // Get all .md files in docsDir
-    const files = await readdir(docsDir);
-    const mdFiles = files.filter((file) => file.endsWith(".md"));
-
-    // Generate expected file names from documentation structure
-    const expectedFiles = new Set();
-
-    // Add main document files
-    for (const { path } of documentStructure) {
-      const flatName = path.replace(/^\//, "").replace(/\//g, "-");
-
-      // Main language file
-      const mainFileName = generateFileName(flatName, locale);
-      expectedFiles.add(mainFileName);
-
-      // Add translation files for each language
-      for (const lang of translateLanguages) {
-        const translateFileName = generateFileName(flatName, lang);
-        expectedFiles.add(translateFileName);
-      }
-    }
-
-    // Find files to delete (files that are not in expectedFiles and not _sidebar.md)
-    const filesToDelete = mdFiles.filter(
-      (file) => !expectedFiles.has(file) && file !== "_sidebar.md",
-    );
-
-    // Delete invalid files
-    for (const file of filesToDelete) {
-      try {
-        const filePath = join(docsDir, file);
-        await unlink(filePath);
-        results.push({
-          path: filePath,
-          success: true,
-          message: `Deleted invalid file: ${file}`,
-        });
-      } catch (err) {
-        console.warn(`Failed to delete invalid file: ${file}, error: ${err.message}`);
-      }
-    }
-
-    if (filesToDelete.length > 0) {
-      console.log(`Cleaned up ${filesToDelete.length} invalid .md files from ${docsDir}`);
-    }
-  } catch (err) {
-    // If docsDir doesn't exist or can't be read, that's okay
-    if (err.code !== "ENOENT") {
-      throw err;
-    }
-  }
-
-  return results;
-}
-
-// Generate sidebar content, support nested structure, and the order is consistent with documentStructure