@aigne/doc-smith 0.9.10 → 0.9.11-beta

package/agents/utils/load-sources.mjs

@@ -1,405 +0,0 @@
-import { statSync } from "node:fs";
-import { readFile } from "node:fs/promises";
-import path from "node:path";
-import imageSize from "image-size";
-import {
-  DEFAULT_EXCLUDE_PATTERNS,
-  DEFAULT_INCLUDE_PATTERNS,
-  INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD,
-} from "../../utils/constants/index.mjs";
-import { loadDocumentStructure } from "../../utils/docs-finder-utils.mjs";
-import {
-  buildSourcesContent,
-  calculateTokens,
-  getMimeType,
-  isRemoteFile,
-  loadFilesFromPaths,
-  readFileContents,
-} from "../../utils/file-utils.mjs";
-import { isOpenAPISpecFile } from "../../utils/openapi/index.mjs";
-import {
-  getCurrentGitHead,
-  getModifiedFilesBetweenCommits,
-  toRelativePath,
-} from "../../utils/utils.mjs";
-
-const imageExts = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".heic", ".heif"];
-const videoExts = [
-  ".mp4",
-  ".mpeg",
-  ".mpg",
-  ".mov",
-  ".avi",
-  ".flv",
-  ".mkv",
-  ".webm",
-  ".wmv",
-  ".m4v",
-  ".3gpp",
-];
-
-export default async function loadSources(
-  {
-    sources = [],
-    sourcesPath = [],
-    includePatterns,
-    excludePatterns,
-    outputDir,
-    docsDir,
-    "doc-path": docPath,
-    boardId,
-    useDefaultPatterns = true,
-    lastGitHead,
-    reset = false,
-    media,
-  } = {},
-  options,
-) {
-  let files = Array.isArray(sources) ? [...sources] : [];
-  const { minImageWidth } = media || { minImageWidth: 800 };
-
-  if (sourcesPath) {
-    const sourcesPathList = Array.isArray(sourcesPath) ? sourcesPath : [sourcesPath];
-    const pickSourcesPath = [];
-    const omitSourcesPath = [];
-    sourcesPathList.forEach((x) => {
-      if (typeof x !== "string" || !x) {
-        return;
-      }
-      if (x.startsWith("!")) {
-        omitSourcesPath.push(x.substring(1));
-      } else {
-        pickSourcesPath.push(x);
-      }
-    });
-
-    const customExcludePatterns = omitSourcesPath
-      .map((x) => {
-        try {
-          const stats = statSync(x);
-          if (stats.isFile()) {
-            return x;
-          }
-          if (stats.isDirectory()) {
-            return `${x}/**`;
-          }
-          return null;
-        } catch (error) {
-          console.warn(`Failed to stat path ${x}: ${error.message}`);
-          return null;
-        }
-      })
-      .filter(Boolean);
-    const allFiles = await loadFilesFromPaths(pickSourcesPath, {
-      includePatterns,
-      excludePatterns: [
-        ...new Set([
-          ...(excludePatterns || []),
-          ...customExcludePatterns,
-          ...videoExts.map((x) => `**/*${x}`),
-        ]),
-      ],
-      useDefaultPatterns,
-      defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
-      defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
-    });
-
-    files = files.concat(allFiles);
-  }
-
-  files = [...new Set(files)];
-
-  // Define media file extensions
-  const mediaExtensions = [
-    ...imageExts,
-    // ignore video temporary
-    // ...videoExts
-  ];
-
-  // Separate source files from media files
-  const sourceFilesPaths = [];
-  const mediaFiles = [];
-
-  // Helper function to determine file type from extension
-  const getFileType = (filePath) => {
-    const ext = path.extname(filePath).toLowerCase();
-
-    if (imageExts.includes(ext)) return "image";
-    if (videoExts.includes(ext)) return "video";
-    return "media";
-  };
-
-  let filteredImageCount = 0;
-
-  await Promise.all(
-    files.map(async (file) => {
-      const ext = path.extname(file).toLowerCase();
-
-      if (mediaExtensions.includes(ext) && !isRemoteFile(file)) {
-        // This is a media file
-        const relativePath = path.relative(docsDir, file);
-        const fileName = path.basename(file);
-        const description = path.parse(fileName).name;
-
-        const mediaItem = {
-          name: fileName,
-          path: relativePath,
-          type: getFileType(relativePath),
-          description,
-          mimeType: getMimeType(file),
-        };
-
-        // For image files, get dimensions and filter by width
-        if (mediaItem.type === "image") {
-          try {
-            const buffer = await readFile(file);
-            const dimensions = imageSize(buffer);
-            mediaItem.width = dimensions.width;
-            mediaItem.height = dimensions.height;
-
-            // Filter out images with width less than minImageWidth, but not SVG images
-            if (dimensions.width < minImageWidth && mediaItem.mimeType !== "image/svg+xml") {
-              filteredImageCount++;
-              console.log(
-                `Ignored image: ${fileName} (${dimensions.width}x${dimensions.height}px < ${minImageWidth}px minimum)`,
-              );
-              return;
-            }
-          } catch (err) {
-            console.warn(`⚠️  Failed to get dimensions for ${fileName}: ${err.message}`);
-          }
-        }
-
-        mediaFiles.push(mediaItem);
-      } else {
-        // This is a source file
-        sourceFilesPaths.push(file);
-      }
-    }),
-  );
-
-  // Log summary of filtered images
-  if (filteredImageCount > 0) {
-    console.log(
-      `\nTotal ${filteredImageCount} low-resolution image(s) filtered for better documentation quality (minimum width: ${minImageWidth}px)\n`,
-    );
-  }
-
-  // Read all source files using the utility function
-  let sourceFiles = (await readFileContents(sourceFilesPaths, process.cwd())).map((i) => ({
-    ...i,
-    tokens: calculateTokens(`\n${i.sourceId}\n${i.content}`),
-  }));
-
-  // filter OpenAPI doc should after check isLargeContext
-  sourceFiles = sourceFiles.filter((file) => {
-    if (options?.context?.userContext.openAPISpec) return true;
-
-    const isOpenAPI = isOpenAPISpecFile(file.content);
-    if (isOpenAPI && options?.context?.userContext) {
-      options.context.userContext.openAPISpec = file;
-    }
-    return !isOpenAPI;
-  });
-
-  const totalTokens = sourceFiles.reduce((sum, file) => sum + file.tokens, 0);
-  const totalLines = sourceFiles.reduce(
-    (sum, file) => sum + file.content.split("\n").filter(Boolean).length,
-    0,
-  );
-
-  const dataSources = splitSourcesToChunks(sourceFiles, INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD).map(
-    (i) => ({ dataSourceChunk: buildSourcesContent(i) }),
-  );
-
-  const remoteFileList = [];
-
-  sourceFiles.forEach((file) => {
-    if (isRemoteFile(file.sourceId)) {
-      remoteFileList.push(file);
-    }
-  });
-  if (options?.context?.userContext) {
-    options.context.userContext.remoteFileList = remoteFileList;
-  }
-
-  // all files path
-  const allFilesPaths = sourceFiles.map((x) => `- ${toRelativePath(x.sourceId)}`).join("\n");
-
-  // Get the last documentation structure
-  const originalDocumentStructure = await loadDocumentStructure(outputDir);
-
-  // Get the last output result of the specified path
-  let content;
-  if (docPath && !reset && docsDir) {
-    // First try direct path matching (original format)
-    const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
-    const fileFullName = `${flatName}.md`;
-    let filePath = path.join(docsDir, fileFullName);
-
-    try {
-      content = await readFile(filePath, "utf8");
-    } catch (err) {
-      if (err.code !== "ENOENT") {
-        console.warn(`Error reading document file ${filePath}: ${err.message}`);
-      }
-
-      // If not found and boardId is provided, try boardId-flattenedPath format
-      if (boardId && docPath.startsWith(`${boardId}-`)) {
-        // Extract the flattened path part after boardId-
-        const flattenedPath = docPath.substring(boardId.length + 1);
-        const boardIdFileFullName = `${flattenedPath}.md`;
-        filePath = path.join(docsDir, boardIdFileFullName);
-
-        try {
-          content = await readFile(filePath, "utf8");
-        } catch (boardIdErr) {
-          if (boardIdErr.code !== "ENOENT") {
-            console.warn(`Error reading document file ${filePath}: ${boardIdErr.message}`);
-          }
-          // The file does not exist, content remains undefined
-        }
-      }
-    }
-  }
-
-  // Get git change detection data
-  let modifiedFiles = [];
-  let currentGitHead = null;
-
-  if (lastGitHead) {
-    try {
-      currentGitHead = getCurrentGitHead();
-      if (currentGitHead && currentGitHead !== lastGitHead) {
-        modifiedFiles = getModifiedFilesBetweenCommits(lastGitHead, currentGitHead);
-        console.log(`Detected ${modifiedFiles.length} modified files since last generation`);
-      }
-    } catch (error) {
-      console.warn("Failed to detect git changes:", error.message);
-    }
-  }
-
-  return {
-    dataSources,
-    content,
-    originalDocumentStructure,
-    files,
-    modifiedFiles,
-    totalTokens,
-    totalLines,
-    mediaFiles,
-    allFilesPaths,
-  };
-}
-
-loadSources.input_schema = {
-  type: "object",
-  properties: {
-    sources: {
-      type: "array",
-      items: { type: "string" },
-      description: "Array of paths to the sources files",
-    },
-    sourcesPath: {
-      anyOf: [{ type: "string" }, { type: "array", items: { type: "string" } }],
-      description: "Directory or directories to recursively read files from",
-    },
-    includePatterns: {
-      anyOf: [{ type: "string" }, { type: "array", items: { type: "string" } }],
-      description: "Glob patterns to filter files by path or filename. If not set, include all.",
-    },
-    excludePatterns: {
-      anyOf: [{ type: "string" }, { type: "array", items: { type: "string" } }],
-      description: "Glob patterns to exclude files by path or filename. If not set, exclude none.",
-    },
-    useDefaultPatterns: {
-      type: "boolean",
-      description: "Whether to use default include/exclude patterns. Defaults to true.",
-    },
-    "doc-path": {
-      type: "string",
-      description: "The document path to load content for",
-    },
-    boardId: {
-      type: "string",
-      description: "The board ID for boardId-flattenedPath format matching",
-    },
-    lastGitHead: {
-      type: "string",
-      description: "The git HEAD from last generation for change detection",
-    },
-  },
-  required: [],
-};
-
-loadSources.output_schema = {
-  type: "object",
-  properties: {
-    dataSources: {
-      type: "array",
-      items: {
-        type: "object",
-        properties: {
-          dataSourceChunk: { type: "string" },
-        },
-      },
-    },
-    files: {
-      type: "array",
-      items: { type: "string" },
-      description: "Array of file paths that were loaded",
-    },
-    modifiedFiles: {
-      type: "array",
-      items: { type: "string" },
-      description: "Array of modified files since last generation",
-    },
-    mediaFiles: {
-      type: "array",
-      items: {
-        type: "object",
-        properties: {
-          name: { type: "string" },
-          path: { type: "string" },
-          type: { type: "string" },
-          width: { type: "number" },
-          height: { type: "number" },
-          mimeType: { type: "string" },
-        },
-      },
-      description: "Array of media file objects (images/videos)",
-    },
-  },
-};
-
-loadSources.task_render_mode = "hide";
-
-function splitSourcesToChunks(sources, maxTokens) {
-  const chunks = [];
-
-  let currentChunk = [];
-  let currentTokens = 0;
-
-  for (const source of sources) {
-    const sourceTokens = source.tokens;
-
-    if (currentTokens + sourceTokens > maxTokens) {
-      // Start a new chunk
-      if (currentChunk.length > 0) {
-        chunks.push(currentChunk);
-      }
-      currentChunk = [source];
-      currentTokens = sourceTokens;
-    } else {
-      // Add to current chunk
-      currentChunk.push(source);
-      currentTokens += sourceTokens;
-    }
-  }
-
-  if (currentChunk.length > 0) {
-    chunks.push(currentChunk);
-  }
-
-  return chunks;
-}

package/agents/utils/map-reasoning-effort-level.mjs

@@ -1,15 +0,0 @@
-import {
-  DEFAULT_REASONING_EFFORT_LEVEL,
-  DEFAULT_REASONING_EFFORT_VALUE,
-  REASONING_EFFORT_LEVELS,
-} from "../../utils/constants/index.mjs";
-
-export default function mapReasoningEffortLevel({ level }, options) {
-  const g =
-    REASONING_EFFORT_LEVELS[level] || REASONING_EFFORT_LEVELS[DEFAULT_REASONING_EFFORT_LEVEL];
-
-  return {
-    reasoningEffort:
-      g[options.context.userContext.thinkingEffort] ?? DEFAULT_REASONING_EFFORT_VALUE,
-  };
-}

package/agents/utils/post-generate.mjs

@@ -1,133 +0,0 @@
-import { readdir, unlink } from "node:fs/promises";
-import { join } from "node:path";
-import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
-import { getCurrentGitHead, saveGitHeadToConfig } from "../../utils/utils.mjs";
-
-/**
- * @param {Object} params
- * @param {Array<{path: string, content: string, title: string}>} params.documentStructure
- * @param {string} params.docsDir
- * @param {Array<string>} [params.translateLanguages] - Translation languages
- * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
- */
-export default async function postGenerate({
-  documentStructure,
-  docsDir,
-  translateLanguages = [],
-  locale,
-  projectInfoMessage,
-}) {
-  const _results = [];
-  // Save current git HEAD to config.yaml for change detection
-  try {
-    const gitHead = getCurrentGitHead();
-    await saveGitHeadToConfig(gitHead);
-  } catch (err) {
-    console.warn("Failed to save git HEAD:", err.message);
-  }
-
-  // Clean up invalid .md files that are no longer in the documentation structure
-  try {
-    await cleanupInvalidFiles(documentStructure, docsDir, translateLanguages, locale);
-  } catch (err) {
-    console.error("Failed to cleanup invalid .md files:", err.message);
-  }
-
-  const message = `
-✅ Documentation generated successfully! (\`${documentStructure.length}\` documents → \`${docsDir}\`)
-${projectInfoMessage || ""}
-🚀 Next: Make your documentation live and generate a shareable link, run: \`aigne doc publish\`
-💡 Optional: Update specific document (\`aigne doc update\`) or refine documentation structure (\`aigne doc create\`)
-  `;
-
-  // Shutdown mermaid worker pool to ensure clean exit
-  try {
-    await shutdownMermaidWorkerPool();
-  } catch (error) {
-    console.warn("Failed to shutdown mermaid worker pool:", error.message);
-  }
-
-  return {
-    message,
-  };
-}
-
-/**
- * Generate filename based on flatName and language
- * @param {string} flatName - Flattened path name
- * @param {string} language - Language code
- * @returns {string} - Generated filename
- */
-function generateFileName(flatName, language) {
-  const isEnglish = language === "en";
-  return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
-}
-
-/**
- * Clean up .md files that are no longer in the documentation structure
- * @param {Array<{path: string, title: string}>} documentStructure
- * @param {string} docsDir
- * @param {Array<string>} translateLanguages
- * @param {string} locale - Main language locale (e.g., 'en', 'zh', 'fr')
- * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
- */
-async function cleanupInvalidFiles(documentStructure, docsDir, translateLanguages, locale) {
-  const results = [];
-
-  try {
-    // Get all .md files in docsDir
-    const files = await readdir(docsDir);
-    const mdFiles = files.filter((file) => file.endsWith(".md"));
-
-    // Generate expected file names from documentation structure
-    const expectedFiles = new Set();
-
-    // Add main document files
-    for (const { path } of documentStructure) {
-      const flatName = path.replace(/^\//, "").replace(/\//g, "-");
-
-      // Main language file
-      const mainFileName = generateFileName(flatName, locale);
-      expectedFiles.add(mainFileName);
-
-      // Add translation files for each language
-      for (const lang of translateLanguages) {
-        const translateFileName = generateFileName(flatName, lang);
-        expectedFiles.add(translateFileName);
-      }
-    }
-
-    // Find files to delete (files that are not in expectedFiles and not _sidebar.md)
-    const filesToDelete = mdFiles.filter(
-      (file) => !expectedFiles.has(file) && file !== "_sidebar.md",
-    );
-
-    // Delete invalid files
-    for (const file of filesToDelete) {
-      try {
-        const filePath = join(docsDir, file);
-        await unlink(filePath);
-        results.push({
-          path: filePath,
-          success: true,
-          message: `Deleted invalid file: ${file}`,
-        });
-      } catch (err) {
-        console.warn(`Failed to delete invalid file: ${file}, error: ${err.message}`);
-      }
-    }
-
-    if (filesToDelete.length > 0) {
-      console.log(`Cleaned up ${filesToDelete.length} invalid .md files from ${docsDir}`);
-    }
-  } catch (err) {
-    // If docsDir doesn't exist or can't be read, that's okay
-    if (err.code !== "ENOENT") {
-      throw err;
-    }
-  }
-
-  return results;
-}
-
-// Generate sidebar content, support nested structure, and the order is consistent with documentStructure

package/agents/utils/read-current-document-content.mjs

@@ -1,46 +0,0 @@
-import { readFileContent } from "../../utils/docs-finder-utils.mjs";
-import { getFileName } from "../../utils/utils.mjs";
-
-/**
- * Read current document content from file system
- * Used when skipping document generation (e.g., for diagram-only updates)
- * Only reads content if intentType is diagram-related, otherwise returns input unchanged
- */
-export default async function readCurrentDocumentContent(input) {
-  const { path, docsDir, locale = "en", intentType } = input;
-
-  // Only read content if intentType is diagram-related
-  // Otherwise, return input unchanged to allow document generation to proceed
-  if (!intentType || !["addDiagram", "updateDiagram", "deleteDiagram"].includes(intentType)) {
-    return input;
-  }
-
-  if (!path || !docsDir) {
-    return input;
-  }
-
-  try {
-    // Read document content using the same utility as other parts of the system
-    const fileName = getFileName(path, locale);
-    const content = await readFileContent(docsDir, fileName);
-
-    if (!content) {
-      console.warn(`⚠️  Could not read content from ${fileName}`);
-      return input;
-    }
-
-    // Return content as both content, documentContent, and originalContent
-    // This matches the structure expected by downstream agents
-    return {
-      ...input,
-      content,
-      documentContent: content,
-      originalContent: content,
-    };
-  } catch (error) {
-    console.warn(`Failed to read current content for ${path}: ${error.message}`);
-    return input;
-  }
-}
-
-readCurrentDocumentContent.task_render_mode = "hide";

package/agents/utils/save-doc-translation.mjs

@@ -1,30 +0,0 @@
-import { saveDocTranslation as _saveDocTranslation } from "../../utils/utils.mjs";
-
-export default async function saveDocTranslation({
-  path,
-  docsDir,
-  translation,
-  language,
-  labels,
-  isShowMessage = false,
-}) {
-  await _saveDocTranslation({
-    path,
-    docsDir,
-    language,
-    translation,
-    labels,
-  });
-
-  // Note: Diagram image translation is now handled by translate-diagram-images.mjs
-  // in the translate workflow, which compares timestamps and only translates when needed
-
-  if (isShowMessage) {
-    const message = `✅ Translation completed successfully.`;
-    return { message };
-  }
-
-  return {};
-}
-
-saveDocTranslation.task_render_mode = "hide";

package/agents/utils/save-doc.mjs

@@ -1,54 +0,0 @@
-import { shutdownMermaidWorkerPool } from "../../utils/mermaid-worker-pool.mjs";
-import { saveDoc as _saveDoc } from "../../utils/utils.mjs";
-
-export default async function saveDoc({
-  path,
-  content,
-  docsDir,
-  labels,
-  locale,
-  feedback,
-  isShowMessage = false,
-  intentType,
-  originalContent,
-  ...rest
-}) {
-  await _saveDoc({
-    path,
-    content,
-    docsDir,
-    labels,
-    locale,
-  });
-
-  // Note: Diagram image translation is handled separately during translate operation
-  // In update/add operations, we only update the main document with timestamp
-  // Translation documents will be updated during translate operation when timestamps differ
-
-  if (isShowMessage) {
-    // Shutdown mermaid worker pool to ensure clean exit
-    try {
-      await shutdownMermaidWorkerPool();
-    } catch (error) {
-      console.warn("Failed to shutdown mermaid worker pool:", error.message);
-    }
-
-    const message = `✅ Document updated successfully.`;
-    return { message };
-  }
-
-  return {
-    path,
-    content,
-    docsDir,
-    labels,
-    locale,
-    feedback,
-    isShowMessage,
-    intentType,
-    originalContent,
-    ...rest,
-  };
-}
-
-saveDoc.task_render_mode = "hide";