@aigne/doc-smith 0.8.12-beta.7 → 0.8.12-beta.8

package/agents/media/batch-generate-media-description.yaml

@@ -0,0 +1,44 @@
+type: team
+name: batchGenerateMediaDescription
+description: Batch generate media (image/video) descriptions with concurrency
+skills:
+  - url: ./generate-media-description.yaml
+task_render_mode: collapse
+task_title: Generate Media Description
+input_schema:
+  type: object
+  properties:
+    mediaToDescribe:
+      type: array
+      items:
+        type: object
+        properties:
+          name:
+            type: string
+          width:
+            type: number
+          height:
+            type: number
+          hash:
+            type: string
+          path:
+            type: string
+          type:
+            type: string
+          mediaFile:
+            type: array
+            items:
+              type: object
+              properties:
+                type:
+                  type: string
+                path:
+                  type: string
+                filename:
+                  type: string
+                mimeType:
+                  type: string
+      description: Array of media files (images/videos) that need descriptions
+iterate_on: mediaToDescribe
+concurrency: 5
+mode: sequential

package/agents/media/generate-media-description.yaml

@@ -0,0 +1,47 @@
+name: generateMediaDescription
+description: Generate description for a single media file (image/video)
+model: "google/gemini-2.5-flash"
+modalities: ["text", "image"]
+instructions:
+  - role: system
+    url: ../../prompts/media/media-description/system-prompt.md
+  - role: user
+    url: ../../prompts/media/media-description/user-prompt.md
+input_file_key: mediaFile
+include_input_in_output: true
+input_schema:
+  type: object
+  properties:
+    name:
+      type: string
+      description: Media file name
+    width:
+      type: number
+      description: Media width in pixels
+    height:
+      type: number
+      description: Media height in pixels
+    hash:
+      type: string
+    path:
+      type: string
+      description: Media path
+    type:
+      type: string
+      description: Media type (image/video)
+    mediaFile:
+      type: array
+      items:
+        type: object
+        properties:
+          type:
+            type: string
+          path:
+            type: string
+          filename:
+            type: string
+          mimeType:
+            type: string
+  required:
+    - name
+output_key: description

package/agents/media/load-media-description.mjs

@@ -0,0 +1,238 @@
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { parse, stringify } from "yaml";
+import { getMediaDescriptionCachePath } from "../../utils/file-utils.mjs";
+
+const SIZE_THRESHOLD = 10 * 1024 * 1024; // 10MB
+
+// Supported MIME types for Gemini AI
+const SUPPORTED_IMAGE_TYPES = new Set([
+  "image/png",
+  "image/jpeg",
+  "image/webp",
+  "image/heic",
+  "image/heif",
+]);
+
+const SUPPORTED_VIDEO_TYPES = new Set([
+  "video/mp4",
+  "video/mpeg",
+  "video/mov",
+  "video/avi",
+  "video/x-flv",
+  "video/mpg",
+  "video/webm",
+  "video/wmv",
+  "video/3gpp",
+]);
+
+/**
+ * Calculate hash for a media file
+ * For files < 10MB: use file content
+ * For files >= 10MB: use path + size + mtime to avoid memory issues
+ * @param {string} absolutePath - The absolute path to the media file
+ * @returns {Promise<string>} - The hash of the file
+ */
+async function calculateMediaHash(absolutePath) {
+  const stats = await stat(absolutePath);
+
+  if (stats.size < SIZE_THRESHOLD) {
+    // Small file: use full content
+    const content = await readFile(absolutePath);
+    return createHash("sha256").update(content).digest("hex");
+  }
+
+  // Large file: use path + size + mtime
+  const hashInput = `${absolutePath}:${stats.size}:${stats.mtimeMs}`;
+  return createHash("sha256").update(hashInput).digest("hex");
+}
+
+/**
+ * Load media descriptions from cache and generate new ones if needed
+ * @param {Object} input - Input parameters
+ * @param {Array} input.mediaFiles - Array of media file objects from load-sources
+ * @param {string} input.docsDir - Base directory for documentation
+ * @param {Object} options - Agent options
+ * @returns {Promise<Object>} - Updated assetsContent with media descriptions
+ */
+export default async function loadMediaDescription(input, options) {
+  const { mediaFiles = [], docsDir } = input;
+
+  // Filter to get image and video files with supported MIME types
+  const mediaFilesToProcess = mediaFiles.filter((file) => {
+    if (file.type === "image") {
+      return SUPPORTED_IMAGE_TYPES.has(file.mimeType);
+    }
+    if (file.type === "video") {
+      return SUPPORTED_VIDEO_TYPES.has(file.mimeType);
+    }
+    return false;
+  });
+
+  // Path to media description cache file
+  const cacheFilePath = getMediaDescriptionCachePath();
+
+  // Load existing cache
+  let cache = {};
+  if (existsSync(cacheFilePath)) {
+    try {
+      const cacheContent = await readFile(cacheFilePath, "utf8");
+      const parsedCache = parse(cacheContent);
+      cache = parsedCache?.descriptions || {};
+    } catch (error) {
+      console.warn("Failed to read media description cache:", error.message);
+    }
+  }
+
+  // Find media files without descriptions
+  const mediaToDescribe = [];
+  const mediaHashMap = new Map();
+
+  const absoluteDocsDir = path.resolve(process.cwd(), docsDir);
+
+  // Only process media files that need AI description
+  for (const mediaFile of mediaFilesToProcess) {
+    // Convert relative path to absolute path for consistent hashing
+    // mediaFiles.path is relative to docsDir
+    const absolutePath = path.join(absoluteDocsDir, mediaFile.path);
+    const mediaHash = await calculateMediaHash(absolutePath);
+    mediaHashMap.set(mediaFile.path, mediaHash);
+
+    if (!cache[mediaHash]) {
+      mediaToDescribe.push({
+        ...mediaFile,
+        hash: mediaHash,
+        path: mediaFile.path,
+        mediaFile: [
+          {
+            type: "local",
+            path: absolutePath,
+            filename: mediaFile.name,
+            mimeType: mediaFile.mimeType,
+          },
+        ],
+      });
+    }
+  }
+
+  // Generate descriptions for media files without cache - use team agent for concurrent processing
+  const newDescriptions = {};
+  if (mediaToDescribe.length > 0) {
+    try {
+      // Use batch team agent for concurrent processing
+      const results = await options.context.invoke(
+        options.context.agents["batchGenerateMediaDescription"],
+        {
+          mediaToDescribe,
+        },
+      );
+
+      // Process results - results is an array of individual results
+      if (Array.isArray(results?.mediaToDescribe)) {
+        for (const result of results.mediaToDescribe) {
+          if (result?.hash && result?.description) {
+            newDescriptions[result.hash] = {
+              path: result.path,
+              description: result.description,
+              generatedAt: new Date().toISOString(),
+            };
+          }
+        }
+      }
+
+      // Merge new descriptions into cache
+      Object.assign(cache, newDescriptions);
+
+      // Save updated cache
+      await mkdir(path.dirname(cacheFilePath), { recursive: true });
+      const cacheYaml = stringify({
+        descriptions: cache,
+        lastUpdated: new Date().toISOString(),
+      });
+      await writeFile(cacheFilePath, cacheYaml, "utf8");
+
+      console.log(`Generated descriptions for ${Object.keys(newDescriptions).length} media files`);
+    } catch (error) {
+      console.error("Failed to generate media descriptions:", error.message);
+    }
+  }
+
+  // Build enhanced assetsContent with descriptions
+  let enhancedAssetsContent = "# Available Media Assets for Documentation\n\n";
+
+  if (mediaFiles.length > 0) {
+    enhancedAssetsContent += "```yaml\n";
+    enhancedAssetsContent += "assets:\n";
+
+    for (const asset of mediaFiles) {
+      enhancedAssetsContent += `  - name: "${asset.name}"\n`;
+      enhancedAssetsContent += `    path: "${asset.path}"\n`;
+      enhancedAssetsContent += `    type: "${asset.type}"\n`;
+
+      // Add description for images and videos
+      if (asset.type === "image" || asset.type === "video") {
+        const mediaHash = mediaHashMap.get(asset.path);
+        const cachedDesc = cache[mediaHash];
+        if (cachedDesc?.description) {
+          enhancedAssetsContent += `    description: "${cachedDesc.description}"\n`;
+        }
+      }
+
+      // Add dimensions for images and videos
+      if (asset.width && asset.height) {
+        enhancedAssetsContent += `    width: ${asset.width}\n`;
+        enhancedAssetsContent += `    height: ${asset.height}\n`;
+      }
+    }
+
+    enhancedAssetsContent += "```\n";
+  }
+
+  return {
+    ...input,
+    assetsContent: enhancedAssetsContent,
+    mediaFiles,
+  };
+}
+
+loadMediaDescription.input_schema = {
+  type: "object",
+  properties: {
+    mediaFiles: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          name: { type: "string" },
+          path: { type: "string" },
+          type: { type: "string" },
+          width: { type: "number" },
+          height: { type: "number" },
+          mimeType: { type: "string" },
+        },
+      },
+      description: "Array of media file objects (images/videos)",
+    },
+    docsDir: {
+      type: "string",
+      description: "Base directory for documentation",
+    },
+  },
+  required: ["mediaFiles", "docsDir"],
+};
+
+loadMediaDescription.output_schema = {
+  type: "object",
+  properties: {
+    assetsContent: {
+      type: "string",
+      description: "Enhanced assets content with media descriptions",
+    },
+    mediaFiles: {
+      type: "array",
+      description: "Array of media file objects",
+    },
+  },
+};

package/agents/update/index.yaml

@@ -30,6 +30,7 @@ skills:
     default_input:
       requiredFeedback: false
   - ../utils/format-document-structure.mjs
+  - ../media/load-media-description.mjs
   - ../update/check-update-is-single.mjs
   - ../update/save-and-translate-document.mjs
   - url: ../utils/action-success.mjs

package/agents/utils/load-sources.mjs

@@ -1,10 +1,12 @@
 import { readFile } from "node:fs/promises";
 import path from "node:path";
+import imageSize from "image-size";
 import {
   buildSourcesContent,
   calculateFileStats,
   loadFilesFromPaths,
   readFileContents,
+  getMimeType,
 } from "../../utils/file-utils.mjs";
 import {
   getCurrentGitHead,
@@ -29,8 +31,10 @@ export default async function loadSources({
   useDefaultPatterns = true,
   lastGitHead,
   reset = false,
+  media,
 } = {}) {
   let files = Array.isArray(sources) ? [...sources] : [];
+  const { minImageWidth } = media || { minImageWidth: 800 };
 
   if (sourcesPath) {
     const allFiles = await loadFilesFromPaths(sourcesPath, {
@@ -58,36 +62,102 @@ export default async function loadSources({
     ".bmp",
     ".webp",
     ".svg",
+    ".heic",
+    ".heif",
     ".mp4",
+    ".mpeg",
+    ".mpg",
     ".mov",
     ".avi",
+    ".flv",
     ".mkv",
     ".webm",
+    ".wmv",
     ".m4v",
+    ".3gpp",
   ];
 
   // Separate source files from media files
   const sourceFilesPaths = [];
   const mediaFiles = [];
 
-  for (const file of files) {
-    const ext = path.extname(file).toLowerCase();
-
-    if (mediaExtensions.includes(ext)) {
-      // This is a media file
-      const relativePath = path.relative(docsDir, file);
-      const fileName = path.basename(file);
-      const description = path.parse(fileName).name;
-
-      mediaFiles.push({
-        name: fileName,
-        path: relativePath,
-        description,
-      });
-    } else {
-      // This is a source file
-      sourceFilesPaths.push(file);
-    }
+  // Helper function to determine file type from extension
+  const getFileType = (filePath) => {
+    const ext = path.extname(filePath).toLowerCase();
+    const imageExts = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".heic", ".heif"];
+    const videoExts = [
+      ".mp4",
+      ".mpeg",
+      ".mpg",
+      ".mov",
+      ".avi",
+      ".flv",
+      ".mkv",
+      ".webm",
+      ".wmv",
+      ".m4v",
+      ".3gpp",
+    ];
+
+    if (imageExts.includes(ext)) return "image";
+    if (videoExts.includes(ext)) return "video";
+    return "media";
+  };
+
+  let filteredImageCount = 0;
+
+  await Promise.all(
+    files.map(async (file) => {
+      const ext = path.extname(file).toLowerCase();
+
+      if (mediaExtensions.includes(ext)) {
+        // This is a media file
+        const relativePath = path.relative(docsDir, file);
+        const fileName = path.basename(file);
+        const description = path.parse(fileName).name;
+
+        const mediaItem = {
+          name: fileName,
+          path: relativePath,
+          type: getFileType(relativePath),
+          description,
+          mimeType: getMimeType(file),
+        };
+
+        // For image files, get dimensions and filter by width
+        if (mediaItem.type === "image") {
+          try {
+            const buffer = await readFile(file);
+            const dimensions = imageSize(buffer);
+            mediaItem.width = dimensions.width;
+            mediaItem.height = dimensions.height;
+
+            // Filter out images with width less than minImageWidth
+            if (dimensions.width < minImageWidth) {
+              filteredImageCount++;
+              console.log(
+                `Filtered image: ${fileName} (${dimensions.width}x${dimensions.height}px < ${minImageWidth}px minimum)`,
+              );
+              return;
+            }
+          } catch (err) {
+            console.warn(`⚠️  Failed to get dimensions for ${fileName}: ${err.message}`);
+          }
+        }
+
+        mediaFiles.push(mediaItem);
+      } else {
+        // This is a source file
+        sourceFilesPaths.push(file);
+      }
+    }),
+  );
+
+  // Log summary of filtered images
+  if (filteredImageCount > 0) {
+    console.log(
+      `\nTotal ${filteredImageCount} low-resolution image(s) filtered for better documentation quality (minimum width: ${minImageWidth}px)\n`,
+    );
   }
 
   // Read all source files using the utility function
@@ -179,37 +249,6 @@ export default async function loadSources({
     }
   }
 
-  // Generate assets content from media files
-  let assetsContent = "# Available Media Assets for Documentation\n\n";
-
-  if (mediaFiles.length > 0) {
-    // Helper function to determine file type from extension
-    const getFileType = (filePath) => {
-      const ext = path.extname(filePath).toLowerCase();
-      const imageExts = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg"];
-      const videoExts = [".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v"];
-
-      if (imageExts.includes(ext)) return "image";
-      if (videoExts.includes(ext)) return "video";
-      return "media";
-    };
-
-    const mediaYaml = mediaFiles.map((file) => ({
-      name: file.name,
-      path: file.path,
-      type: getFileType(file.path),
-    }));
-
-    assetsContent += "```yaml\n";
-    assetsContent += "assets:\n";
-    mediaYaml.forEach((asset) => {
-      assetsContent += `  - name: "${asset.name}"\n`;
-      assetsContent += `    path: "${asset.path}"\n`;
-      assetsContent += `    type: "${asset.type}"\n`;
-    });
-    assetsContent += "```\n";
-  }
-
   return {
     datasources: allSources,
     content,
@@ -218,7 +257,7 @@ export default async function loadSources({
     modifiedFiles,
     totalTokens,
     totalLines,
-    assetsContent,
+    mediaFiles,
     isLargeContext,
     allFilesPaths,
   };
@@ -280,9 +319,20 @@ loadSources.output_schema = {
       items: { type: "string" },
       description: "Array of modified files since last generation",
     },
-    assetsContent: {
-      type: "string",
-      description: "Markdown content for available media assets",
+    mediaFiles: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          name: { type: "string" },
+          path: { type: "string" },
+          type: { type: "string" },
+          width: { type: "number" },
+          height: { type: "number" },
+          mimeType: { type: "string" },
+        },
+      },
+      description: "Array of media file objects (images/videos)",
     },
   },
 };

package/aigne.yaml

@@ -49,6 +49,11 @@ agents:
   - ./agents/publish/publish-docs.mjs
   - ./agents/publish/index.yaml
 
+  # Media
+  - ./agents/media/load-media-description.mjs
+  - ./agents/media/batch-generate-media-description.yaml
+  - ./agents/media/generate-media-description.yaml
+
   # Clear/Cleanup
   - ./agents/clear/choose-contents.mjs
   - ./agents/clear/clear-document-structure.mjs
@@ -56,6 +61,7 @@ agents:
   - ./agents/clear/clear-document-config.mjs
   - ./agents/clear/clear-auth-tokens.mjs
   - ./agents/clear/clear-deployment-config.mjs
+  - ./agents/clear/clear-media-description.mjs
 
   # Utilities
   - ./agents/utils/load-sources.mjs