@mixio-pro/kalaasetu-mcp 1.1.3 → 1.2.0

package/src/tools/gemini.ts

@@ -11,6 +11,7 @@ import * as wav from "wav";
 import { PassThrough } from "stream";
 import { getStorage } from "../storage";
 import { generateTimestampedFilename } from "../utils/filename";
+import { safeToolExecute } from "../utils/tool-wrapper";
 
 const ai = new GoogleGenAI({
   apiKey: process.env.GEMINI_API_KEY || "",
@@ -128,11 +129,17 @@ async function uploadFileToGemini(filePath: string): Promise<any> {
       fs.unlinkSync(localPath);
     }
 
-    // Wait for file processing to complete
+    // Wait for file processing to complete (max 60 seconds)
     let getFile = await ai.files.get({ name: uploadedFile.name! });
-    while (getFile.state === "PROCESSING") {
+    let attempts = 0;
+    while (getFile.state === "PROCESSING" && attempts < 20) {
       await new Promise((resolve) => setTimeout(resolve, 3000));
       getFile = await ai.files.get({ name: uploadedFile.name! });
+      attempts++;
+    }
+
+    if (getFile.state === "PROCESSING") {
+      throw new Error("File processing timed out after 60 seconds");
     }
 
     if (getFile.state === "FAILED") {
@@ -213,74 +220,77 @@ export const geminiTextToImage = {
       .optional()
       .describe("Optional reference image file paths to guide generation"),
   }),
+  timeoutMs: 300000,
   execute: async (args: {
     prompt: string;
     aspect_ratio?: string;
     output_path?: string;
     reference_images?: string[];
   }) => {
-    try {
-      const contents: any[] = [args.prompt];
+    return safeToolExecute(async () => {
+      try {
+        const contents: any[] = [args.prompt];
 
-      if (args.reference_images && Array.isArray(args.reference_images)) {
-        for (const refPath of args.reference_images) {
-          contents.push(await fileToGenerativePart(refPath));
+        if (args.reference_images && Array.isArray(args.reference_images)) {
+          for (const refPath of args.reference_images) {
+            contents.push(await fileToGenerativePart(refPath));
+          }
         }
-      }
 
-      const response = await ai.models.generateContent({
-        model: "gemini-3-pro-image-preview",
-        contents: contents,
-        config: {
-          responseModalities: ["TEXT", "IMAGE"],
-          imageConfig: {
-            aspectRatio: args.aspect_ratio || "9:16",
+        const response = await ai.models.generateContent({
+          model: "gemini-3-pro-image-preview",
+          contents: contents,
+          config: {
+            responseModalities: ["TEXT", "IMAGE"],
+            imageConfig: {
+              aspectRatio: args.aspect_ratio || "9:16",
+            },
           },
-        },
-      });
-
-      const images = [];
-      let textResponse = "";
-
-      if (response.candidates && response.candidates[0]?.content?.parts) {
-        for (const part of response.candidates[0].content.parts) {
-          if (part.text) {
-            textResponse += part.text;
-          } else if (part.inlineData?.data) {
-            const imageData = part.inlineData.data;
-            // Always save the image - use provided path or generate one
-            const outputPath =
-              args.output_path ||
-              generateTimestampedFilename("generated_image.png");
-            const storage = getStorage();
-            const url = await storage.writeFile(
-              outputPath,
-              Buffer.from(imageData, "base64")
-            );
-            images.push({
-              url,
-              filename: outputPath,
-              mimeType: "image/png",
-            });
+        });
+
+        const images = [];
+        let textResponse = "";
+
+        if (response.candidates && response.candidates[0]?.content?.parts) {
+          for (const part of response.candidates[0].content.parts) {
+            if (part.text) {
+              textResponse += part.text;
+            } else if (part.inlineData?.data) {
+              const imageData = part.inlineData.data;
+              // Always save the image - use provided path or generate one
+              const outputPath =
+                args.output_path ||
+                generateTimestampedFilename("generated_image.png");
+              const storage = getStorage();
+              const url = await storage.writeFile(
+                outputPath,
+                Buffer.from(imageData, "base64")
+              );
+              images.push({
+                url,
+                filename: outputPath,
+                mimeType: "image/png",
+              });
+            }
           }
         }
-      }
 
-      if (images.length > 0) {
-        // Return the URL directly for easy parsing
-        return JSON.stringify({
-          url: images?.[0]?.url,
-          images,
-          message: textResponse || "Image generated successfully",
-        });
-      }
+        if (images.length > 0) {
+          // Return the URL directly for easy parsing
+          return JSON.stringify({
+            url: images?.[0]?.url,
+            images,
+            message: textResponse || "Image generated successfully",
+          });
+        }
 
-      return (
-        textResponse || "Image generation completed but no image was produced"
-      );
-    } catch (error: any) {
-      throw new Error(`Image generation failed: ${error.message}`);
-    }
+        return (
+          textResponse || "Image generation completed but no image was produced"
+        );
+      } catch (error: any) {
+        throw new Error(`Image generation failed: ${error.message}`);
+      }
+    }, "gemini-generateImage");
   },
 };
 
@@ -300,63 +310,68 @@ export const geminiEditImage = {
       .optional()
       .describe("Additional image paths for reference"),
   }),
+  timeoutMs: 300000,
   execute: async (args: {
     image_path: string;
     prompt: string;
     output_path?: string;
     reference_images?: string[];
   }) => {
-    try {
-      const imagePart = await fileToGenerativePart(args.image_path);
-      const contents: any[] = [args.prompt, imagePart];
+    return safeToolExecute(async () => {
+      try {
+        const imagePart = await fileToGenerativePart(args.image_path);
+        const contents: any[] = [args.prompt, imagePart];
 
-      if (args.reference_images) {
-        for (const refPath of args.reference_images) {
-          contents.push(await fileToGenerativePart(refPath));
+        if (args.reference_images) {
+          for (const refPath of args.reference_images) {
+            contents.push(await fileToGenerativePart(refPath));
+          }
         }
-      }
 
-      const response = await ai.models.generateContent({
-        model: "gemini-3-pro-image-preview",
-        contents: contents,
-      });
-
-      const images = [];
-      let textResponse = "";
-
-      if (response.candidates && response.candidates[0]?.content?.parts) {
-        for (const part of response.candidates[0].content.parts) {
-          if (part.text) {
-            textResponse += part.text;
-          } else if (part.inlineData?.data) {
-            const imageData = part.inlineData.data;
-            if (args.output_path) {
-              const storage = getStorage();
-              const url = await storage.writeFile(
-                args.output_path,
-                Buffer.from(imageData, "base64")
-              );
-              images.push({
-                url,
-                filename: args.output_path,
-                mimeType: "image/png",
-              });
+        const response = await ai.models.generateContent({
+          model: "gemini-3-pro-image-preview",
+          contents: contents,
+        });
+
+        const images = [];
+        let textResponse = "";
+
+        if (response.candidates && response.candidates[0]?.content?.parts) {
+          for (const part of response.candidates[0].content.parts) {
+            if (part.text) {
+              textResponse += part.text;
+            } else if (part.inlineData?.data) {
+              const imageData = part.inlineData.data;
+              if (args.output_path) {
+                const storage = getStorage();
+                const url = await storage.writeFile(
+                  args.output_path,
+                  Buffer.from(imageData, "base64")
+                );
+                images.push({
+                  url,
+                  filename: args.output_path,
+                  mimeType: "image/png",
+                });
+              }
             }
           }
         }
-      }
 
-      if (images.length > 0) {
-        return JSON.stringify({
-          images,
-          message: textResponse || "Image edited successfully",
-        });
-      }
+        if (images.length > 0) {
+          return JSON.stringify({
+            images,
+            message: textResponse || "Image edited successfully",
+          });
+        }
 
-      return textResponse || "Image editing completed but no response received";
-    } catch (error: any) {
-      throw new Error(`Image editing failed: ${error.message}`);
-    }
+        return (
+          textResponse || "Image editing completed but no response received"
+        );
+      } catch (error: any) {
+        throw new Error(`Image editing failed: ${error.message}`);
+      }
+    }, "gemini-editImage");
   },
 };
 
@@ -370,59 +385,62 @@ export const geminiAnalyzeImages = {
       .describe("Array of image file paths to analyze"),
     prompt: z.string().describe("Text prompt or question about the images"),
   }),
+  timeoutMs: 300000,
   execute: async (args: { image_paths: string[]; prompt: string }) => {
-    try {
-      // Handle array parsing
-      if (!args.image_paths) {
-        throw new Error("Image paths not provided");
-      }
+    return safeToolExecute(async () => {
+      try {
+        // Handle array parsing
+        if (!args.image_paths) {
+          throw new Error("Image paths not provided");
+        }
 
-      // Convert to array if passed as string
-      let imagePaths: string[];
-      if (typeof args.image_paths === "string") {
-        const strValue = args.image_paths as string;
-        if (strValue.startsWith("[") && strValue.endsWith("]")) {
-          try {
-            imagePaths = JSON.parse(strValue);
-          } catch {
-            throw new Error("Invalid image_paths format");
+        // Convert to array if passed as string
+        let imagePaths: string[];
+        if (typeof args.image_paths === "string") {
+          const strValue = args.image_paths as string;
+          if (strValue.startsWith("[") && strValue.endsWith("]")) {
+            try {
+              imagePaths = JSON.parse(strValue);
+            } catch {
+              throw new Error("Invalid image_paths format");
+            }
+          } else {
+            imagePaths = [strValue];
           }
+        } else if (Array.isArray(args.image_paths)) {
+          imagePaths = args.image_paths;
         } else {
-          imagePaths = [strValue];
+          throw new Error("Invalid image_paths: must be array or string");
         }
-      } else if (Array.isArray(args.image_paths)) {
-        imagePaths = args.image_paths;
-      } else {
-        throw new Error("Invalid image_paths: must be array or string");
-      }
 
-      if (imagePaths.length === 0) {
-        throw new Error("At least one image path must be provided");
-      }
+        if (imagePaths.length === 0) {
+          throw new Error("At least one image path must be provided");
+        }
 
-      const contents: any[] = [args.prompt];
+        const contents: any[] = [args.prompt];
 
-      for (const imagePath of imagePaths) {
-        contents.push(await fileToGenerativePart(imagePath));
-      }
+        for (const imagePath of imagePaths) {
+          contents.push(await fileToGenerativePart(imagePath));
+        }
 
-      const response = await ai.models.generateContent({
-        model: "gemini-2.5-pro",
-        contents: contents,
-      });
+        const response = await ai.models.generateContent({
+          model: "gemini-2.5-pro",
+          contents: contents,
+        });
 
-      let result = "";
-      if (response.candidates && response.candidates[0]?.content?.parts) {
-        for (const part of response.candidates[0].content.parts) {
-          if (part.text) {
-            result += part.text;
+        let result = "";
+        if (response.candidates && response.candidates[0]?.content?.parts) {
+          for (const part of response.candidates[0].content.parts) {
+            if (part.text) {
+              result += part.text;
+            }
           }
         }
+        return result || "Analysis completed but no text response received";
+      } catch (error: any) {
+        throw new Error(`Image analysis failed: ${error.message}`);
       }
-      return result || "Analysis completed but no text response received";
-    } catch (error: any) {
-      throw new Error(`Image analysis failed: ${error.message}`);
-    }
+    }, "gemini-analyzeImages");
   },
 };
 
@@ -444,53 +462,56 @@ export const geminiSingleSpeakerTts = {
         "Output WAV file path (optional, defaults to timestamp-based filename)"
       ),
   }),
+  timeoutMs: 300000,
   execute: async (args: {
     text: string;
     voice_name: string;
     output_path?: string;
   }) => {
-    try {
-      const response = await ai.models.generateContent({
-        model: "gemini-2.5-pro-preview-tts",
-        contents: [{ parts: [{ text: args.text }] }],
-        config: {
-          responseModalities: ["AUDIO"],
-          speechConfig: {
-            voiceConfig: {
-              prebuiltVoiceConfig: {
-                voiceName: args.voice_name || "Despina",
+    return safeToolExecute(async () => {
+      try {
+        const response = await ai.models.generateContent({
+          model: "gemini-2.5-pro-preview-tts",
+          contents: [{ parts: [{ text: args.text }] }],
+          config: {
+            responseModalities: ["AUDIO"],
+            speechConfig: {
+              voiceConfig: {
+                prebuiltVoiceConfig: {
+                  voiceName: args.voice_name || "Despina",
+                },
               },
             },
           },
-        },
-      });
+        });
 
-      const data =
-        response.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
-      if (!data) {
-        throw new Error("No audio data received from Gemini API");
-      }
+        const data =
+          response.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
+        if (!data) {
+          throw new Error("No audio data received from Gemini API");
+        }
 
-      const audioBuffer = Buffer.from(data, "base64");
+        const audioBuffer = Buffer.from(data, "base64");
 
-      // Use provided output path or generate default with timestamp
-      const outputPath =
-        args.output_path || generateTimestampedFilename("voice_output.wav");
+        // Use provided output path or generate default with timestamp
+        const outputPath =
+          args.output_path || generateTimestampedFilename("voice_output.wav");
 
-      const storage = getStorage();
-      const url = await storage.writeFile(outputPath, audioBuffer);
-
-      return JSON.stringify({
-        audio: {
-          url,
-          filename: outputPath,
-          mimeType: "audio/wav",
-        },
-        message: "Audio generated successfully",
-      });
-    } catch (error: any) {
-      throw new Error(`Voice generation failed: ${error.message}`);
-    }
+        const storage = getStorage();
+        const url = await storage.writeFile(outputPath, audioBuffer);
+
+        return JSON.stringify({
+          audio: {
+            url,
+            filename: outputPath,
+            mimeType: "audio/wav",
+          },
+          message: "Audio generated successfully",
+        });
+      } catch (error: any) {
+        throw new Error(`Voice generation failed: ${error.message}`);
+      }
+    }, "gemini-generateSpeech");
   },
 };
 
@@ -530,6 +551,7 @@ export const geminiAnalyzeVideos = {
         "Media resolution: 'default' or 'low' (low resolution uses ~100 tokens/sec vs 300 tokens/sec)"
       ),
   }),
+  timeoutMs: 300000,
   execute: async (args: {
     video_inputs: string[];
     prompt: string;
@@ -538,86 +560,90 @@ export const geminiAnalyzeVideos = {
     end_offset?: string;
     media_resolution?: string;
   }) => {
-    try {
-      // Handle array parsing
-      if (!args.video_inputs) {
-        throw new Error("Video inputs not provided");
-      }
+    return safeToolExecute(async () => {
+      try {
+        // Handle array parsing
+        if (!args.video_inputs) {
+          throw new Error("Video inputs not provided");
+        }
 
-      // Convert to array if passed as string
-      let videoInputs: string[];
-      if (typeof args.video_inputs === "string") {
-        const strValue = args.video_inputs as string;
-        if (strValue.startsWith("[") && strValue.endsWith("]")) {
-          try {
-            videoInputs = JSON.parse(strValue);
-          } catch {
-            throw new Error("Invalid video_inputs format");
+        // Convert to array if passed as string
+        let videoInputs: string[];
+        if (typeof args.video_inputs === "string") {
+          const strValue = args.video_inputs as string;
+          if (strValue.startsWith("[") && strValue.endsWith("]")) {
+            try {
+              videoInputs = JSON.parse(strValue);
+            } catch {
+              throw new Error("Invalid video_inputs format");
+            }
+          } else {
+            videoInputs = [strValue];
           }
+        } else if (Array.isArray(args.video_inputs)) {
+          videoInputs = args.video_inputs;
         } else {
-          videoInputs = [strValue];
+          throw new Error("Invalid video_inputs: must be array or string");
         }
-      } else if (Array.isArray(args.video_inputs)) {
-        videoInputs = args.video_inputs;
-      } else {
-        throw new Error("Invalid video_inputs: must be array or string");
-      }
 
-      if (videoInputs.length === 0) {
-        throw new Error("At least one video input must be provided");
-      }
+        if (videoInputs.length === 0) {
+          throw new Error("At least one video input must be provided");
+        }
 
-      if (videoInputs.length > 10) {
-        throw new Error(
-          "Maximum 10 videos per request allowed for Gemini 2.5+ models"
-        );
-      }
+        if (videoInputs.length > 10) {
+          throw new Error(
+            "Maximum 10 videos per request allowed for Gemini 2.5+ models"
+          );
+        }
 
-      // Prepare video parts for content
-      const videoParts: any[] = [];
+        // Prepare video parts for content
+        const videoParts: any[] = [];
 
-      // Process each video input
-      for (const videoInput of videoInputs) {
-        const videoConfig = {
-          fps: args.fps || (isYouTubeUrl(videoInput) ? 1 : 5), // Default 5 FPS for local, 1 FPS for YouTube
-          startOffset: args.start_offset,
-          endOffset: args.end_offset,
-        };
+        // Process each video input
+        for (const videoInput of videoInputs) {
+          const videoConfig = {
+            fps: args.fps || (isYouTubeUrl(videoInput) ? 1 : 5), // Default 5 FPS for local, 1 FPS for YouTube
+            startOffset: args.start_offset,
+            endOffset: args.end_offset,
+          };
 
-        const videoPart = await processVideoInput(videoInput, videoConfig);
-        videoParts.push(videoPart);
-      }
+          const videoPart = await processVideoInput(videoInput, videoConfig);
+          videoParts.push(videoPart);
+        }
 
-      // Build content using createUserContent and createPartFromUri for uploaded files
-      const contentParts: any[] = [args.prompt];
+        // Build content using createUserContent and createPartFromUri for uploaded files
+        const contentParts: any[] = [args.prompt];
 
-      for (const videoPart of videoParts) {
-        if (videoPart.uri && videoPart.mimeType) {
-          contentParts.push(
-            createPartFromUri(videoPart.uri, videoPart.mimeType)
-          );
+        for (const videoPart of videoParts) {
+          if (videoPart.uri && videoPart.mimeType) {
+            contentParts.push(
+              createPartFromUri(videoPart.uri, videoPart.mimeType)
+            );
+          }
         }
-      }
 
-      const finalContents = createUserContent(contentParts);
+        const finalContents = createUserContent(contentParts);
 
-      const response = await ai.models.generateContent({
-        model: "gemini-2.5-pro",
-        contents: finalContents,
-      });
+        const response = await ai.models.generateContent({
+          model: "gemini-2.5-pro",
+          contents: finalContents,
+        });
 
-      let result = "";
-      if (response.candidates && response.candidates[0]?.content?.parts) {
-        for (const part of response.candidates[0].content.parts) {
-          if (part.text) {
-            result += part.text;
+        let result = "";
+        if (response.candidates && response.candidates[0]?.content?.parts) {
+          for (const part of response.candidates[0].content.parts) {
+            if (part.text) {
+              result += part.text;
+            }
           }
         }
-      }
 
-      return result || "Video analysis completed but no text response received";
-    } catch (error: any) {
-      throw new Error(`Video analysis failed: ${error.message}`);
-    }
+        return (
+          result || "Video analysis completed but no text response received"
+        );
+      } catch (error: any) {
+        throw new Error(`Video analysis failed: ${error.message}`);
+      }
+    }, "gemini-analyzeVideos");
   },
 };