@mixio-pro/kalaasetu-mcp 2.1.0 → 2.1.2-beta

package/src/tools/gemini.ts

@@ -56,7 +56,7 @@ async function saveWaveFile(
   pcmData: Buffer,
   channels = 1,
   rate = 24000,
-  sampleWidth = 2
+  sampleWidth = 2,
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     const writer = new wav.Writer({
@@ -159,7 +159,7 @@ async function uploadFileToGemini(filePath: string): Promise<any> {
 // Helper function to process video input intelligently
 async function processVideoInput(
   input: string,
-  config?: { fps?: number; startOffset?: string; endOffset?: string }
+  config?: { fps?: number; startOffset?: string; endOffset?: string },
 ): Promise<any> {
   if (isYouTubeUrl(input)) {
     return {
@@ -221,27 +221,27 @@ export const geminiTextToImage = {
       .string()
       .optional()
       .describe(
-        "Supported ratios: 1:1, 3:4, 4:3, 9:16, or 16:9. Default is 9:16."
+        "Supported ratios: 1:1, 3:4, 4:3, 9:16, or 16:9. Default is 9:16.",
       ),
     output_path: z
       .string()
       .optional()
       .describe(
         "Optional: specific local path or filename to save the image (e.g., 'outputs/hero.png'). " +
-          "If omitted, a timestamped filename is generated automatically."
+          "If omitted, a timestamped filename is generated automatically.",
       ),
     reference_images: z
       .array(z.string())
       .optional()
       .describe(
-        "Optional: local paths or URLs of images to use as visual references for style or composition."
+        "Optional: local paths or URLs of images to use as visual references for style or composition.",
       ),
     enhancer_preset: z
       .string()
       .optional()
       .describe(
         "Optional: Name of a prompt enhancer preset to apply (e.g., 'cinematic', 'photorealistic', 'anime'). " +
-          "Automatically enhances the prompt with professional style modifiers."
+          "Automatically enhances the prompt with professional style modifiers.",
       ),
   }),
   timeoutMs: 300000,
@@ -252,79 +252,84 @@ export const geminiTextToImage = {
     reference_images?: string[];
     enhancer_preset?: string;
   }) => {
-    return safeToolExecute(async () => {
-      try {
-        // Apply prompt enhancement if preset specified
-        let enhancedPrompt = args.prompt;
-        if (args.enhancer_preset) {
-          const enhancer = resolveEnhancer(args.enhancer_preset);
-          if (enhancer.hasTransformations()) {
-            enhancedPrompt = enhancer.enhance(args.prompt);
+    return safeToolExecute(
+      async () => {
+        try {
+          // Apply prompt enhancement if preset specified
+          let enhancedPrompt = args.prompt;
+          if (args.enhancer_preset) {
+            const enhancer = resolveEnhancer(args.enhancer_preset);
+            if (enhancer.hasTransformations()) {
+              enhancedPrompt = enhancer.enhance(args.prompt);
+            }
           }
-        }
 
-        const contents: any[] = [enhancedPrompt];
+          const contents: any[] = [enhancedPrompt];
 
-        if (args.reference_images && Array.isArray(args.reference_images)) {
-          for (const refPath of args.reference_images) {
-            contents.push(await fileToGenerativePart(refPath));
+          if (args.reference_images && Array.isArray(args.reference_images)) {
+            for (const refPath of args.reference_images) {
+              contents.push(await fileToGenerativePart(refPath));
+            }
           }
-        }
 
-        const response = await ai.models.generateContent({
-          model: "gemini-3-pro-image-preview",
-          contents: contents,
-          config: {
-            responseModalities: ["TEXT", "IMAGE"],
-            imageConfig: {
-              aspectRatio: args.aspect_ratio || "9:16",
+          const response = await ai.models.generateContent({
+            model: "gemini-3-pro-image-preview",
+            contents: contents,
+            config: {
+              responseModalities: ["TEXT", "IMAGE"],
+              imageConfig: {
+                aspectRatio: args.aspect_ratio || "9:16",
+              },
             },
-          },
-        });
-
-        const images = [];
-        let textResponse = "";
-
-        if (response.candidates && response.candidates[0]?.content?.parts) {
-          for (const part of response.candidates[0].content.parts) {
-            if (part.text) {
-              textResponse += part.text;
-            } else if (part.inlineData?.data) {
-              const imageData = part.inlineData.data;
-              // Always save the image - use provided path or generate one
-              const outputPath =
-                args.output_path ||
-                generateTimestampedFilename("generated_image.png");
-              const storage = getStorage();
-              const url = await storage.writeFile(
-                outputPath,
-                Buffer.from(imageData, "base64")
-              );
-              images.push({
-                url,
-                filename: outputPath,
-                mimeType: "image/png",
-              });
+          });
+
+          const images = [];
+          let textResponse = "";
+
+          if (response.candidates && response.candidates[0]?.content?.parts) {
+            for (const part of response.candidates[0].content.parts) {
+              if (part.text) {
+                textResponse += part.text;
+              } else if (part.inlineData?.data) {
+                const imageData = part.inlineData.data;
+                // Always save the image - use provided path or generate one
+                const outputPath =
+                  args.output_path ||
+                  generateTimestampedFilename("generated_image.png");
+                const storage = getStorage();
+                const url = await storage.writeFile(
+                  outputPath,
+                  Buffer.from(imageData, "base64"),
+                );
+                images.push({
+                  url,
+                  filename: outputPath,
+                  mimeType: "image/png",
+                });
+              }
             }
           }
-        }
 
-        if (images.length > 0) {
-          // Return the URL directly for easy parsing
-          return JSON.stringify({
-            url: images?.[0]?.url,
-            images,
-            message: textResponse || "Image generated successfully",
-          });
-        }
+          if (images.length > 0) {
+            // Return the URL directly for easy parsing
+            return JSON.stringify({
+              url: images?.[0]?.url,
+              images,
+              message: textResponse || "Image generated successfully",
+            });
+          }
 
-        return (
-          textResponse || "Image generation completed but no image was produced"
-        );
-      } catch (error: any) {
-        throw new Error(`Image generation failed: ${error.message}`);
-      }
-    }, "gemini-generateImage");
+          return (
+            textResponse ||
+            "Image generation completed but no image was produced"
+          );
+        } catch (error: any) {
+          throw new Error(`Image generation failed: ${error.message}`);
+        }
+      },
+      "gemini-generateImage",
+      { toolName: "generateImage" },
+    );
   },
 };
 
@@ -339,31 +344,31 @@ export const geminiEditImage = {
     image_path: z
       .string()
       .describe(
-        "Absolute local path or URL to the source image file to be edited."
+        "Absolute local path or URL to the source image file to be edited.",
       ),
     prompt: z
       .string()
       .describe(
-        "Instructional text describing the edits or modifications required."
+        "Instructional text describing the edits or modifications required.",
       ),
     output_path: z
       .string()
       .optional()
       .describe(
-        "Optional: specific local path to save the edited result. Defaults to generated timestamp."
+        "Optional: specific local path to save the edited result. Defaults to generated timestamp.",
       ),
     reference_images: z
       .array(z.string())
       .optional()
       .describe(
-        "Optional: additional images to guide the edit (e.g., to reference a specific character or object style)."
+        "Optional: additional images to guide the edit (e.g., to reference a specific character or object style).",
       ),
     enhancer_preset: z
       .string()
       .optional()
       .describe(
         "Optional: Name of a prompt enhancer preset to apply (e.g., 'cinematic', 'photorealistic'). " +
-          "Enhances the edit instructions with professional style modifiers."
+          "Enhances the edit instructions with professional style modifiers.",
       ),
   }),
   timeoutMs: 300000,
@@ -374,70 +379,74 @@ export const geminiEditImage = {
     reference_images?: string[];
     enhancer_preset?: string;
   }) => {
-    return safeToolExecute(async () => {
-      try {
-        // Apply prompt enhancement if preset specified
-        let enhancedPrompt = args.prompt;
-        if (args.enhancer_preset) {
-          const enhancer = resolveEnhancer(args.enhancer_preset);
-          if (enhancer.hasTransformations()) {
-            enhancedPrompt = enhancer.enhance(args.prompt);
+    return safeToolExecute(
+      async () => {
+        try {
+          // Apply prompt enhancement if preset specified
+          let enhancedPrompt = args.prompt;
+          if (args.enhancer_preset) {
+            const enhancer = resolveEnhancer(args.enhancer_preset);
+            if (enhancer.hasTransformations()) {
+              enhancedPrompt = enhancer.enhance(args.prompt);
+            }
           }
-        }
 
-        const imagePart = await fileToGenerativePart(args.image_path);
-        const contents: any[] = [enhancedPrompt, imagePart];
+          const imagePart = await fileToGenerativePart(args.image_path);
+          const contents: any[] = [enhancedPrompt, imagePart];
 
-        if (args.reference_images) {
-          for (const refPath of args.reference_images) {
-            contents.push(await fileToGenerativePart(refPath));
+          if (args.reference_images) {
+            for (const refPath of args.reference_images) {
+              contents.push(await fileToGenerativePart(refPath));
+            }
           }
-        }
 
-        const response = await ai.models.generateContent({
-          model: "gemini-3-pro-image-preview",
-          contents: contents,
-        });
-
-        const images = [];
-        let textResponse = "";
-
-        if (response.candidates && response.candidates[0]?.content?.parts) {
-          for (const part of response.candidates[0].content.parts) {
-            if (part.text) {
-              textResponse += part.text;
-            } else if (part.inlineData?.data) {
-              const imageData = part.inlineData.data;
-              if (args.output_path) {
-                const storage = getStorage();
-                const url = await storage.writeFile(
-                  args.output_path,
-                  Buffer.from(imageData, "base64")
-                );
-                images.push({
-                  url,
-                  filename: args.output_path,
-                  mimeType: "image/png",
-                });
+          const response = await ai.models.generateContent({
+            model: "gemini-3-pro-image-preview",
+            contents: contents,
+          });
+
+          const images = [];
+          let textResponse = "";
+
+          if (response.candidates && response.candidates[0]?.content?.parts) {
+            for (const part of response.candidates[0].content.parts) {
+              if (part.text) {
+                textResponse += part.text;
+              } else if (part.inlineData?.data) {
+                const imageData = part.inlineData.data;
+                if (args.output_path) {
+                  const storage = getStorage();
+                  const url = await storage.writeFile(
+                    args.output_path,
+                    Buffer.from(imageData, "base64"),
+                  );
+                  images.push({
+                    url,
+                    filename: args.output_path,
+                    mimeType: "image/png",
+                  });
+                }
               }
             }
           }
-        }
 
-        if (images.length > 0) {
-          return JSON.stringify({
-            images,
-            message: textResponse || "Image edited successfully",
-          });
-        }
+          if (images.length > 0) {
+            return JSON.stringify({
+              images,
+              message: textResponse || "Image edited successfully",
+            });
+          }
 
-        return (
-          textResponse || "Image editing completed but no response received"
-        );
-      } catch (error: any) {
-        throw new Error(`Image editing failed: ${error.message}`);
-      }
-    }, "gemini-editImage");
+          return (
+            textResponse || "Image editing completed but no response received"
+          );
+        } catch (error: any) {
+          throw new Error(`Image editing failed: ${error.message}`);
+        }
+      },
+      "gemini-editImage",
+      { toolName: "editImage" },
+    );
   },
 };
 
@@ -452,7 +461,7 @@ export const geminiAnalyzeImages = {
     image_paths: z
       .array(z.string())
       .describe(
-        "An array of absolute local file paths or publicly accessible URLs to analyze."
+        "An array of absolute local file paths or publicly accessible URLs to analyze.",
       ),
     prompt: z
       .string()
@@ -460,60 +469,64 @@ export const geminiAnalyzeImages = {
   }),
   timeoutMs: 300000,
   execute: async (args: { image_paths: string[]; prompt: string }) => {
-    return safeToolExecute(async () => {
-      try {
-        // Handle array parsing
-        if (!args.image_paths) {
-          throw new Error("Image paths not provided");
-        }
+    return safeToolExecute(
+      async () => {
+        try {
+          // Handle array parsing
+          if (!args.image_paths) {
+            throw new Error("Image paths not provided");
+          }
 
-        // Convert to array if passed as string
-        let imagePaths: string[];
-        if (typeof args.image_paths === "string") {
-          const strValue = args.image_paths as string;
-          if (strValue.startsWith("[") && strValue.endsWith("]")) {
-            try {
-              imagePaths = JSON.parse(strValue);
-            } catch {
-              throw new Error("Invalid image_paths format");
+          // Convert to array if passed as string
+          let imagePaths: string[];
+          if (typeof args.image_paths === "string") {
+            const strValue = args.image_paths as string;
+            if (strValue.startsWith("[") && strValue.endsWith("]")) {
+              try {
+                imagePaths = JSON.parse(strValue);
+              } catch {
+                throw new Error("Invalid image_paths format");
+              }
+            } else {
+              imagePaths = [strValue];
             }
+          } else if (Array.isArray(args.image_paths)) {
+            imagePaths = args.image_paths;
           } else {
-            imagePaths = [strValue];
+            throw new Error("Invalid image_paths: must be array or string");
           }
-        } else if (Array.isArray(args.image_paths)) {
-          imagePaths = args.image_paths;
-        } else {
-          throw new Error("Invalid image_paths: must be array or string");
-        }
 
-        if (imagePaths.length === 0) {
-          throw new Error("At least one image path must be provided");
-        }
+          if (imagePaths.length === 0) {
+            throw new Error("At least one image path must be provided");
+          }
 
-        const contents: any[] = [args.prompt];
+          const contents: any[] = [args.prompt];
 
-        for (const imagePath of imagePaths) {
-          contents.push(await fileToGenerativePart(imagePath));
-        }
+          for (const imagePath of imagePaths) {
+            contents.push(await fileToGenerativePart(imagePath));
+          }
 
-        const response = await ai.models.generateContent({
-          model: "gemini-2.5-pro",
-          contents: contents,
-        });
+          const response = await ai.models.generateContent({
+            model: "gemini-2.5-pro",
+            contents: contents,
+          });
 
-        let result = "";
-        if (response.candidates && response.candidates[0]?.content?.parts) {
-          for (const part of response.candidates[0].content.parts) {
-            if (part.text) {
-              result += part.text;
+          let result = "";
+          if (response.candidates && response.candidates[0]?.content?.parts) {
+            for (const part of response.candidates[0].content.parts) {
+              if (part.text) {
+                result += part.text;
+              }
             }
           }
+          return result || "Analysis completed but no text response received";
+        } catch (error: any) {
+          throw new Error(`Image analysis failed: ${error.message}`);
         }
-        return result || "Analysis completed but no text response received";
-      } catch (error: any) {
-        throw new Error(`Image analysis failed: ${error.message}`);
-      }
-    }, "gemini-analyzeImages");
+      },
+      "gemini-analyzeImages",
+      { toolName: "analyzeImages" },
+    );
   },
 };
 
@@ -529,13 +542,13 @@ export const geminiSingleSpeakerTts = {
     voice_name: z
       .string()
       .describe(
-        "Supported voices: 'Despina' (Female, versatile), 'Kore' (Female, calm), 'Erinome' (Female, expressive), or 'Enceladus' (Male, neutral)."
+        "Supported voices: 'Despina' (Female, versatile), 'Kore' (Female, calm), 'Erinome' (Female, expressive), or 'Enceladus' (Male, neutral).",
       ),
     output_path: z
       .string()
       .optional()
       .describe(
-        "Optional: Output WAV file path. Defaults to a timestamped filename in the output directory."
+        "Optional: Output WAV file path. Defaults to a timestamped filename in the output directory.",
       ),
   }),
   timeoutMs: 300000,
@@ -544,50 +557,54 @@ export const geminiSingleSpeakerTts = {
     voice_name: string;
     output_path?: string;
   }) => {
-    return safeToolExecute(async () => {
-      try {
-        const response = await ai.models.generateContent({
-          model: "gemini-2.5-pro-preview-tts",
-          contents: [{ parts: [{ text: args.text }] }],
-          config: {
-            responseModalities: ["AUDIO"],
-            speechConfig: {
-              voiceConfig: {
-                prebuiltVoiceConfig: {
-                  voiceName: args.voice_name || "Despina",
+    return safeToolExecute(
+      async () => {
+        try {
+          const response = await ai.models.generateContent({
+            model: "gemini-2.5-pro-preview-tts",
+            contents: [{ parts: [{ text: args.text }] }],
+            config: {
+              responseModalities: ["AUDIO"],
+              speechConfig: {
+                voiceConfig: {
+                  prebuiltVoiceConfig: {
+                    voiceName: args.voice_name || "Despina",
+                  },
                 },
               },
             },
-          },
-        });
+          });
 
-        const data =
-          response.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
-        if (!data) {
-          throw new Error("No audio data received from Gemini API");
-        }
+          const data =
+            response.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
+          if (!data) {
+            throw new Error("No audio data received from Gemini API");
+          }
 
-        const audioBuffer = Buffer.from(data, "base64");
+          const audioBuffer = Buffer.from(data, "base64");
 
-        // Use provided output path or generate default with timestamp
-        const outputPath =
-          args.output_path || generateTimestampedFilename("voice_output.wav");
+          // Use provided output path or generate default with timestamp
+          const outputPath =
+            args.output_path || generateTimestampedFilename("voice_output.wav");
 
-        const storage = getStorage();
-        const url = await storage.writeFile(outputPath, audioBuffer);
-
-        return JSON.stringify({
-          audio: {
-            url,
-            filename: outputPath,
-            mimeType: "audio/wav",
-          },
-          message: "Audio generated successfully",
-        });
-      } catch (error: any) {
-        throw new Error(`Voice generation failed: ${error.message}`);
-      }
-    }, "gemini-generateSpeech");
+          const storage = getStorage();
+          const url = await storage.writeFile(outputPath, audioBuffer);
+
+          return JSON.stringify({
+            audio: {
+              url,
+              filename: outputPath,
+              mimeType: "audio/wav",
+            },
+            message: "Audio generated successfully",
+          });
+        } catch (error: any) {
+          throw new Error(`Voice generation failed: ${error.message}`);
+        }
+      },
+      "gemini-generateSpeech",
+      { toolName: "generateSpeech" },
+    );
   },
 };
 
@@ -603,18 +620,18 @@ export const geminiAnalyzeVideos = {
       .array(z.string())
       .describe(
         "An array containing absolute paths to local videos or YouTube URLs. Max 10 per request. " +
-          "Note: Local files are automatically optimized for processing."
+          "Note: Local files are automatically optimized for processing.",
       ),
     prompt: z
       .string()
       .describe(
-        "The question or instruction regarding the video. Use MM:SS or HH:MM:SS for precise time references."
+        "The question or instruction regarding the video. Use MM:SS or HH:MM:SS for precise time references.",
       ),
     fps: z
       .number()
       .optional()
       .describe(
-        "Optional: Target frames per second for processing. Lower FPS (1-5) is recommended for long videos to save tokens."
+        "Optional: Target frames per second for processing. Lower FPS (1-5) is recommended for long videos to save tokens.",
       ),
     start_offset: z
       .string()
@@ -628,7 +645,7 @@ export const geminiAnalyzeVideos = {
       .string()
       .optional()
       .describe(
-        "Processing resolution: 'default' or 'low'. 'low' significantly reduces token usage for simple visual tasks."
+        "Processing resolution: 'default' or 'low'. 'low' significantly reduces token usage for simple visual tasks.",
       ),
   }),
   timeoutMs: 300000,
@@ -640,90 +657,94 @@ export const geminiAnalyzeVideos = {
     end_offset?: string;
     media_resolution?: string;
   }) => {
-    return safeToolExecute(async () => {
-      try {
-        // Handle array parsing
-        if (!args.video_inputs) {
-          throw new Error("Video inputs not provided");
-        }
+    return safeToolExecute(
+      async () => {
+        try {
+          // Handle array parsing
+          if (!args.video_inputs) {
+            throw new Error("Video inputs not provided");
+          }
 
-        // Convert to array if passed as string
-        let videoInputs: string[];
-        if (typeof args.video_inputs === "string") {
-          const strValue = args.video_inputs as string;
-          if (strValue.startsWith("[") && strValue.endsWith("]")) {
-            try {
-              videoInputs = JSON.parse(strValue);
-            } catch {
-              throw new Error("Invalid video_inputs format");
+          // Convert to array if passed as string
+          let videoInputs: string[];
+          if (typeof args.video_inputs === "string") {
+            const strValue = args.video_inputs as string;
+            if (strValue.startsWith("[") && strValue.endsWith("]")) {
+              try {
+                videoInputs = JSON.parse(strValue);
+              } catch {
+                throw new Error("Invalid video_inputs format");
+              }
+            } else {
+              videoInputs = [strValue];
             }
+          } else if (Array.isArray(args.video_inputs)) {
+            videoInputs = args.video_inputs;
           } else {
-            videoInputs = [strValue];
+            throw new Error("Invalid video_inputs: must be array or string");
           }
-        } else if (Array.isArray(args.video_inputs)) {
-          videoInputs = args.video_inputs;
-        } else {
-          throw new Error("Invalid video_inputs: must be array or string");
-        }
 
-        if (videoInputs.length === 0) {
-          throw new Error("At least one video input must be provided");
-        }
+          if (videoInputs.length === 0) {
+            throw new Error("At least one video input must be provided");
+          }
 
-        if (videoInputs.length > 10) {
-          throw new Error(
-            "Maximum 10 videos per request allowed for Gemini 2.5+ models"
-          );
-        }
+          if (videoInputs.length > 10) {
+            throw new Error(
+              "Maximum 10 videos per request allowed for Gemini 2.5+ models",
+            );
+          }
 
-        // Prepare video parts for content
-        const videoParts: any[] = [];
+          // Prepare video parts for content
+          const videoParts: any[] = [];
 
-        // Process each video input
-        for (const videoInput of videoInputs) {
-          const videoConfig = {
-            fps: args.fps || (isYouTubeUrl(videoInput) ? 1 : 5), // Default 5 FPS for local, 1 FPS for YouTube
-            startOffset: args.start_offset,
-            endOffset: args.end_offset,
-          };
+          // Process each video input
+          for (const videoInput of videoInputs) {
+            const videoConfig = {
+              fps: args.fps || (isYouTubeUrl(videoInput) ? 1 : 5), // Default 5 FPS for local, 1 FPS for YouTube
+              startOffset: args.start_offset,
+              endOffset: args.end_offset,
+            };
 
-          const videoPart = await processVideoInput(videoInput, videoConfig);
-          videoParts.push(videoPart);
-        }
+            const videoPart = await processVideoInput(videoInput, videoConfig);
+            videoParts.push(videoPart);
+          }
 
-        // Build content using createUserContent and createPartFromUri for uploaded files
-        const contentParts: any[] = [args.prompt];
+          // Build content using createUserContent and createPartFromUri for uploaded files
+          const contentParts: any[] = [args.prompt];
 
-        for (const videoPart of videoParts) {
-          if (videoPart.uri && videoPart.mimeType) {
-            contentParts.push(
-              createPartFromUri(videoPart.uri, videoPart.mimeType)
-            );
+          for (const videoPart of videoParts) {
+            if (videoPart.uri && videoPart.mimeType) {
+              contentParts.push(
+                createPartFromUri(videoPart.uri, videoPart.mimeType),
+              );
+            }
           }
-        }
 
-        const finalContents = createUserContent(contentParts);
+          const finalContents = createUserContent(contentParts);
 
-        const response = await ai.models.generateContent({
-          model: "gemini-2.5-pro",
-          contents: finalContents,
-        });
+          const response = await ai.models.generateContent({
+            model: "gemini-2.5-pro",
+            contents: finalContents,
+          });
 
-        let result = "";
-        if (response.candidates && response.candidates[0]?.content?.parts) {
-          for (const part of response.candidates[0].content.parts) {
-            if (part.text) {
-              result += part.text;
+          let result = "";
+          if (response.candidates && response.candidates[0]?.content?.parts) {
+            for (const part of response.candidates[0].content.parts) {
+              if (part.text) {
+                result += part.text;
+              }
             }
           }
-        }
 
-        return (
-          result || "Video analysis completed but no text response received"
-        );
-      } catch (error: any) {
-        throw new Error(`Video analysis failed: ${error.message}`);
-      }
-    }, "gemini-analyzeVideos");
+          return (
+            result || "Video analysis completed but no text response received"
+          );
+        } catch (error: any) {
+          throw new Error(`Video analysis failed: ${error.message}`);
+        }
+      },
+      "gemini-analyzeVideos",
+      { toolName: "analyzeVideos" },
+    );
   },
 };