@mixio-pro/kalaasetu-mcp 1.0.3 → 1.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mixio-pro/kalaasetu-mcp",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "A powerful Model Context Protocol server providing AI tools for content generation and analysis",
   "type": "module",
   "module": "src/index.ts",

package/src/tools/image-to-video.sdk-backup.ts

@@ -0,0 +1,161 @@
+import { z } from "zod";
+import * as fs from "fs";
+import * as path from "path";
+import { GoogleGenAI } from "@google/genai";
+
+async function wait(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function fileToBase64(filePath: string): { data: string; mimeType: string } {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`File not found: ${filePath}`);
+  }
+  const buf = fs.readFileSync(filePath);
+  const data = Buffer.from(buf).toString("base64");
+  // Detect mime type from extension
+  const ext = path.extname(filePath).toLowerCase();
+  const mimeType = ext === '.jpg' || ext === '.jpeg' ? 'image/jpeg' : 
+                   ext === '.png' ? 'image/png' :
+                   ext === '.webp' ? 'image/webp' : 'image/png';
+  return { data, mimeType };
+}
+
+export const imageToVideo = {
+  name: "imageToVideo",
+  description: "Generate videos from an image as starting first frame using Vertex Veo models (predictLongRunning + fetchPredictOperation).",
+  parameters: z.object({
+    prompt: z.string().describe("Text description for the video"),
+    image_path: z.string().optional().describe("Path to source image for image-to-video generation"),
+    aspect_ratio: z.string().optional().describe("Video aspect ratio: '16:9' or '9:16' (default: '9:16')"),
+    duration_seconds: z.number().optional().describe("Video duration in seconds: 4, 6, or 8 (default: 6)"),
+    resolution: z.string().optional().describe("Video resolution: '720p' or '1080p' (default: '720p')"),
+    negative_prompt: z.string().optional().describe("Text describing what not to include in the video"),
+    person_generation: z.string().optional().describe("Controls generation of people: 'allow_adult' (default for image-to-video) or 'allow_all'"),
+    reference_images: z.array(z.string()).optional().describe("Additional image paths for reference (max 3)"),
+    output_path: z.string().optional().describe("Output MP4 file path (if multiple predictions, index suffix is added)"),
+    gemini_api_key: z.string().optional().describe("Gemini API key (uses GEMINI_API_KEY env var if not provided)"),
+    model_id: z.string().optional().describe("Model ID (default: veo-2.0-generate-001)"),
+  }),
+  execute: async (args: {
+    prompt: string;
+    image_path?: string;
+    aspect_ratio?: string;
+    duration_seconds?: number;
+    resolution?: string;
+    negative_prompt?: string;
+    person_generation?: string;
+    reference_images?: string[];
+    output_path?: string;
+    gemini_api_key?: string;
+    model_id?: string;
+  }) => {
+    const apiKey = args.gemini_api_key || process.env.GEMINI_API_KEY;
+    if (!apiKey) {
+      throw new Error("Gemini API key is required. Set GEMINI_API_KEY environment variable or pass gemini_api_key parameter. Get one at https://aistudio.google.com/app/apikey");
+    }
+
+    const model = args.model_id || "veo-2.0-generate-001";
+    
+    // Initialize Google GenAI client
+    const genai = new GoogleGenAI({ apiKey });
+
+    // Build config for video generation
+    const config: any = {};
+    
+    if (args.duration_seconds !== undefined) {
+      config.duration_seconds = args.duration_seconds;
+    } else {
+      config.duration_seconds = 6; // default
+    }
+    
+    if (args.aspect_ratio) {
+      config.aspect_ratio = args.aspect_ratio;
+    }
+
+    try {
+      // Start video generation operation
+      console.log(`Starting video generation with model: ${model}`);
+      let operation = await genai.models.generateVideos({
+        model,
+        prompt: args.prompt,
+        config,
+      });
+
+      console.log("Operation started, waiting for completion...");
+      
+      // Poll until operation is complete (max 10 minutes)
+      let tries = 0;
+      const maxTries = 60; // 10 minutes with 10s intervals
+      
+      while (!operation.done && tries < maxTries) {
+        await wait(10000); // Wait 10 seconds
+        tries++;
+        console.log(`Polling attempt ${tries}/${maxTries}...`);
+        
+        operation = await genai.operations.getVideosOperation({
+          operation: operation,
+        });
+      }
+
+      if (!operation.done) {
+        throw new Error("Video generation timed out after 10 minutes");
+      }
+
+      console.log("Operation completed!");
+      console.log("Full Response:", JSON.stringify(operation.response, null, 2));
+
+      // Extract generated videos from response
+      const generatedVideos = operation.response?.generatedVideos || [];
+      
+      if (!generatedVideos || generatedVideos.length === 0) {
+        const respStr = JSON.stringify(operation.response, null, 2);
+        return `Video generation completed but no videos found in response.\n\nFull Response:\n${respStr.slice(0, 2000)}${respStr.length > 2000 ? '\n...(truncated)' : ''}`;
+      }
+
+      // Download and save videos
+      const outputs: string[] = [];
+      
+      for (let i = 0; i < generatedVideos.length; i++) {
+        const generatedVideo = generatedVideos[i];
+        const videoUri = generatedVideo?.video?.uri;
+        
+        if (!videoUri) {
+          console.warn(`Video ${i} has no URI`);
+          continue;
+        }
+
+        console.log(`Downloading video ${i + 1}/${generatedVideos.length}...`);
+        
+        // Download video from URI
+        const videoUrl = `${videoUri}&key=${apiKey}`;
+        const response = await fetch(videoUrl);
+        
+        if (!response.ok) {
+          throw new Error(`Failed to download video: ${response.status} ${response.statusText}`);
+        }
+        
+        const buffer = await response.arrayBuffer();
+        
+        // Save video to file
+        const filePath = args.output_path
+          ? (i === 0 ? args.output_path : args.output_path.replace(/\.mp4$/i, `_${i}.mp4`))
+          : `video_output_${Date.now()}${i === 0 ? '' : '_' + i}.mp4`;
+        const absPath = path.resolve(filePath);
+        
+        fs.writeFileSync(absPath, Buffer.from(buffer));
+        outputs.push(absPath);
+        console.log(`Saved video to: ${absPath}`);
+      }
+
+      if (outputs.length > 0) {
+        return `Video(s) saved successfully:\n${outputs.map((p, i) => `${i + 1}. ${p}`).join('\n')}`;
+      }
+
+      return "Video generation completed but no videos were saved.";
+      
+    } catch (error: any) {
+      throw new Error(`Video generation failed: ${error.message || JSON.stringify(error)}`);
+    }
+  },
+};

package/src/tools/image-to-video.ts

@@ -1,7 +1,6 @@
 import { z } from "zod";
 import * as fs from "fs";
 import * as path from "path";
-import { GoogleGenAI } from "@google/genai";
 
 async function wait(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -23,25 +22,27 @@ function fileToBase64(filePath: string): { data: string; mimeType: string } {
 
 export const imageToVideo = {
   name: "imageToVideo",
-  description: "Generate videos from an image as starting first frame using Vertex Veo models (predictLongRunning + fetchPredictOperation).",
+  description: "Generate videos from an image as starting first frame using Gemini Veo models via HTTP API with Gemini API key.",
   parameters: z.object({
     prompt: z.string().describe("Text description for the video"),
     image_path: z.string().optional().describe("Path to source image for image-to-video generation"),
+    last_frame_path: z.string().optional().describe("Path to last frame image to guide ending frame (optional)"),
     aspect_ratio: z.string().optional().describe("Video aspect ratio: '16:9' or '9:16' (default: '9:16')"),
-    duration_seconds: z.number().optional().describe("Video duration in seconds: 4, 6, or 8 (default: 6)"),
+    duration_seconds: z.union([z.string(), z.number()]).optional().describe("Video duration in seconds: 4, 6, or 8 (default: 6)"),
     resolution: z.string().optional().describe("Video resolution: '720p' or '1080p' (default: '720p')"),
     negative_prompt: z.string().optional().describe("Text describing what not to include in the video"),
     person_generation: z.string().optional().describe("Controls generation of people: 'allow_adult' (default for image-to-video) or 'allow_all'"),
     reference_images: z.array(z.string()).optional().describe("Additional image paths for reference (max 3)"),
     output_path: z.string().optional().describe("Output MP4 file path (if multiple predictions, index suffix is added)"),
     gemini_api_key: z.string().optional().describe("Gemini API key (uses GEMINI_API_KEY env var if not provided)"),
-    model_id: z.string().optional().describe("Model ID (default: veo-2.0-generate-001)"),
+    model_id: z.string().optional().describe("Model ID (default: veo-3.1-generate-preview)"),
   }),
   execute: async (args: {
     prompt: string;
     image_path?: string;
+    last_frame_path?: string;
     aspect_ratio?: string;
-    duration_seconds?: number;
+    duration_seconds?: string | number;
     resolution?: string;
     negative_prompt?: string;
     person_generation?: string;
@@ -55,87 +56,188 @@ export const imageToVideo = {
       throw new Error("Gemini API key is required. Set GEMINI_API_KEY environment variable or pass gemini_api_key parameter. Get one at https://aistudio.google.com/app/apikey");
     }
 
-    const model = args.model_id || "veo-2.0-generate-001";
-    
-    // Initialize Google GenAI client
-    const genai = new GoogleGenAI({ apiKey });
-
-    // Build config for video generation
-    const config: any = {};
-    
-    if (args.duration_seconds !== undefined) {
-      config.duration_seconds = args.duration_seconds;
-    } else {
-      config.duration_seconds = 6; // default
-    }
-    
-    if (args.aspect_ratio) {
-      config.aspect_ratio = args.aspect_ratio;
-    }
+    const modelId = args.model_id || "veo-3.1-generate-preview";
+    const baseUrl = "https://generativelanguage.googleapis.com/v1beta";
+
+    // Convert duration_seconds to number, handling both string and number inputs
+    const durationSeconds = args.duration_seconds 
+      ? (typeof args.duration_seconds === 'string' ? parseInt(args.duration_seconds) : args.duration_seconds)
+      : 6; // default
 
     try {
-      // Start video generation operation
-      console.log(`Starting video generation with model: ${model}`);
-      let operation = await genai.models.generateVideos({
-        model,
-        prompt: args.prompt,
-        config,
+      // Build the request body for predictLongRunning
+      const instances: any[] = [
+        {
+          prompt: args.prompt,
+        },
+      ];
+
+      // Add image if provided (first frame)
+      if (args.image_path) {
+        const { data, mimeType } = fileToBase64(args.image_path);
+        instances[0].image = {
+          bytesBase64Encoded: data,
+          mimeType,
+        };
+      }
+
+      // Add last frame if provided (for interpolation)
+      if (args.last_frame_path) {
+        const { data, mimeType } = fileToBase64(args.last_frame_path);
+        instances[0].lastFrame = {
+          bytesBase64Encoded: data,
+          mimeType,
+        };
+      }
+
+      // Add reference images if provided
+      if (args.reference_images && args.reference_images.length > 0) {
+        const refImages = args.reference_images.slice(0, 3).map((imgPath) => {
+          const { data, mimeType } = fileToBase64(imgPath);
+          return {
+            image: {
+              bytesBase64Encoded: data,
+              mimeType,
+            },
+            referenceType: "asset",
+          };
+        });
+        instances[0].referenceImages = refImages;
+      }
+
+      // Build parameters - NOTE: Parameters go in "parameters" object, not in instances
+      const parameters: any = {};
+
+      if (args.aspect_ratio) {
+        parameters.aspectRatio = args.aspect_ratio;
+      }
+      
+      if (durationSeconds) {
+        parameters.durationSeconds = durationSeconds;
+      }
+      
+      if (args.resolution) {
+        parameters.resolution = args.resolution;
+      }
+
+      if (args.negative_prompt) {
+        parameters.negativePrompt = args.negative_prompt;
+      }
+
+      if (args.person_generation) {
+        parameters.personGeneration = args.person_generation;
+      }
+
+      const requestBody: any = {
+        instances,
+      };
+
+      // Only add parameters if we have any
+      if (Object.keys(parameters).length > 0) {
+        requestBody.parameters = parameters;
+      }
+
+      console.log(`Starting video generation with model: ${modelId}`);
+
+      // Step 1: Start the long-running operation
+      const url = `${baseUrl}/models/${modelId}:predictLongRunning`;
+      const response = await fetch(url, {
+        method: "POST",
+        headers: {
+          "x-goog-api-key": apiKey,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(requestBody),
       });
 
-      console.log("Operation started, waiting for completion...");
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`Video generation request failed: ${response.status} ${errorText}`);
+      }
+
+      const operation = await response.json() as any;
+      const operationName: string = operation.name || operation.operation || "";
       
-      // Poll until operation is complete (max 10 minutes)
+      if (!operationName) {
+        throw new Error("No operation name returned from API");
+      }
+
+      console.log(`Operation started: ${operationName}`);
+
+      // Step 2: Poll the operation status by getting the operation directly
+      let currentOp: any = operation;
+      let done = !!operation.done;
       let tries = 0;
       const maxTries = 60; // 10 minutes with 10s intervals
-      
-      while (!operation.done && tries < maxTries) {
+
+      while (!done && tries < maxTries) {
         await wait(10000); // Wait 10 seconds
         tries++;
         console.log(`Polling attempt ${tries}/${maxTries}...`);
-        
-        operation = await genai.operations.getVideosOperation({
-          operation: operation,
+
+        // Poll by getting the operation status directly
+        const pollResponse = await fetch(`${baseUrl}/${operationName}`, {
+          method: "GET",
+          headers: {
+            "x-goog-api-key": apiKey,
+          },
         });
+
+        if (!pollResponse.ok) {
+          const errorText = await pollResponse.text();
+          throw new Error(`Operation polling failed: ${pollResponse.status} ${errorText}`);
+        }
+
+        currentOp = await pollResponse.json() as any;
+        done = !!currentOp.done || !!currentOp.response;
       }
 
-      if (!operation.done) {
+      if (!done) {
         throw new Error("Video generation timed out after 10 minutes");
       }
 
       console.log("Operation completed!");
-      console.log("Full Response:", JSON.stringify(operation.response, null, 2));
 
-      // Extract generated videos from response
-      const generatedVideos = operation.response?.generatedVideos || [];
+      // Step 3: Extract and download videos
+      const resp = currentOp.response || currentOp;
       
-      if (!generatedVideos || generatedVideos.length === 0) {
-        const respStr = JSON.stringify(operation.response, null, 2);
-        return `Video generation completed but no videos found in response.\n\nFull Response:\n${respStr.slice(0, 2000)}${respStr.length > 2000 ? '\n...(truncated)' : ''}`;
+      // The response structure is: response.generateVideoResponse.generatedSamples[].video.uri
+      const generateVideoResponse = resp?.generateVideoResponse;
+      const generatedSamples = generateVideoResponse?.generatedSamples || [];
+      
+      if (!generatedSamples || generatedSamples.length === 0) {
+        let jsonStr = "";
+        try { jsonStr = JSON.stringify(resp, null, 2); } catch {}
+        return `Video generation completed but no generatedSamples found.\n\nFull Response:\n${jsonStr.slice(0, 1000)}${jsonStr.length > 1000 ? '\n...(truncated)' : ''}`;
       }
 
-      // Download and save videos
       const outputs: string[] = [];
-      
-      for (let i = 0; i < generatedVideos.length; i++) {
-        const generatedVideo = generatedVideos[i];
-        const videoUri = generatedVideo?.video?.uri;
+
+      // Download videos from URIs
+      for (let i = 0; i < generatedSamples.length; i++) {
+        const sample = generatedSamples[i];
+        const videoUri = sample?.video?.uri;
         
         if (!videoUri) {
-          console.warn(`Video ${i} has no URI`);
+          console.warn(`Sample ${i} has no video URI`);
           continue;
         }
 
-        console.log(`Downloading video ${i + 1}/${generatedVideos.length}...`);
+        console.log(`Downloading video ${i + 1}/${generatedSamples.length} from ${videoUri}...`);
         
-        // Download video from URI
-        const videoUrl = `${videoUri}&key=${apiKey}`;
-        const response = await fetch(videoUrl);
-        
-        if (!response.ok) {
-          throw new Error(`Failed to download video: ${response.status} ${response.statusText}`);
+        // Download video from URI with API key
+        const videoResponse = await fetch(videoUri, {
+          method: "GET",
+          headers: {
+            "x-goog-api-key": apiKey,
+          },
+        });
+
+        if (!videoResponse.ok) {
+          throw new Error(`Failed to download video: ${videoResponse.status} ${videoResponse.statusText}`);
         }
-        
-        const buffer = await response.arrayBuffer();
+
+        const videoBuffer = await videoResponse.arrayBuffer();
         
         // Save video to file
         const filePath = args.output_path
@@ -143,7 +245,7 @@ export const imageToVideo = {
           : `video_output_${Date.now()}${i === 0 ? '' : '_' + i}.mp4`;
         const absPath = path.resolve(filePath);
         
-        fs.writeFileSync(absPath, Buffer.from(buffer));
+        fs.writeFileSync(absPath, Buffer.from(videoBuffer));
         outputs.push(absPath);
         console.log(`Saved video to: ${absPath}`);
       }