@mixio-pro/kalaasetu-mcp 1.0.4 → 1.0.5-beta

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mixio-pro/kalaasetu-mcp",
-  "version": "1.0.4",
+  "version": "1.0.5-beta",
   "description": "A powerful Model Context Protocol server providing AI tools for content generation and analysis",
   "type": "module",
   "module": "src/index.ts",

package/src/tools/image-to-video.ts

@@ -1,99 +1,125 @@
-import { z } from "zod";
+// @ts-nocheck
 import * as fs from "fs";
+import { GoogleAuth } from "google-auth-library";
+import { exec } from "child_process";
 import * as path from "path";
+import { z } from "zod";
 
 async function wait(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-function fileToBase64(filePath: string): { data: string; mimeType: string } {
-  if (!fs.existsSync(filePath)) {
-    throw new Error(`File not found: ${filePath}`);
+async function fetchAccessToken(): Promise<string> {
+  try {
+    const auth = new GoogleAuth({ scopes: ["https://www.googleapis.com/auth/cloud-platform"] });
+    const client = await auth.getClient();
+    const token = await client.getAccessToken();
+    if (!token || typeof token !== "string") {
+      throw new Error("No token from GoogleAuth");
+    }
+    return token;
+  } catch (e) {
+    // Fallback to gcloud
+    return await new Promise((resolve, reject) => {
+      exec("gcloud auth print-access-token", (err, stdout, stderr) => {
+        if (err) {
+          reject(new Error(`Failed to fetch an access token (ADC and gcloud): ${stderr || err.message}`));
+          return;
+        }
+        const t = (stdout || "").trim();
+        if (!t) {
+          reject(new Error("Failed to fetch an access token: empty token from gcloud"));
+          return;
+        }
+        resolve(t);
+      });
+    });
+  }
+}
+
+function fileToBase64(path: string): { data: string; mimeType: string } {
+  if (!fs.existsSync(path)) {
+    throw new Error(`File not found: ${path}`);
   }
-  const buf = fs.readFileSync(filePath);
+  const buf = fs.readFileSync(path);
   const data = Buffer.from(buf).toString("base64");
-  // Detect mime type from extension
-  const ext = path.extname(filePath).toLowerCase();
-  const mimeType = ext === '.jpg' || ext === '.jpeg' ? 'image/jpeg' : 
-                   ext === '.png' ? 'image/png' :
-                   ext === '.webp' ? 'image/webp' : 'image/png';
+  // Default to PNG if not sure, similar to existing code
+  const mimeType = "image/png";
   return { data, mimeType };
 }
 
-export const imageToVideo = {
-  name: "imageToVideo",
-  description: "Generate videos from an image as starting first frame using Gemini Veo models via HTTP API with Gemini API key.",
+export const imageToVideo = ({
+  name: "image_to_video",
+  description: "Generate videos from an image as starting first frame using Vertex Veo models (predictLongRunning + fetchPredictOperation).",
   parameters: z.object({
     prompt: z.string().describe("Text description for the video"),
     image_path: z.string().optional().describe("Path to source image for image-to-video generation"),
     last_frame_path: z.string().optional().describe("Path to last frame image to guide ending frame (optional)"),
     aspect_ratio: z.string().optional().describe("Video aspect ratio: '16:9' or '9:16' (default: '9:16')"),
-    duration_seconds: z.union([z.string(), z.number()]).optional().describe("Video duration in seconds: 4, 6, or 8 (default: 6)"),
+    duration_seconds: z.string().optional().describe("Video duration in seconds: '4', '6', or '8' (default: '6')"),
     resolution: z.string().optional().describe("Video resolution: '720p' or '1080p' (default: '720p')"),
     negative_prompt: z.string().optional().describe("Text describing what not to include in the video"),
     person_generation: z.string().optional().describe("Controls generation of people: 'allow_adult' (default for image-to-video) or 'allow_all'"),
     reference_images: z.array(z.string()).optional().describe("Additional image paths for reference (max 3)"),
     output_path: z.string().optional().describe("Output MP4 file path (if multiple predictions, index suffix is added)"),
-    gemini_api_key: z.string().optional().describe("Gemini API key (uses GEMINI_API_KEY env var if not provided)"),
-    model_id: z.string().optional().describe("Model ID (default: veo-3.1-generate-preview)"),
+    project_id: z.string().optional().describe("GCP Project ID (default: mixio-pro)"),
+    location_id: z.string().optional().describe("Vertex region (default: us-central1)"),
+    model_id: z.string().optional().describe("Model ID (default: veo-3.1-fast-generate-preview)"),
+    generate_audio: z.boolean().optional().describe("Boolean flag to enable generation of audio along with the video").default(false)
   }),
-  execute: async (args: {
-    prompt: string;
-    image_path?: string;
-    last_frame_path?: string;
-    aspect_ratio?: string;
-    duration_seconds?: string | number;
-    resolution?: string;
-    negative_prompt?: string;
-    person_generation?: string;
-    reference_images?: string[];
-    output_path?: string;
-    gemini_api_key?: string;
-    model_id?: string;
-  }) => {
-    const apiKey = args.gemini_api_key || process.env.GEMINI_API_KEY;
-    if (!apiKey) {
-      throw new Error("Gemini API key is required. Set GEMINI_API_KEY environment variable or pass gemini_api_key parameter. Get one at https://aistudio.google.com/app/apikey");
-    }
+  async execute(args) {
+    const projectId = args.project_id || "mixio-pro";
+    const location = args.location_id || "us-central1";
+    const modelId = args.model_id || "veo-3.1-fast-generate-preview";
 
-    const modelId = args.model_id || "veo-3.1-generate-preview";
-    const baseUrl = "https://generativelanguage.googleapis.com/v1beta";
+    const token = await fetchAccessToken();
 
-    // Convert duration_seconds to number, handling both string and number inputs
-    const durationSeconds = args.duration_seconds 
-      ? (typeof args.duration_seconds === 'string' ? parseInt(args.duration_seconds) : args.duration_seconds)
-      : 6; // default
-
-    try {
-      // Build the request body for predictLongRunning
-      const instances: any[] = [
-        {
-          prompt: args.prompt,
-        },
-      ];
+    const url = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${modelId}:predictLongRunning`;
 
-      // Add image if provided (first frame)
-      if (args.image_path) {
-        const { data, mimeType } = fileToBase64(args.image_path);
-        instances[0].image = {
+    let imagePart: any = undefined;
+    if (args.image_path) {
+      const { data, mimeType } = fileToBase64(args.image_path);
+      imagePart = {
+        image: {
           bytesBase64Encoded: data,
           mimeType,
-        };
-      }
+        },
+      };
+    }
 
-      // Add last frame if provided (for interpolation)
-      if (args.last_frame_path) {
-        const { data, mimeType } = fileToBase64(args.last_frame_path);
-        instances[0].lastFrame = {
+    let lastFramePart: any = undefined;
+    if (args.last_frame_path) {
+      const { data, mimeType } = fileToBase64(args.last_frame_path);
+      lastFramePart = {
+        lastFrame: {
           bytesBase64Encoded: data,
           mimeType,
-        };
+        },
+      };
+    }
+
+    let referenceImages: any[] | undefined = undefined;
+    if (args.reference_images) {
+      let refImages: string[];
+      if (typeof args.reference_images === "string") {
+        if (args.reference_images.startsWith("[") && args.reference_images.endsWith("]")) {
+          try {
+            refImages = JSON.parse(args.reference_images);
+          } catch {
+            throw new Error("Invalid reference_images format");
+          }
+        } else {
+          refImages = [args.reference_images];
+        }
+      } else if (Array.isArray(args.reference_images)) {
+        refImages = args.reference_images;
+      } else {
+        throw new Error("Invalid reference_images: must be array or string");
       }
 
-      // Add reference images if provided
-      if (args.reference_images && args.reference_images.length > 0) {
-        const refImages = args.reference_images.slice(0, 3).map((imgPath) => {
-          const { data, mimeType } = fileToBase64(imgPath);
+      if (refImages.length > 0) {
+        referenceImages = refImages.slice(0, 3).map((p) => {
+          const { data, mimeType } = fileToBase64(p);
           return {
             image: {
               bytesBase64Encoded: data,
@@ -102,162 +128,105 @@ export const imageToVideo = {
             referenceType: "asset",
           };
         });
-        instances[0].referenceImages = refImages;
-      }
-
-      // Build parameters - NOTE: Parameters go in "parameters" object, not in instances
-      const parameters: any = {};
-
-      if (args.aspect_ratio) {
-        parameters.aspectRatio = args.aspect_ratio;
-      }
-      
-      if (durationSeconds) {
-        parameters.durationSeconds = durationSeconds;
-      }
-      
-      if (args.resolution) {
-        parameters.resolution = args.resolution;
-      }
-
-      if (args.negative_prompt) {
-        parameters.negativePrompt = args.negative_prompt;
-      }
-
-      if (args.person_generation) {
-        parameters.personGeneration = args.person_generation;
       }
+    }
 
-      const requestBody: any = {
-        instances,
-      };
+    const personGeneration = args.person_generation || (args.image_path ? "allow_adult" : "allow_all");
+
+    const instances: any[] = [
+      {
+        prompt: args.prompt,
+        ...(imagePart || {}),
+        ...(lastFramePart || {}),
+        ...(referenceImages ? { referenceImages } : {}),
+      },
+    ];
+
+    const parameters: any = {
+      aspectRatio: args.aspect_ratio || "9:16",
+      durationSeconds: parseInt(args.duration_seconds) || 6,
+      resolution: args.resolution || "720p",
+      negativePrompt: args.negative_prompt,
+      generateAudio: args.generate_audio || false,
+      personGeneration,
+    };
+
+    const res = await fetch(url, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ instances, parameters }),
+    });
+
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`Vertex request failed: ${res.status} ${text}`);
+    }
 
-      // Only add parameters if we have any
-      if (Object.keys(parameters).length > 0) {
-        requestBody.parameters = parameters;
-      }
+    const op = await res.json();
+    const name: string = op.name || op.operation || "";
+    if (!name) {
+      throw new Error("Vertex did not return an operation name for long-running request");
+    }
 
-      console.log(`Starting video generation with model: ${modelId}`);
+    let current = op;
+    let done = !!op.done;
+    let tries = 0;
 
-      // Step 1: Start the long-running operation
-      const url = `${baseUrl}/models/${modelId}:predictLongRunning`;
-      const response = await fetch(url, {
+    // Poll using fetchPredictOperation as per Vertex recommendation
+    const fetchUrl = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${modelId}:fetchPredictOperation`;
+    while (!done && tries < 60) {
+      await wait(10000);
+      const poll = await fetch(fetchUrl, {
         method: "POST",
         headers: {
-          "x-goog-api-key": apiKey,
+          Authorization: `Bearer ${token}`,
           "Content-Type": "application/json",
         },
-        body: JSON.stringify(requestBody),
+        body: JSON.stringify({ operationName: name }),
       });
-
-      if (!response.ok) {
-        const errorText = await response.text();
-        throw new Error(`Video generation request failed: ${response.status} ${errorText}`);
-      }
-
-      const operation = await response.json() as any;
-      const operationName: string = operation.name || operation.operation || "";
-      
-      if (!operationName) {
-        throw new Error("No operation name returned from API");
-      }
-
-      console.log(`Operation started: ${operationName}`);
-
-      // Step 2: Poll the operation status by getting the operation directly
-      let currentOp: any = operation;
-      let done = !!operation.done;
-      let tries = 0;
-      const maxTries = 60; // 10 minutes with 10s intervals
-
-      while (!done && tries < maxTries) {
-        await wait(10000); // Wait 10 seconds
-        tries++;
-        console.log(`Polling attempt ${tries}/${maxTries}...`);
-
-        // Poll by getting the operation status directly
-        const pollResponse = await fetch(`${baseUrl}/${operationName}`, {
-          method: "GET",
-          headers: {
-            "x-goog-api-key": apiKey,
-          },
-        });
-
-        if (!pollResponse.ok) {
-          const errorText = await pollResponse.text();
-          throw new Error(`Operation polling failed: ${pollResponse.status} ${errorText}`);
-        }
-
-        currentOp = await pollResponse.json() as any;
-        done = !!currentOp.done || !!currentOp.response;
-      }
-
-      if (!done) {
-        throw new Error("Video generation timed out after 10 minutes");
-      }
-
-      console.log("Operation completed!");
-
-      // Step 3: Extract and download videos
-      const resp = currentOp.response || currentOp;
-      
-      // The response structure is: response.generateVideoResponse.generatedSamples[].video.uri
-      const generateVideoResponse = resp?.generateVideoResponse;
-      const generatedSamples = generateVideoResponse?.generatedSamples || [];
-      
-      if (!generatedSamples || generatedSamples.length === 0) {
-        let jsonStr = "";
-        try { jsonStr = JSON.stringify(resp, null, 2); } catch {}
-        return `Video generation completed but no generatedSamples found.\n\nFull Response:\n${jsonStr.slice(0, 1000)}${jsonStr.length > 1000 ? '\n...(truncated)' : ''}`;
+      if (!poll.ok) {
+        const text = await poll.text();
+        throw new Error(`Vertex operation poll failed: ${poll.status} ${text}`);
       }
+      current = await poll.json();
+      done = !!current.done || !!current.response;
+      tries++;
+    }
 
-      const outputs: string[] = [];
-
-      // Download videos from URIs
-      for (let i = 0; i < generatedSamples.length; i++) {
-        const sample = generatedSamples[i];
-        const videoUri = sample?.video?.uri;
-        
-        if (!videoUri) {
-          console.warn(`Sample ${i} has no video URI`);
-          continue;
-        }
-
-        console.log(`Downloading video ${i + 1}/${generatedSamples.length} from ${videoUri}...`);
-        
-        // Download video from URI with API key
-        const videoResponse = await fetch(videoUri, {
-          method: "GET",
-          headers: {
-            "x-goog-api-key": apiKey,
-          },
-        });
-
-        if (!videoResponse.ok) {
-          throw new Error(`Failed to download video: ${videoResponse.status} ${videoResponse.statusText}`);
+    const resp = current.response || current;
+    // Decode from response.videos[].bytesBase64Encoded only
+    const outputs: string[] = [];
+    const saveVideo = (base64: string, index: number) => {
+      if (!base64) return;
+      const filePath = args.output_path
+        ? (index === 0 ? args.output_path : args.output_path.replace(/\.mp4$/i, `_${index}.mp4`))
+        : `video_output_${Date.now()}${index === 0 ? '' : '_' + index}.mp4`;
+      const absPath = path.resolve(filePath);
+      const buf = Buffer.from(base64, 'base64');
+      fs.writeFileSync(absPath, buf);
+      outputs.push(absPath);
+    };
+
+    if (Array.isArray(resp?.videos) && resp.videos.length > 0) {
+      for (let i = 0; i < resp.videos.length; i++) {
+        const v = resp.videos[i] || {};
+        if (typeof v.bytesBase64Encoded === 'string') {
+          saveVideo(v.bytesBase64Encoded, i);
         }
-
-        const videoBuffer = await videoResponse.arrayBuffer();
-        
-        // Save video to file
-        const filePath = args.output_path
-          ? (i === 0 ? args.output_path : args.output_path.replace(/\.mp4$/i, `_${i}.mp4`))
-          : `video_output_${Date.now()}${i === 0 ? '' : '_' + i}.mp4`;
-        const absPath = path.resolve(filePath);
-        
-        fs.writeFileSync(absPath, Buffer.from(videoBuffer));
-        outputs.push(absPath);
-        console.log(`Saved video to: ${absPath}`);
-      }
-
-      if (outputs.length > 0) {
-        return `Video(s) saved successfully:\n${outputs.map((p, i) => `${i + 1}. ${p}`).join('\n')}`;
       }
-
-      return "Video generation completed but no videos were saved.";
-      
-    } catch (error: any) {
-      throw new Error(`Video generation failed: ${error.message || JSON.stringify(error)}`);
     }
+    if (outputs.length > 0) {
+      return `Video(s) saved: ${outputs.join(', ')}`;
+    }
+
+    // If nothing saved, return a concise summary plus head/tail snippets of JSON
+    let jsonStr = "";
+    try { jsonStr = JSON.stringify(resp); } catch {}
+    const head150 = jsonStr ? jsonStr.slice(0, 150) : "";
+    const tail50 = jsonStr ? jsonStr.slice(Math.max(0, jsonStr.length - 50)) : "";
+    return `Vertex operation done but no videos array present. operationName=${name}. json_head150=${head150} json_tail50=${tail50}`;
   },
-};
+});