opencode-nanobanana 0.1.0

package/src/index.ts

@@ -0,0 +1,23 @@
+/**
+ * OpenCode Visual Toolkit
+ * 
+ * A comprehensive plugin providing visual capabilities:
+ * - Image generation (Nano Banana)
+ * - Visual analysis
+ * - Design-to-code
+ * - App asset pipelines (iOS, Android, macOS, Web)
+ */
+
+// Re-export providers
+export * from './providers/gemini.js';
+
+// Re-export utilities
+export * from './utils/file-handler.js';
+export * from './utils/image-processing.js';
+export * from './utils/ffmpeg.js';
+
+// Re-export tools
+export * from './tools/video/generate-video.js';
+export * from './tools/video/image-to-video.js';
+export * from './tools/video/storyboard-video.js';
+export * from './tools/video/extend-video.js';

package/src/providers/gemini.ts

@@ -0,0 +1,288 @@
+import { GoogleGenAI } from '@google/genai';
+
+export interface VideoGenerationOptions {
+  aspectRatio?: '16:9' | '9:16';
+  resolution?: '720p' | '1080p';
+  duration?: 4 | 6 | 8;
+  numberOfVideos?: number;
+  negativePrompt?: string;
+}
+
+export interface ImageAnimationOptions {
+  aspectRatio?: '16:9' | '9:16';
+  resolution?: '720p' | '1080p';
+  duration?: 4 | 6 | 8;
+}
+
+export interface VideoExtensionOptions {
+  aspectRatio?: '16:9' | '9:16';
+  resolution?: '720p' | '1080p';
+}
+
+export interface ReferenceImage {
+  buffer: Buffer;
+  description: string;
+}
+
+export interface VideoGenerationResult {
+  buffer: Buffer;
+  url?: string;
+  generationTime: number;
+}
+
+export class GeminiProvider {
+  private ai: GoogleGenAI;
+  private apiKey: string;
+  private readonly VEO_MODEL = 'veo-3.0-generate-001';
+  private readonly VEO_PREVIEW_MODEL = 'veo-3.1-generate-preview';
+  private readonly POLL_INTERVAL_MS = 10000;
+
+  constructor(apiKey: string) {
+    this.apiKey = apiKey;
+    this.ai = new GoogleGenAI({ apiKey });
+  }
+
+  async generateVideo(
+    prompt: string,
+    options: VideoGenerationOptions = {}
+  ): Promise<VideoGenerationResult> {
+    const startTime = Date.now();
+
+    const {
+      aspectRatio = '16:9',
+      resolution = '720p',
+      duration = 8,
+      numberOfVideos = 1,
+      negativePrompt,
+    } = options;
+
+    let operation = await this.ai.models.generateVideos({
+      model: this.VEO_MODEL,
+      prompt,
+      config: {
+        numberOfVideos,
+        aspectRatio,
+        resolution,
+        durationSeconds: duration,
+        ...(negativePrompt && { negativePrompt }),
+      },
+    });
+
+    operation = await this.pollOperation(operation);
+
+    if (!operation.response?.generatedVideos?.[0]) {
+      throw new Error('No video was generated');
+    }
+
+    const video = operation.response.generatedVideos[0];
+    const videoUrl = video.video?.uri;
+
+    if (!videoUrl) {
+      throw new Error('Video URL not available');
+    }
+
+    const buffer = await this.downloadVideo(videoUrl);
+    const generationTime = Date.now() - startTime;
+
+    return { buffer, url: videoUrl, generationTime };
+  }
+
+  async animateImage(
+    imageBuffer: Buffer,
+    prompt: string,
+    options: ImageAnimationOptions = {}
+  ): Promise<VideoGenerationResult> {
+    const startTime = Date.now();
+
+    const {
+      aspectRatio = '16:9',
+      resolution = '720p',
+      duration = 8,
+    } = options;
+
+    const imageBase64 = imageBuffer.toString('base64');
+    const mimeType = this.detectImageMimeType(imageBuffer);
+
+    let operation = await this.ai.models.generateVideos({
+      model: this.VEO_MODEL,
+      prompt,
+      image: {
+        imageBytes: imageBase64,
+        mimeType,
+      },
+      config: {
+        numberOfVideos: 1,
+        aspectRatio,
+        resolution,
+        durationSeconds: duration,
+      },
+    });
+
+    operation = await this.pollOperation(operation);
+
+    if (!operation.response?.generatedVideos?.[0]) {
+      throw new Error('No video was generated from image');
+    }
+
+    const video = operation.response.generatedVideos[0];
+    const videoUrl = video.video?.uri;
+
+    if (!videoUrl) {
+      throw new Error('Video URL not available');
+    }
+
+    const buffer = await this.downloadVideo(videoUrl);
+    const generationTime = Date.now() - startTime;
+
+    return { buffer, url: videoUrl, generationTime };
+  }
+
+  async extendVideo(
+    videoBuffer: Buffer,
+    prompt: string,
+    options: VideoExtensionOptions = {}
+  ): Promise<VideoGenerationResult> {
+    const startTime = Date.now();
+
+    const {
+      aspectRatio = '16:9',
+      resolution = '720p',
+    } = options;
+
+    const videoBase64 = videoBuffer.toString('base64');
+
+    let operation = await this.ai.models.generateVideos({
+      model: this.VEO_MODEL,
+      prompt,
+      video: {
+        videoBytes: videoBase64,
+        mimeType: 'video/mp4',
+      },
+      config: {
+        numberOfVideos: 1,
+        aspectRatio,
+        resolution,
+      },
+    });
+
+    operation = await this.pollOperation(operation);
+
+    if (!operation.response?.generatedVideos?.[0]) {
+      throw new Error('Video extension failed');
+    }
+
+    const video = operation.response.generatedVideos[0];
+    const videoUrl = video.video?.uri;
+
+    if (!videoUrl) {
+      throw new Error('Video URL not available');
+    }
+
+    const buffer = await this.downloadVideo(videoUrl);
+    const generationTime = Date.now() - startTime;
+
+    return { buffer, url: videoUrl, generationTime };
+  }
+
+  async generateVideoWithReferences(
+    prompt: string,
+    referenceImages: ReferenceImage[],
+    options: VideoGenerationOptions = {}
+  ): Promise<VideoGenerationResult> {
+    if (referenceImages.length === 0 || referenceImages.length > 3) {
+      throw new Error('Reference images must be between 1 and 3');
+    }
+
+    const startTime = Date.now();
+
+    const {
+      aspectRatio = '16:9',
+      resolution = '720p',
+      duration = 8,
+      numberOfVideos = 1,
+    } = options;
+
+    const references = referenceImages.map((ref) => ({
+      image: {
+        imageBytes: ref.buffer.toString('base64'),
+        mimeType: this.detectImageMimeType(ref.buffer),
+      },
+    }));
+
+    let operation = await this.ai.models.generateVideos({
+      model: this.VEO_PREVIEW_MODEL,
+      prompt,
+      config: {
+        numberOfVideos,
+        aspectRatio,
+        resolution,
+        durationSeconds: duration,
+        referenceImages: references,
+      },
+    });
+
+    operation = await this.pollOperation(operation);
+
+    if (!operation.response?.generatedVideos?.[0]) {
+      console.error('Operation response:', JSON.stringify(operation, null, 2));
+      throw new Error('No video was generated with references');
+    }
+
+    const video = operation.response.generatedVideos[0];
+    const videoUrl = video.video?.uri;
+
+    if (!videoUrl) {
+      throw new Error('Video URL not available');
+    }
+
+    const buffer = await this.downloadVideo(videoUrl);
+    const generationTime = Date.now() - startTime;
+
+    return { buffer, url: videoUrl, generationTime };
+  }
+
+  private async pollOperation(operation: any): Promise<any> {
+    let currentOperation = operation;
+
+    while (!currentOperation.done) {
+      await new Promise((resolve) => setTimeout(resolve, this.POLL_INTERVAL_MS));
+      currentOperation = await this.ai.operations.getVideosOperation({
+        operation: currentOperation,
+      });
+    }
+
+    return currentOperation;
+  }
+
+  private async downloadVideo(url: string): Promise<Buffer> {
+    const response = await fetch(url, {
+      headers: {
+        'x-goog-api-key': this.apiKey,
+      },
+      redirect: 'follow',
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to download video: ${response.statusText}`);
+    }
+
+    const arrayBuffer = await response.arrayBuffer();
+    return Buffer.from(arrayBuffer);
+  }
+
+  private detectImageMimeType(buffer: Buffer): string {
+    if (buffer[0] === 0xff && buffer[1] === 0xd8) {
+      return 'image/jpeg';
+    }
+    if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
+      return 'image/png';
+    }
+    if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) {
+      return 'image/gif';
+    }
+    if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46) {
+      return 'image/webp';
+    }
+    return 'image/png';
+  }
+}

package/src/tools/video/extend-video.ts

@@ -0,0 +1,71 @@
+import { GeminiProvider } from '../../providers/gemini.js';
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+export interface ExtendVideoOptions {
+  videoPath: string;
+  prompt: string;
+  aspectRatio?: '16:9' | '9:16';
+  resolution?: '720p' | '1080p';
+  outputPath?: string;
+  apiKey: string;
+}
+
+export interface ExtendVideoResult {
+  videoPath: string;
+  generationTime: number;
+}
+
+export async function extendVideo(
+  options: ExtendVideoOptions
+): Promise<ExtendVideoResult> {
+  const startTime = Date.now();
+
+  const {
+    videoPath,
+    prompt,
+    aspectRatio = '16:9',
+    resolution = '720p',
+    outputPath,
+    apiKey,
+  } = options;
+
+  console.log('🎬 Extending video...');
+  console.log(`   Video: ${videoPath}`);
+  console.log(`   Prompt: ${prompt}`);
+  console.log(`   Resolution: ${resolution}`);
+  console.log(`   Aspect Ratio: ${aspectRatio}`);
+
+  const provider = new GeminiProvider(apiKey);
+
+  try {
+    console.log('   Loading video...');
+    const videoBuffer = await readFile(videoPath);
+
+    console.log('   Generating extension...');
+    const result = await provider.extendVideo(videoBuffer, prompt, {
+      aspectRatio,
+      resolution,
+    });
+
+    const finalOutputPath = outputPath || join(tmpdir(), `extended-${Date.now()}.mp4`);
+
+    console.log('   Saving extended video...');
+    await writeFile(finalOutputPath, result.buffer);
+
+    const totalTime = Date.now() - startTime;
+
+    console.log(`✨ Video extension complete!`);
+    console.log(`   Output: ${finalOutputPath}`);
+    console.log(`   Generation time: ${(totalTime / 1000).toFixed(2)}s`);
+
+    return {
+      videoPath: finalOutputPath,
+      generationTime: totalTime,
+    };
+  } catch (error) {
+    console.error('❌ Video extension failed:', error);
+    throw error;
+  }
+}

package/src/tools/video/generate-video.ts

@@ -0,0 +1,70 @@
+import { GeminiProvider } from '../../providers/gemini.js';
+import { writeFile } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+export interface GenerateVideoOptions {
+  prompt: string;
+  aspectRatio?: '16:9' | '9:16';
+  resolution?: '720p' | '1080p';
+  duration?: 4 | 6 | 8;
+  outputPath?: string;
+  apiKey: string;
+}
+
+export interface GenerateVideoResult {
+  videoPath: string;
+  generationTime: number;
+}
+
+export async function generateVideo(
+  options: GenerateVideoOptions
+): Promise<GenerateVideoResult> {
+  const startTime = Date.now();
+
+  const {
+    prompt,
+    aspectRatio = '16:9',
+    resolution = '720p',
+    duration = 8,
+    outputPath,
+    apiKey,
+  } = options;
+
+  console.log('🎬 Generating video...');
+  console.log(`   Prompt: ${prompt}`);
+  console.log(`   Resolution: ${resolution}`);
+  console.log(`   Duration: ${duration}s`);
+  console.log(`   Aspect Ratio: ${aspectRatio}`);
+  console.log(`   Audio: native (Veo 3.0)`);
+
+  const provider = new GeminiProvider(apiKey);
+
+  try {
+    console.log('   Generating with Veo 3.0...');
+    const result = await provider.generateVideo(prompt, {
+      aspectRatio,
+      resolution,
+      duration,
+    });
+
+    const finalOutputPath = outputPath || join(tmpdir(), `video-${Date.now()}.mp4`);
+
+    console.log('   Saving video...');
+    await writeFile(finalOutputPath, result.buffer);
+
+    const totalTime = Date.now() - startTime;
+
+    console.log(`✨ Video generation complete!`);
+    console.log(`   Output: ${finalOutputPath}`);
+    console.log(`   Generation time: ${(totalTime / 1000).toFixed(2)}s`);
+
+    return {
+      videoPath: finalOutputPath,
+      generationTime: totalTime,
+    };
+  } catch (error) {
+    console.error('❌ Video generation failed:', error);
+    throw error;
+  }
+}

package/src/tools/video/image-to-video.ts

@@ -0,0 +1,76 @@
+import { GeminiProvider } from '../../providers/gemini.js';
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+export interface ImageToVideoOptions {
+  imagePath: string;
+  prompt: string;
+  aspectRatio?: '16:9' | '9:16';
+  resolution?: '720p' | '1080p';
+  duration?: 4 | 6 | 8;
+  outputPath?: string;
+  apiKey: string;
+}
+
+export interface ImageToVideoResult {
+  videoPath: string;
+  generationTime: number;
+}
+
+export async function imageToVideo(
+  options: ImageToVideoOptions
+): Promise<ImageToVideoResult> {
+  const startTime = Date.now();
+
+  const {
+    imagePath,
+    prompt,
+    aspectRatio = '16:9',
+    resolution = '720p',
+    duration = 8,
+    outputPath,
+    apiKey,
+  } = options;
+
+  console.log('🎬 Animating image to video...');
+  console.log(`   Image: ${imagePath}`);
+  console.log(`   Prompt: ${prompt}`);
+  console.log(`   Resolution: ${resolution}`);
+  console.log(`   Duration: ${duration}s`);
+  console.log(`   Aspect Ratio: ${aspectRatio}`);
+  console.log(`   Audio: native (Veo 3.0)`);
+
+  const provider = new GeminiProvider(apiKey);
+
+  try {
+    console.log('   Loading image...');
+    const imageBuffer = await readFile(imagePath);
+
+    console.log('   Animating with Veo 3.0...');
+    const result = await provider.animateImage(imageBuffer, prompt, {
+      aspectRatio,
+      resolution,
+      duration,
+    });
+
+    const finalOutputPath = outputPath || join(tmpdir(), `animated-${Date.now()}.mp4`);
+
+    console.log('   Saving video...');
+    await writeFile(finalOutputPath, result.buffer);
+
+    const totalTime = Date.now() - startTime;
+
+    console.log(`✨ Image animation complete!`);
+    console.log(`   Output: ${finalOutputPath}`);
+    console.log(`   Generation time: ${(totalTime / 1000).toFixed(2)}s`);
+
+    return {
+      videoPath: finalOutputPath,
+      generationTime: totalTime,
+    };
+  } catch (error) {
+    console.error('❌ Image animation failed:', error);
+    throw error;
+  }
+}