opencode-nanobanana 0.1.0

package/src/tools/video/storyboard-video.ts

@@ -0,0 +1,325 @@
+/**
+ * Storyboard Video Generation Tool
+ *
+ * Generates a multi-scene video by creating individual scenes in parallel
+ * and stitching them together with transitions using FFmpeg.
+ */
+
+import { GeminiProvider, type ReferenceImage } from '../../providers/gemini.js';
+import {
+  checkFfmpegInstalled,
+  concatenateVideos,
+  addAudioTrack,
+  type ConcatenateOptions,
+} from '../../utils/ffmpeg.js';
+import { writeFile, unlink, readFile } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+export interface StoryboardVideoOptions {
+  scenes: string[];
+  style?: string;
+  characterDescription?: string;
+  referenceImages?: string[];
+  aspectRatio?: '16:9' | '9:16';
+  transition?: 'cut' | 'crossfade' | 'fade';
+  transitionDuration?: number;
+  backgroundMusic?: string;
+  musicVolume?: number;
+  outputPath?: string;
+  apiKey: string;
+}
+
+export interface StoryboardVideoResult {
+  /** Path to the final stitched video */
+  videoPath: string;
+  /** Total generation time in milliseconds */
+  totalTime: number;
+  /** Per-scene generation times */
+  sceneTimes: Array<{ scene: number; time: number }>;
+  /** Number of scenes successfully generated */
+  successCount: number;
+  /** Number of scenes that failed */
+  failureCount: number;
+}
+
+/**
+ * Generate a multi-scene storyboard video
+ *
+ * This function generates multiple video scenes in parallel and stitches them
+ * together with transitions to create a cohesive video narrative.
+ *
+ * @param options - Configuration options for the storyboard video
+ * @returns Result object with video path and timing information
+ *
+ * @example
+ * Basic usage:
+ * ```typescript
+ * const result = await generateStoryboardVideo({
+ *   apiKey: 'your-api-key',
+ *   scenes: [
+ *     'A serene mountain landscape at sunrise',
+ *     'A hiker reaching the summit',
+ *     'Panoramic view from the peak'
+ *   ],
+ *   style: 'cinematic',
+ *   transition: 'crossfade',
+ *   outputPath: './mountain-journey.mp4'
+ * });
+ * ```
+ *
+ * @example
+ * With character consistency:
+ * ```typescript
+ * const result = await generateStoryboardVideo({
+ *   apiKey: 'your-api-key',
+ *   characterDescription: 'A young woman with long brown hair wearing a red jacket',
+ *   referenceImages: ['./character-ref.jpg'],
+ *   scenes: [
+ *     'Walking through a forest',
+ *     'Discovering a hidden waterfall',
+ *     'Setting up camp at sunset'
+ *   ],
+ *   style: 'cinematic',
+ *   transition: 'crossfade'
+ * });
+ * ```
+ *
+ * @example
+ * With background music:
+ * ```typescript
+ * const result = await generateStoryboardVideo({
+ *   apiKey: 'your-api-key',
+ *   scenes: ['Scene 1', 'Scene 2', 'Scene 3'],
+ *   generateAudio: true,           // Native Veo audio (default)
+ *   backgroundMusic: './music.mp3', // Add background music
+ *   musicVolume: 0.3,               // 30% volume for background music
+ *   transition: 'crossfade'
+ * });
+ * ```
+ */
+export async function generateStoryboardVideo(
+  options: StoryboardVideoOptions
+): Promise<StoryboardVideoResult> {
+  const startTime = Date.now();
+
+  // Validate inputs
+  if (!options.scenes || options.scenes.length === 0) {
+    throw new Error('At least one scene is required');
+  }
+
+  // Check FFmpeg availability
+  const ffmpegAvailable = await checkFfmpegInstalled();
+  if (!ffmpegAvailable) {
+    throw new Error(
+      'FFmpeg is not installed or not available in PATH. Please install FFmpeg to use this tool.'
+    );
+  }
+
+  // Initialize provider
+  const provider = new GeminiProvider(options.apiKey);
+
+  const {
+    scenes,
+    style,
+    characterDescription,
+    referenceImages,
+    aspectRatio = '16:9',
+    transition = 'crossfade',
+    transitionDuration = 0.5,
+    backgroundMusic,
+    musicVolume = 0.3,
+    outputPath = join(tmpdir(), `storyboard-${Date.now()}.mp4`),
+  } = options;
+
+  // Load reference images if provided
+  let loadedReferences: ReferenceImage[] | undefined;
+  if (referenceImages && referenceImages.length > 0) {
+    if (referenceImages.length > 3) {
+      throw new Error('Maximum of 3 reference images allowed');
+    }
+
+    console.log(`📸 Loading ${referenceImages.length} reference image(s)...`);
+    loadedReferences = await Promise.all(
+      referenceImages.map(async (imagePath, index) => {
+        const buffer = await readFile(imagePath);
+        return {
+          buffer,
+          description: `Reference image ${index + 1} for character/scene consistency`,
+        };
+      })
+    );
+  }
+
+  console.log(`\n🎬 Starting storyboard generation with ${scenes.length} scene(s)`);
+  console.log(`⚙️  Configuration:`);
+  console.log(`   - Aspect ratio: ${aspectRatio}`);
+  console.log(`   - Transition: ${transition} (${transitionDuration}s)`);
+  console.log(`   - Audio: native (Veo 3.0)${backgroundMusic ? ' + background music' : ''}`);
+
+  if (characterDescription) {
+    console.log(`👤 Using character description: "${characterDescription}"`);
+  }
+  if (loadedReferences) {
+    console.log(`🖼️  Using ${loadedReferences.length} reference image(s) for consistency`);
+  }
+
+  console.log(`\n📹 Generating scenes sequentially (to avoid rate limits)...`);
+
+  type SceneResult = {
+    index: number;
+    path: string | null;
+    time: number;
+    success: boolean;
+    error?: string;
+  };
+  
+  const sceneResults: SceneResult[] = [];
+  
+  for (let index = 0; index < scenes.length; index++) {
+    const sceneDescription = scenes[index]!;
+    const sceneStartTime = Date.now();
+    console.log(`   [${index + 1}/${scenes.length}] Generating: "${sceneDescription.slice(0, 50)}${sceneDescription.length > 50 ? '...' : ''}"`);
+
+    try {
+      let prompt = sceneDescription;
+
+      if (characterDescription) {
+        prompt = `${characterDescription}. ${prompt}`;
+      }
+
+      if (style) {
+        prompt = `${style} style: ${prompt}`;
+      }
+
+      let result;
+      if (loadedReferences && loadedReferences.length > 0) {
+        result = await provider.generateVideoWithReferences(
+          prompt,
+          loadedReferences,
+          {
+            aspectRatio,
+            resolution: '720p',
+            duration: 8,
+            numberOfVideos: 1,
+          }
+        );
+      } else {
+        result = await provider.generateVideo(prompt, {
+          aspectRatio,
+          resolution: '720p',
+          duration: 8,
+          numberOfVideos: 1,
+        });
+      }
+
+      const tempPath = join(tmpdir(), `scene-${index}-${Date.now()}.mp4`);
+      await writeFile(tempPath, result.buffer);
+
+      const sceneTime = Date.now() - sceneStartTime;
+      console.log(`   ✅ [${index + 1}/${scenes.length}] Completed in ${(sceneTime / 1000).toFixed(1)}s`);
+
+      sceneResults.push({
+        index,
+        path: tempPath,
+        time: sceneTime,
+        success: true,
+      });
+    } catch (error) {
+      const sceneTime = Date.now() - sceneStartTime;
+      console.error(`   ❌ [${index + 1}/${scenes.length}] Failed after ${(sceneTime / 1000).toFixed(1)}s:`, error);
+
+      sceneResults.push({
+        index,
+        path: null,
+        time: sceneTime,
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  // Filter successful scenes
+  const successfulScenes = sceneResults.filter(
+    (result): result is { index: number; path: string; time: number; success: true } =>
+      result.success && result.path !== null
+  );
+
+  if (successfulScenes.length === 0) {
+    throw new Error('All scenes failed to generate. No video to stitch.');
+  }
+
+  if (successfulScenes.length < sceneResults.length) {
+    console.warn(
+      `⚠️  ${sceneResults.length - successfulScenes.length} scene(s) failed. Continuing with ${successfulScenes.length} successful scene(s).`
+    );
+  }
+
+  // Sort by original index to maintain scene order
+  successfulScenes.sort((a, b) => a.index - b.index);
+  const videoPaths = successfulScenes.map((s) => s.path);
+
+  // Stitch videos together
+  console.log(`\n🎞️  Stitching ${videoPaths.length} scene(s) together...`);
+  console.log(`   - Transition: ${transition}`);
+  console.log(`   - Duration: ${transitionDuration}s`);
+
+  const concatenateOptions: ConcatenateOptions = {
+    transition,
+    transitionDuration,
+  };
+
+  // Determine final output path based on whether we need to add background music
+  const stitchedVideoPath = backgroundMusic
+    ? join(tmpdir(), `stitched-${Date.now()}.mp4`)
+    : outputPath;
+
+  try {
+    await concatenateVideos(videoPaths, stitchedVideoPath, concatenateOptions);
+
+    // Add background music if provided
+    if (backgroundMusic) {
+      console.log(`\n🎵 Adding background music...`);
+      console.log(`   - Volume: ${(musicVolume * 100).toFixed(0)}%`);
+      await addAudioTrack(stitchedVideoPath, backgroundMusic, outputPath, musicVolume);
+      console.log(`   ✅ Audio mixing complete`);
+      // Clean up temporary stitched video
+      await unlink(stitchedVideoPath).catch(() => {});
+    }
+  } finally {
+    // Clean up temporary scene files
+    for (const videoPath of videoPaths) {
+      try {
+        await unlink(videoPath);
+      } catch {
+        // Ignore cleanup errors
+      }
+    }
+  }
+
+  const totalTime = Date.now() - startTime;
+
+  // Calculate average scene generation time
+  const avgSceneTime = successfulScenes.length > 0
+    ? successfulScenes.reduce((sum, s) => sum + s.time, 0) / successfulScenes.length
+    : 0;
+
+  console.log(`\n✨ Storyboard generation complete!`);
+  console.log(`📊 Summary:`);
+  console.log(`   - Total time: ${(totalTime / 1000).toFixed(1)}s`);
+  console.log(`   - Scenes generated: ${successfulScenes.length}/${sceneResults.length}`);
+  console.log(`   - Average scene time: ${(avgSceneTime / 1000).toFixed(1)}s`);
+  console.log(`   - Success rate: ${((successfulScenes.length / sceneResults.length) * 100).toFixed(0)}%`);
+  console.log(`📁 Output: ${outputPath}\n`);
+
+  return {
+    videoPath: outputPath,
+    totalTime,
+    sceneTimes: sceneResults.map((r) => ({
+      scene: r.index + 1,
+      time: r.time,
+    })),
+    successCount: successfulScenes.length,
+    failureCount: sceneResults.length - successfulScenes.length,
+  };
+}

package/src/utils/ffmpeg.ts

@@ -0,0 +1,266 @@
+/**
+ * FFmpeg Utility Module
+ *
+ * Provides video processing operations using FFmpeg
+ */
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { existsSync } from 'fs';
+
+const execAsync = promisify(exec);
+
+export interface ConcatenateOptions {
+  transition?: 'cut' | 'crossfade' | 'fade';
+  transitionDuration?: number;
+}
+
+/**
+ * Check if FFmpeg is installed and available
+ */
+export async function checkFfmpegInstalled(): Promise<boolean> {
+  try {
+    await execAsync('ffmpeg -version');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Get the duration of a video in seconds
+ */
+export async function getVideoDuration(videoPath: string): Promise<number> {
+  if (!existsSync(videoPath)) {
+    throw new Error(`Video file not found: ${videoPath}`);
+  }
+
+  try {
+    const { stdout } = await execAsync(
+      `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "${videoPath}"`
+    );
+    const duration = parseFloat(stdout.trim());
+    if (isNaN(duration)) {
+      throw new Error(`Failed to parse video duration: ${stdout}`);
+    }
+    return duration;
+  } catch (error) {
+    throw new Error(`Failed to get video duration: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+/**
+ * Trim a video from startTime for the specified duration
+ */
+export async function trimVideo(
+  videoPath: string,
+  startTime: number,
+  duration: number,
+  outputPath: string
+): Promise<void> {
+  if (!existsSync(videoPath)) {
+    throw new Error(`Video file not found: ${videoPath}`);
+  }
+
+  if (startTime < 0 || duration <= 0) {
+    throw new Error(`Invalid trim parameters: startTime=${startTime}, duration=${duration}`);
+  }
+
+  try {
+    await execAsync(
+      `ffmpeg -i "${videoPath}" -ss ${startTime} -t ${duration} -c copy "${outputPath}" -y`
+    );
+  } catch (error) {
+    throw new Error(`Failed to trim video: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+/**
+ * Add an audio track to a video
+ */
+export async function addAudioTrack(
+  videoPath: string,
+  audioPath: string,
+  outputPath: string,
+  volume: number = 1.0
+): Promise<void> {
+  if (!existsSync(videoPath)) {
+    throw new Error(`Video file not found: ${videoPath}`);
+  }
+
+  if (!existsSync(audioPath)) {
+    throw new Error(`Audio file not found: ${audioPath}`);
+  }
+
+  if (volume < 0 || volume > 1) {
+    throw new Error(`Volume must be between 0 and 1, got: ${volume}`);
+  }
+
+  try {
+    // Mix video audio with background audio, adjust volume of background
+    await execAsync(
+      `ffmpeg -i "${videoPath}" -i "${audioPath}" -filter_complex "[1:a]volume=${volume}[a1];[0:a][a1]amix=inputs=2:duration=first[aout]" -map 0:v -map "[aout]" -c:v copy -c:a aac "${outputPath}" -y`
+    );
+  } catch (error) {
+    throw new Error(`Failed to add audio track: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+/**
+ * Concatenate multiple videos with transitions
+ */
+export async function concatenateVideos(
+  videoPaths: string[],
+  outputPath: string,
+  options: ConcatenateOptions = {}
+): Promise<void> {
+  if (videoPaths.length === 0) {
+    throw new Error('No videos provided for concatenation');
+  }
+
+  // Check all input files exist
+  for (const videoPath of videoPaths) {
+    if (!existsSync(videoPath)) {
+      throw new Error(`Video file not found: ${videoPath}`);
+    }
+  }
+
+  const { transition = 'crossfade', transitionDuration = 0.5 } = options;
+
+  try {
+    if (transition === 'cut') {
+      // Simple concatenation without transitions
+      await concatenateWithCut(videoPaths, outputPath);
+    } else if (transition === 'crossfade') {
+      // Crossfade transition between videos
+      await concatenateWithCrossfade(videoPaths, outputPath, transitionDuration);
+    } else if (transition === 'fade') {
+      // Fade to black transition between videos
+      await concatenateWithFade(videoPaths, outputPath, transitionDuration);
+    } else {
+      throw new Error(`Unknown transition type: ${transition}`);
+    }
+  } catch (error) {
+    throw new Error(`Failed to concatenate videos: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+/**
+ * Concatenate videos with simple cut (no transition)
+ */
+async function concatenateWithCut(videoPaths: string[], outputPath: string): Promise<void> {
+  // Create a temporary concat file
+  const concatListPath = outputPath + '.concat.txt';
+  const fs = await import('fs/promises');
+
+  const concatContent = videoPaths.map(path => `file '${path}'`).join('\n');
+  await fs.writeFile(concatListPath, concatContent);
+
+  try {
+    await execAsync(
+      `ffmpeg -f concat -safe 0 -i "${concatListPath}" -c copy "${outputPath}" -y`
+    );
+  } finally {
+    // Clean up concat file
+    await fs.unlink(concatListPath).catch(() => {});
+  }
+}
+
+async function concatenateWithCrossfade(
+  videoPaths: string[],
+  outputPath: string,
+  transitionDuration: number
+): Promise<void> {
+  if (videoPaths.length === 1) {
+    const fs = await import('fs/promises');
+    const firstVideo = videoPaths[0];
+    if (!firstVideo) {
+      throw new Error('No video path provided');
+    }
+    await fs.copyFile(firstVideo, outputPath);
+    return;
+  }
+
+  const durations: number[] = [];
+  for (const videoPath of videoPaths) {
+    const dur = await getVideoDuration(videoPath);
+    durations.push(dur);
+  }
+
+  let filterComplex = '';
+  let audioFilterComplex = '';
+  let currentVideoLabel = '[0:v]';
+  let currentAudioLabel = '[0:a]';
+  let cumulativeOffset = durations[0]! - transitionDuration;
+
+  for (let i = 1; i < videoPaths.length; i++) {
+    const isLast = i === videoPaths.length - 1;
+    const nextVideoLabel = isLast ? '[outv]' : `[v${i}]`;
+    const nextAudioLabel = isLast ? '[outa]' : `[a${i}]`;
+    
+    filterComplex += `${currentVideoLabel}[${i}:v]xfade=transition=fade:duration=${transitionDuration}:offset=${cumulativeOffset}${nextVideoLabel};`;
+    audioFilterComplex += `${currentAudioLabel}[${i}:a]acrossfade=d=${transitionDuration}${nextAudioLabel};`;
+    
+    currentVideoLabel = nextVideoLabel;
+    currentAudioLabel = nextAudioLabel;
+    
+    if (!isLast) {
+      cumulativeOffset += durations[i]! - transitionDuration;
+    }
+  }
+
+  const fullFilter = filterComplex + audioFilterComplex.slice(0, -1);
+  const inputs = videoPaths.map(path => `-i "${path}"`).join(' ');
+
+  await execAsync(
+    `ffmpeg ${inputs} -filter_complex "${fullFilter}" -map "[outv]" -map "[outa]" "${outputPath}" -y`
+  );
+}
+
+/**
+ * Concatenate videos with fade to black transition
+ */
+async function concatenateWithFade(
+  videoPaths: string[],
+  outputPath: string,
+  duration: number
+): Promise<void> {
+  if (videoPaths.length === 1) {
+    // No transition needed for single video
+    const fs = await import('fs/promises');
+    const firstVideo = videoPaths[0];
+    if (!firstVideo) {
+      throw new Error('No video path provided');
+    }
+    await fs.copyFile(firstVideo, outputPath);
+    return;
+  }
+
+  // For fade transition, we fade out each clip and fade in the next
+  // This is complex with FFmpeg, using a simpler concatenation approach
+  const inputs = videoPaths.map(path => `-i "${path}"`).join(' ');
+
+  // Build filter to fade out and fade in
+  let filterComplex = '';
+  for (let i = 0; i < videoPaths.length; i++) {
+    if (i < videoPaths.length - 1) {
+      // Fade out at the end of each video except the last
+      filterComplex += `[${i}:v]fade=t=out:st=0:d=${duration}[v${i}out];`;
+      // Fade in at the start of next video
+      filterComplex += `[${i + 1}:v]fade=t=in:st=0:d=${duration}[v${i + 1}in];`;
+    }
+  }
+
+  // Concatenate the faded segments
+  const concatInputs = videoPaths.map((_, i) => {
+    if (i === 0) return '[v0out]';
+    if (i === videoPaths.length - 1) return `[v${i}in]`;
+    return `[v${i}out][v${i}in]`;
+  }).join('');
+
+  filterComplex += `${concatInputs}concat=n=${videoPaths.length}:v=1:a=0[outv]`;
+
+  await execAsync(
+    `ffmpeg ${inputs} -filter_complex "${filterComplex}" -map "[outv]" "${outputPath}" -y`
+  );
+}

package/src/utils/file-handler.ts

@@ -0,0 +1,10 @@
+/**
+ * File Handler Utilities
+ *
+ * Utilities for handling file operations
+ */
+
+export function ensureDirectory(_path: string): Promise<void> {
+  // Placeholder - will be implemented when needed
+  return Promise.resolve();
+}

package/src/utils/image-processing.ts

@@ -0,0 +1,10 @@
+/**
+ * Image Processing Utilities
+ *
+ * Utilities for processing and manipulating images
+ */
+
+export function resizeImage(buffer: Buffer, _width: number, _height: number): Promise<Buffer> {
+  // Placeholder - will be implemented when needed
+  return Promise.resolve(buffer);
+}

package/test-analyze-screenshot.ts

@@ -0,0 +1,50 @@
+import { GeminiProvider } from './src/providers/gemini.js';
+import * as fs from 'fs/promises';
+
+async function analyzeScreenshot() {
+  console.log('🔍 Analyzing Screenshot...\n');
+  
+  const provider = new GeminiProvider();
+  const imagePath = '/Volumes/DevHub_ext/factory/toolbox/opencode-nanobanana/generated-assets/screenshots/IMG_3619.jpg';
+  
+  console.log(`📁 Image: ${imagePath}\n`);
+  
+  const imageBuffer = await fs.readFile(imagePath);
+  console.log(`📦 Size: ${(imageBuffer.length / 1024).toFixed(1)}KB\n`);
+  
+  const question = `Analyze this UI screenshot in detail. Please provide:
+
+1. **OVERVIEW**: What app/screen is this? What is its purpose?
+
+2. **WHAT'S GOOD** ✅:
+   - Layout and composition
+   - Typography and readability
+   - Color scheme and contrast
+   - Visual hierarchy
+   - User experience elements
+
+3. **WHAT NEEDS IMPROVEMENT** ❌:
+   - Accessibility issues
+   - Design inconsistencies
+   - Usability problems
+   - Visual clutter or confusion
+   - Missing elements
+
+4. **SPECIFIC RECOMMENDATIONS**:
+   - Actionable suggestions to improve the UI
+
+Be specific and reference actual elements you see in the screenshot.`;
+
+  console.log('⏳ Analyzing with AI vision...\n');
+  
+  const startTime = Date.now();
+  const analysis = await provider.analyzeImage(imageBuffer, question);
+  const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+  
+  console.log(`✅ Analysis complete in ${duration}s\n`);
+  console.log('━'.repeat(60));
+  console.log('\n' + analysis + '\n');
+  console.log('━'.repeat(60));
+}
+
+analyzeScreenshot().catch(console.error);