@studiomeyer/mcp-video 1.0.0

package/src/tools/engine/narrated-video.ts

@@ -0,0 +1,267 @@
+/**
+ * Narrated Video Engine
+ * Combines TTS voice generation with website recording
+ * for fully automated explainer videos
+ *
+ * Flow:
+ * 1. Generate speech from script segments
+ * 2. Record website scenes synchronized to speech durations
+ * 3. Merge video + audio into final output
+ */
+
+import { execFile } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import { logger } from '../../lib/logger.js';
+import { generateSpeech } from './tts.js';
+import type { TTSProvider, ElevenLabsVoice, ElevenLabsModel, OpenAIVoice, OpenAIModel } from './tts.js';
+import { recordWebsite } from './capture.js';
+import type { Scene, ViewportPreset } from './types.js';
+import { getMediaDuration } from './audio.js';
+
+// ─── Types ──────────────────────────────────────────────────────────
+
+export interface NarrationSegment {
+  /** Text to speak for this segment */
+  text: string;
+  /** Scene action during this segment */
+  scene: Scene;
+  /** Extra padding time after speech (seconds, default: 0.5) */
+  paddingAfter?: number;
+}
+
+export interface NarratedVideoConfig {
+  /** URL to record */
+  url: string;
+  /** Narration script segments */
+  segments: NarrationSegment[];
+  /** Output path (without extension) */
+  outputPath: string;
+  /** TTS provider (default: elevenlabs) */
+  provider?: TTSProvider;
+  /** Language (default: en) */
+  language?: string;
+  /** Viewport (default: desktop) */
+  viewport?: ViewportPreset;
+
+  // Voice settings
+  /** ElevenLabs voice (default: adam) */
+  elevenLabsVoice?: ElevenLabsVoice | string;
+  /** ElevenLabs model */
+  elevenLabsModel?: ElevenLabsModel;
+  /** OpenAI voice */
+  openaiVoice?: OpenAIVoice;
+  /** OpenAI model */
+  openaiModel?: OpenAIModel;
+  /** Speaking speed (default: 1.0) */
+  speed?: number;
+
+  /** Music volume if background music provided (default: 0.1) */
+  backgroundMusicVolume?: number;
+  /** Background music path (optional) */
+  backgroundMusicPath?: string;
+}
+
+export interface NarratedVideoResult {
+  success: boolean;
+  video: {
+    path: string;
+    duration: number;
+    sizeMB: string;
+  };
+  audio: {
+    totalSegments: number;
+    totalDuration: number;
+    provider: TTSProvider;
+  };
+  url: string;
+}
+
+// ─── Main Function ──────────────────────────────────────────────────
+
+export async function createNarratedVideo(
+  config: NarratedVideoConfig
+): Promise<NarratedVideoResult> {
+  const {
+    url,
+    segments,
+    outputPath,
+    provider = 'elevenlabs',
+    language = 'en',
+    viewport = 'desktop',
+  } = config;
+
+  const tempDir = `/tmp/narrated-video-${Date.now()}`;
+  fs.mkdirSync(tempDir, { recursive: true });
+
+  try {
+    // ─── Step 1: Generate all speech segments ───────────────────
+    logger.info(`Generating ${segments.length} speech segment(s)...`);
+    const audioPaths: string[] = [];
+    const audioDurations: number[] = [];
+
+    for (let i = 0; i < segments.length; i++) {
+      const seg = segments[i];
+      const audioPath = path.join(tempDir, `segment-${String(i).padStart(3, '0')}.mp3`);
+
+      const ttsResult = await generateSpeech({
+        text: seg.text,
+        outputPath: audioPath,
+        provider,
+        language,
+        elevenLabsVoice: config.elevenLabsVoice,
+        elevenLabsModel: config.elevenLabsModel,
+        openaiVoice: config.openaiVoice,
+        openaiModel: config.openaiModel,
+        speed: config.speed,
+      });
+
+      audioPaths.push(ttsResult.audioPath);
+      audioDurations.push(ttsResult.duration);
+      logger.info(`Segment ${i + 1}: "${seg.text.slice(0, 50)}..." → ${ttsResult.duration.toFixed(1)}s`);
+    }
+
+    // ─── Step 2: Concatenate audio segments ─────────────────────
+    logger.info('Concatenating audio segments...');
+    const fullAudioPath = path.join(tempDir, 'full-narration.mp3');
+    await concatenateAudio(audioPaths, fullAudioPath);
+
+    const totalAudioDuration = await getMediaDuration(fullAudioPath);
+    logger.info(`Total narration: ${totalAudioDuration.toFixed(1)}s`);
+
+    // ─── Step 3: Build scenes with matched durations ────────────
+    logger.info('Building synchronized scenes...');
+    const scenes: Scene[] = [];
+
+    for (let i = 0; i < segments.length; i++) {
+      const seg = segments[i];
+      const padding = seg.paddingAfter ?? 0.5;
+      const sceneDuration = audioDurations[i] + padding;
+
+      // Override scene duration to match audio
+      const scene = { ...seg.scene };
+      if ('duration' in scene) {
+        scene.duration = sceneDuration;
+      }
+
+      scenes.push(scene);
+    }
+
+    // ─── Step 4: Record website with synced scenes ──────────────
+    logger.info('Recording website with synchronized scenes...');
+    const videoOnlyPath = path.join(tempDir, 'video-only');
+
+    const recordResult = await recordWebsite({
+      url,
+      outputPath: videoOnlyPath,
+      viewport,
+      fps: 60,
+      scenes,
+      cursor: { enabled: false },
+      encoding: { codec: 'h264', crf: 18 },
+    });
+
+    // ─── Step 5: Merge video + audio ────────────────────────────
+    logger.info('Merging video + narration...');
+    const finalPath = outputPath.endsWith('.mp4') ? outputPath : `${outputPath}.mp4`;
+    const outDir = path.dirname(finalPath);
+    if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
+
+    const mergeArgs: string[] = [
+      '-y',
+      '-i', recordResult.video.path,
+      '-i', fullAudioPath,
+    ];
+
+    // Optional background music
+    if (config.backgroundMusicPath && fs.existsSync(config.backgroundMusicPath)) {
+      const musicVol = config.backgroundMusicVolume ?? 0.1;
+      mergeArgs.push('-stream_loop', '-1', '-i', config.backgroundMusicPath);
+
+      // Mix narration + background music
+      mergeArgs.push(
+        '-filter_complex',
+        `[1:a]volume=1.0[narration];[2:a]volume=${musicVol},afade=t=in:st=0:d=2[music];[narration][music]amix=inputs=2:duration=first[aout]`,
+        '-map', '0:v',
+        '-map', '[aout]',
+      );
+    } else {
+      mergeArgs.push('-map', '0:v', '-map', '1:a');
+    }
+
+    mergeArgs.push(
+      '-c:v', 'copy',
+      '-c:a', 'aac', '-b:a', '192k',
+      '-shortest',
+      '-movflags', '+faststart',
+      finalPath,
+    );
+
+    await runFfmpeg(mergeArgs);
+
+    const finalStats = fs.statSync(finalPath);
+    const finalDuration = await getMediaDuration(finalPath);
+
+    // ─── Cleanup ────────────────────────────────────────────────
+    fs.rmSync(tempDir, { recursive: true, force: true });
+
+    logger.info(`Narrated video ready: ${finalPath} (${finalDuration.toFixed(1)}s, ${(finalStats.size / 1024 / 1024).toFixed(2)} MB)`);
+
+    return {
+      success: true,
+      video: {
+        path: finalPath,
+        duration: finalDuration,
+        sizeMB: (finalStats.size / 1024 / 1024).toFixed(2),
+      },
+      audio: {
+        totalSegments: segments.length,
+        totalDuration: totalAudioDuration,
+        provider,
+      },
+      url,
+    };
+  } catch (error) {
+    // Cleanup on error
+    try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
+    throw error;
+  }
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────
+
+/**
+ * Concatenate multiple audio files using ffmpeg concat demuxer
+ */
+async function concatenateAudio(files: string[], outputPath: string): Promise<void> {
+  // Create concat list file
+  const listPath = outputPath + '.txt';
+  const listContent = files.map((f) => `file '${f}'`).join('\n');
+  fs.writeFileSync(listPath, listContent);
+
+  await runFfmpeg([
+    '-y',
+    '-f', 'concat',
+    '-safe', '0',
+    '-i', listPath,
+    '-c:a', 'libmp3lame',
+    '-b:a', '192k',
+    outputPath,
+  ]);
+
+  // Cleanup list file
+  fs.unlinkSync(listPath);
+}
+
+function runFfmpeg(args: string[]): Promise<string> {
+  return new Promise((resolve, reject) => {
+    execFile('ffmpeg', args, { maxBuffer: 50 * 1024 * 1024 }, (error, stdout, stderr) => {
+      if (error) {
+        logger.error(`ffmpeg failed: ${stderr}`);
+        reject(new Error(`ffmpeg failed: ${stderr || error.message}`));
+        return;
+      }
+      resolve(stdout);
+    });
+  });
+}

package/src/tools/engine/scenes.ts

@@ -0,0 +1,309 @@
+/**
+ * Scene execution engine — processes scene definitions into frame captures
+ */
+
+import type { Page } from 'playwright';
+import type { Scene, ScrollScene, HoverScene, ClickScene, TypeScene, WaitScene, EasingName } from './types.js';
+import { applyEasing } from './easing.js';
+import { moveCursorToElement, animateClick } from './cursor.js';
+import { logger } from '../../lib/logger.js';
+
+interface FrameCallback {
+  (frameIndex: number): Promise<void>;
+}
+
+/**
+ * Execute all scenes and capture frames
+ */
+export async function executeScenes(
+  page: Page,
+  scenes: Scene[],
+  fps: number,
+  captureFrame: FrameCallback,
+  cursorEnabled: boolean
+): Promise<number> {
+  let totalFramesCaptured = 0;
+
+  for (let i = 0; i < scenes.length; i++) {
+    const scene = scenes[i];
+    logger.info(`Scene ${i + 1}/${scenes.length}: ${scene.type}`);
+
+    let framesInScene = 0;
+
+    switch (scene.type) {
+      case 'scroll':
+        framesInScene = await executeScrollScene(page, scene, fps, async (fi) => {
+          await captureFrame(totalFramesCaptured + fi);
+        });
+        break;
+
+      case 'pause':
+        framesInScene = await executePauseScene(page, scene.duration, fps, async (fi) => {
+          await captureFrame(totalFramesCaptured + fi);
+        });
+        break;
+
+      case 'hover':
+        framesInScene = await executeHoverScene(page, scene, fps, cursorEnabled, async (fi) => {
+          await captureFrame(totalFramesCaptured + fi);
+        });
+        break;
+
+      case 'click':
+        framesInScene = await executeClickScene(page, scene, fps, cursorEnabled, async (fi) => {
+          await captureFrame(totalFramesCaptured + fi);
+        });
+        break;
+
+      case 'type':
+        framesInScene = await executeTypeScene(page, scene, fps, async (fi) => {
+          await captureFrame(totalFramesCaptured + fi);
+        });
+        break;
+
+      case 'wait':
+        framesInScene = await executeWaitScene(page, scene, fps, async (fi) => {
+          await captureFrame(totalFramesCaptured + fi);
+        });
+        break;
+    }
+
+    totalFramesCaptured += framesInScene;
+    logger.debug(`Scene ${i + 1} captured ${framesInScene} frames (total: ${totalFramesCaptured})`);
+  }
+
+  return totalFramesCaptured;
+}
+
+/**
+ * Execute a scroll scene — the core of cinema-quality videos
+ */
+async function executeScrollScene(
+  page: Page,
+  scene: ScrollScene,
+  fps: number,
+  captureFrame: FrameCallback
+): Promise<number> {
+  const easingName: EasingName = scene.easing ?? 'easeInOutCubic';
+  const totalFrames = Math.ceil(scene.duration * fps);
+
+  // Resolve scroll target
+  const scrollInfo = await page.evaluate((target) => {
+    const docHeight = document.documentElement.scrollHeight;
+    const viewportHeight = window.innerHeight;
+    const currentScroll = window.scrollY;
+    const maxScroll = Math.max(0, docHeight - viewportHeight);
+
+    if (target === 'bottom') return { from: currentScroll, to: maxScroll };
+    if (target === 'top') return { from: currentScroll, to: 0 };
+    if (typeof target === 'number') return { from: currentScroll, to: Math.min(target, maxScroll) };
+
+    // CSS selector — scroll to element
+    const el = document.querySelector(target);
+    if (el) {
+      const rect = el.getBoundingClientRect();
+      const targetY = Math.min(currentScroll + rect.top - 100, maxScroll);
+      return { from: currentScroll, to: targetY };
+    }
+
+    return { from: currentScroll, to: maxScroll };
+  }, scene.to);
+
+  const scrollDistance = scrollInfo.to - scrollInfo.from;
+
+  for (let frame = 0; frame <= totalFrames; frame++) {
+    const progress = frame / totalFrames;
+    const easedScroll = scrollInfo.from + applyEasing(progress, scrollDistance, easingName);
+
+    await page.evaluate((y) => window.scrollTo(0, y), easedScroll);
+
+    // Small delay to let scroll-triggered animations render
+    await page.waitForTimeout(8);
+
+    await captureFrame(frame);
+  }
+
+  return totalFrames + 1;
+}
+
+/**
+ * Execute a pause scene — captures static frames
+ */
+async function executePauseScene(
+  _page: Page,
+  duration: number,
+  fps: number,
+  captureFrame: FrameCallback
+): Promise<number> {
+  const totalFrames = Math.ceil(duration * fps);
+
+  for (let frame = 0; frame < totalFrames; frame++) {
+    await captureFrame(frame);
+  }
+
+  return totalFrames;
+}
+
+/**
+ * Execute a hover scene — move cursor to element and hold
+ */
+async function executeHoverScene(
+  page: Page,
+  scene: HoverScene,
+  fps: number,
+  cursorEnabled: boolean,
+  captureFrame: FrameCallback
+): Promise<number> {
+  let frameCount = 0;
+
+  // Move cursor to element (animated)
+  if (cursorEnabled && scene.animateCursor !== false) {
+    const moveDuration = 600; // ms
+    const moveFrames = Math.ceil((moveDuration / 1000) * fps);
+
+    await moveCursorToElement(page, scene.selector, moveDuration, fps);
+
+    // Capture frames during cursor movement — take snapshots
+    for (let i = 0; i < moveFrames; i++) {
+      await captureFrame(frameCount++);
+    }
+  } else {
+    // Instant hover
+    const pos = await page.evaluate((sel) => {
+      const el = document.querySelector(sel);
+      if (!el) return null;
+      const rect = el.getBoundingClientRect();
+      return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
+    }, scene.selector);
+
+    if (pos) {
+      await page.mouse.move(pos.x, pos.y);
+    }
+  }
+
+  // Hold hover — capture frames
+  const holdFrames = Math.ceil(scene.duration * fps);
+  for (let i = 0; i < holdFrames; i++) {
+    await captureFrame(frameCount++);
+  }
+
+  return frameCount;
+}
+
+/**
+ * Execute a click scene
+ */
+async function executeClickScene(
+  page: Page,
+  scene: ClickScene,
+  fps: number,
+  cursorEnabled: boolean,
+  captureFrame: FrameCallback
+): Promise<number> {
+  let frameCount = 0;
+
+  // Move cursor to element
+  if (cursorEnabled) {
+    const moveFrames = Math.ceil(0.5 * fps);
+    await moveCursorToElement(page, scene.selector, 500, fps);
+    for (let i = 0; i < moveFrames; i++) {
+      await captureFrame(frameCount++);
+    }
+  }
+
+  // Click animation
+  if (cursorEnabled) {
+    await animateClick(page);
+  }
+
+  // Actual click
+  await page.click(scene.selector);
+
+  // Wait for navigation/content
+  if (scene.waitFor === 'networkidle') {
+    await page.waitForLoadState('networkidle').catch(() => {});
+  } else if (scene.waitFor === 'load') {
+    await page.waitForLoadState('load').catch(() => {});
+  } else if (typeof scene.waitFor === 'number') {
+    await page.waitForTimeout(scene.waitFor);
+  }
+
+  // Pause after click
+  const pauseDuration = scene.pauseAfter ?? 1;
+  const pauseFrames = Math.ceil(pauseDuration * fps);
+  for (let i = 0; i < pauseFrames; i++) {
+    await captureFrame(frameCount++);
+  }
+
+  return frameCount;
+}
+
+/**
+ * Execute a type scene
+ */
+async function executeTypeScene(
+  page: Page,
+  scene: TypeScene,
+  fps: number,
+  captureFrame: FrameCallback
+): Promise<number> {
+  let frameCount = 0;
+  const delay = scene.delay ?? 80;
+  const framesPerKeystroke = Math.max(1, Math.ceil((delay / 1000) * fps));
+
+  // Focus the input
+  await page.click(scene.selector);
+  await captureFrame(frameCount++);
+
+  // Type character by character
+  for (const char of scene.text) {
+    await page.keyboard.type(char, { delay: 0 });
+
+    for (let i = 0; i < framesPerKeystroke; i++) {
+      await captureFrame(frameCount++);
+    }
+  }
+
+  return frameCount;
+}
+
+/**
+ * Execute a wait scene — wait for selector, capturing frames
+ */
+async function executeWaitScene(
+  page: Page,
+  scene: WaitScene,
+  fps: number,
+  captureFrame: FrameCallback
+): Promise<number> {
+  const timeout = scene.timeout ?? 5000;
+  const maxFrames = Math.ceil((timeout / 1000) * fps);
+  let frameCount = 0;
+
+  const waitPromise = page.waitForSelector(scene.selector, { timeout }).catch(() => null);
+
+  // Capture frames while waiting
+  const startTime = Date.now();
+  while (Date.now() - startTime < timeout && frameCount < maxFrames) {
+    await captureFrame(frameCount++);
+    await page.waitForTimeout(Math.floor(1000 / fps));
+
+    // Check if element appeared
+    const found = await page.$(scene.selector);
+    if (found) break;
+  }
+
+  await waitPromise;
+  return frameCount;
+}
+
+/**
+ * Create default scenes for a simple scroll-through video
+ */
+export function createDefaultScenes(scrollDuration: number = 18): Scene[] {
+  return [
+    { type: 'pause', duration: 1.5 },
+    { type: 'scroll', to: 'bottom', duration: scrollDuration, easing: 'showcase' },
+    { type: 'pause', duration: 2 },
+  ];
+}