@agent-foundry/replay-server 1.0.2 → 1.0.3

package/src/renderer/PuppeteerRenderer.ts

@@ -1,665 +0,0 @@
-/**
- * Puppeteer-based Video Renderer
- * 
- * This renderer:
- * 1. Launches a headless browser
- * 2. Opens the game at a special replay URL
- * 3. Injects the replay manifest
- * 4. Captures screenshots as JPEG buffers and streams directly to FFmpeg via stdin
- * 5. Encodes video using libx264 (CPU) by default, or h264_nvenc if explicitly enabled
- */
-
-import puppeteer, { Browser, Page } from 'puppeteer';
-import { spawn, ChildProcess } from 'child_process';
-import * as path from 'path';
-
-/**
- * Generic replay manifest interface
- * Supports any game type - games interpret their own schemas
- */
-export interface GenericReplayManifest {
-    schema: string;
-    bundleId?: string;
-    gameId: string;
-    timeline?: unknown[]; // Optional - games may not have timelines (LifeRestart uses this)
-    summary?: {
-        durationSeconds?: number; // JumpArena provides duration directly
-        [key: string]: unknown;
-    };
-    [key: string]: unknown;
-}
-
-/**
- * Replay mode determines how frames are captured:
- * - 'age-based': Discrete ages with replay-next-age events (LifeRestart)
- * - 'continuous': Real-time simulation without events (JumpArena)
- */
-type ReplayMode = 'age-based' | 'continuous';
-
-/**
- * Replay timing configuration calculated from manifest
- */
-interface ReplayTiming {
-    mode: ReplayMode;
-    durationSeconds: number;
-    totalFrames: number;
-}
-
-export interface RenderConfig {
-    /** URL of the game (can be local file:// or http://) */
-    gameUrl: string;
-
-    /** Output video path */
-    outputPath: string;
-
-    /** Video dimensions */
-    width?: number;
-    height?: number;
-
-    /** Frames per second */
-    fps?: number;
-
-    /** Seconds to display each age */
-    secondsPerAge?: number;
-
-    /** Enable hardware acceleration (NVENC). Default: false (uses CPU libx264) */
-    useHardwareAcceleration?: boolean;
-
-    /** Temporary directory for screenshots (deprecated, no longer used) */
-    tempDir?: string;
-
-    /** Whether to keep temp files after rendering (deprecated, no longer used) */
-    keepTempFiles?: boolean;
-
-    /** Callback for progress updates */
-    onProgress?: (progress: RenderProgress) => void;
-}
-
-export interface RenderProgress {
-    stage: 'init' | 'capture' | 'encode' | 'complete' | 'error';
-    current: number;
-    total: number;
-    message: string;
-}
-
-export interface RenderResult {
-    success: boolean;
-    outputPath?: string;
-    duration?: number;
-    error?: string;
-}
-
-const DEFAULT_CONFIG = {
-    width: 720,
-    height: 1280,
-    fps: 16,
-    secondsPerAge: 1.2,
-    keepTempFiles: false,
-    useHardwareAcceleration: false, // Default to CPU encoding
-};
-
-export class PuppeteerRenderer {
-    private browser: Browser | null = null;
-    private page: Page | null = null;
-    private pageErrors: string[] = [];
-    private failedRequests: Array<{ url: string; errorText?: string }> = [];
-
-    /**
-     * Calculate replay timing based on manifest fields
-     * 
-     * Detection priority (field-based, not schema-name based):
-     * 1. If manifest.summary.durationSeconds exists → continuous mode
-     * 2. If manifest.timeline[] exists → age-based mode  
-     * 3. Fallback to 10s continuous mode
-     * 
-     * Continuous mode: Real-time simulation, frames captured at regular intervals
-     * Age-based mode: Discrete ages with replay-next-age events
-     */
-    private calculateReplayTiming(
-        manifest: GenericReplayManifest,
-        config: { fps: number; secondsPerAge: number }
-    ): ReplayTiming {
-        // Priority 1: Check for summary.durationSeconds (continuous mode)
-        const durationSeconds = manifest.summary?.durationSeconds;
-        if (typeof durationSeconds === 'number' && durationSeconds > 0) {
-            const totalFrames = Math.ceil(durationSeconds * config.fps);
-            console.log(`📼 Continuous mode (durationSeconds): ${durationSeconds.toFixed(2)}s, ${totalFrames} frames`);
-            return {
-                mode: 'continuous',
-                durationSeconds,
-                totalFrames,
-            };
-        }
-
-        // Priority 2: Check for timeline[] (age-based mode)
-        if (Array.isArray(manifest.timeline) && manifest.timeline.length > 0) {
-            const timelineLength = manifest.timeline.length;
-            const duration = timelineLength * config.secondsPerAge;
-            const totalFrames = Math.ceil(duration * config.fps);
-            console.log(`📼 Age-based mode (timeline): ${timelineLength} ages, ${duration.toFixed(2)}s, ${totalFrames} frames`);
-            return {
-                mode: 'age-based',
-                durationSeconds: duration,
-                totalFrames,
-            };
-        }
-
-        // Fallback: No duration info, use default continuous mode
-        console.warn('📼 No duration info in manifest, using default 10s continuous mode');
-        return {
-            mode: 'continuous',
-            durationSeconds: 10,
-            totalFrames: Math.ceil(10 * config.fps),
-        };
-    }
-
-    /**
-     * Render a replay manifest to video
-     */
-    async render(manifest: GenericReplayManifest, config: RenderConfig): Promise<RenderResult> {
-        const startTime = Date.now();
-        const fullConfig: RenderConfig & typeof DEFAULT_CONFIG = {
-            ...DEFAULT_CONFIG,
-            gameUrl: config.gameUrl,
-            outputPath: config.outputPath,
-            width: config.width ?? DEFAULT_CONFIG.width,
-            height: config.height ?? DEFAULT_CONFIG.height,
-            fps: config.fps ?? DEFAULT_CONFIG.fps,
-            secondsPerAge: config.secondsPerAge ?? DEFAULT_CONFIG.secondsPerAge,
-            useHardwareAcceleration: config.useHardwareAcceleration ?? DEFAULT_CONFIG.useHardwareAcceleration,
-            onProgress: config.onProgress,
-        };
-
-        try {
-            // Initialize browser
-            this.reportProgress(config.onProgress, 'init', 0, 1, 'Launching browser...');
-            await this.initBrowser(fullConfig);
-
-            // Navigate to game
-            this.reportProgress(config.onProgress, 'init', 0, 1, 'Loading game...');
-            await this.loadGame(fullConfig.gameUrl, manifest);
-
-            // Calculate timing based on manifest schema
-            const timing = this.calculateReplayTiming(manifest, {
-                fps: fullConfig.fps,
-                secondsPerAge: fullConfig.secondsPerAge,
-            });
-
-            // Capture and encode frames in streaming mode
-            this.reportProgress(config.onProgress, 'capture', 0, timing.totalFrames, 'Starting capture & encode...');
-            await this.captureAndEncodeFrames(manifest, fullConfig, timing, config.onProgress);
-
-            const duration = (Date.now() - startTime) / 1000;
-            this.reportProgress(config.onProgress, 'complete', 1, 1, `Completed in ${duration.toFixed(1)}s`);
-
-            return {
-                success: true,
-                outputPath: fullConfig.outputPath,
-                duration,
-            };
-        } catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            this.reportProgress(config.onProgress, 'error', 0, 1, errorMessage);
-
-            return {
-                success: false,
-                error: errorMessage,
-            };
-        } finally {
-            await this.closeBrowser();
-        }
-    }
-
-    private async initBrowser(config: RenderConfig & typeof DEFAULT_CONFIG): Promise<void> {
-        this.browser = await puppeteer.launch({
-            executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium',
-            headless: true,
-            timeout: 120_000, // 2 minutes for cold starts
-            dumpio: true, // CRITICAL: logs chromium stderr/stdout
-            args: [
-                '--no-sandbox',
-                '--disable-setuid-sandbox',
-                '--disable-dev-shm-usage', // /dev/shm is small in containers
-                '--no-zygote', // no zygote process in containers
-                '--single-process', // safer in containerized environments
-                '--disable-gpu',
-                '--disable-software-rasterizer',
-                '--disable-background-networking',
-                '--disable-default-apps',
-                '--disable-extensions',
-                '--disable-sync',
-                '--disable-translate',
-                '--metrics-recording-only',
-                '--mute-audio',
-                '--no-first-run',
-                '--safebrowsing-disable-auto-update',
-                `--window-size=${config.width},${config.height}`,
-            ],
-        });
-
-        this.page = await this.browser.newPage();
-        await this.page.setViewport({
-            width: config.width,
-            height: config.height,
-            deviceScaleFactor: 1,
-        });
-
-        // Set up error tracking
-        this.pageErrors = [];
-        this.failedRequests = [];
-
-        this.page.on('pageerror', (error) => {
-            this.pageErrors.push(error.message);
-            console.error('📼 Page error:', error.message);
-        });
-
-        this.page.on('requestfailed', (request) => {
-            const failure = request.failure();
-            this.failedRequests.push({
-                url: request.url(),
-                errorText: failure?.errorText,
-            });
-            console.error('📼 Request failed:', request.url(), failure?.errorText);
-        });
-    }
-
-    private async loadGame(gameUrl: string, manifest: GenericReplayManifest): Promise<void> {
-        if (!this.page) throw new Error('Browser not initialized');
-
-        // Navigate to game with replay mode query parameter
-        const url = new URL(gameUrl);
-        url.searchParams.set('mode', 'replay');
-
-        console.log(`📼 Navigating to ${url.toString()}`);
-
-        await this.page.goto(url.toString(), {
-            waitUntil: 'networkidle0',
-            timeout: 30000,
-        });
-
-        console.log('📼 Page loaded, waiting for React to initialize...');
-
-        // Wait for React to initialize
-        // Try to detect common app containers, but don't fail if not found
-        try {
-            await this.page.waitForFunction(() => {
-                // Check for common app container patterns
-                const ionApp = document.querySelector('ion-app');
-                const rootDiv = document.querySelector('#root');
-                const appDiv = document.querySelector('[id*="app"], [class*="app"]');
-                
-                // If using Ionic, wait for loading to complete
-                if (ionApp) {
-                    const loading = document.querySelector('ion-spinner');
-                    return loading === null;
-                }
-                
-                // Otherwise, just check if root is mounted
-                return rootDiv !== null || appDiv !== null;
-            }, { timeout: 10000 });
-            console.log('📼 App container detected');
-        } catch (error) {
-            // Non-fatal - continue anyway
-            console.log('📼 App container detection timed out, continuing anyway...');
-        }
-
-        // Log any errors that occurred (but didn't prevent initialization)
-        if (this.pageErrors.length > 0) {
-            console.warn('📼 Page errors detected (non-fatal):', this.pageErrors);
-        }
-        if (this.failedRequests.length > 0) {
-            console.warn('📼 Failed requests (non-fatal):', this.failedRequests);
-        }
-
-        console.log('📼 React initialized, injecting manifest...');
-
-        // Inject the replay manifest with retry logic
-        const maxRetries = 3;
-        let retryCount = 0;
-        let success = false;
-
-        while (retryCount < maxRetries && !success) {
-            // Inject the replay manifest
-            await this.page.evaluate((manifestJson: string) => {
-                // Store manifest in window for the game to read
-                (window as unknown as { __REPLAY_MANIFEST__: unknown }).__REPLAY_MANIFEST__ = JSON.parse(manifestJson);
-
-                console.log('📼 Manifest injected, dispatching event...');
-
-                // Dispatch event to notify the game
-                window.dispatchEvent(new CustomEvent('replay-manifest-loaded', {
-                    detail: JSON.parse(manifestJson)
-                }));
-            }, JSON.stringify(manifest));
-
-            // Wait for the replay ready indicator
-            try {
-                await this.page.waitForSelector('[data-replay-ready="true"]', {
-                    timeout: 5000,
-                });
-                success = true;
-                console.log('📼 Replay mode ready!');
-            } catch {
-                retryCount++;
-                console.log(`📼 Retry ${retryCount}/${maxRetries}: Waiting for replay ready...`);
-
-                if (retryCount < maxRetries) {
-                    // Small delay before retry
-                    await new Promise(resolve => setTimeout(resolve, 500));
-                }
-            }
-        }
-
-        if (!success) {
-            console.warn('📼 Warning: Could not confirm replay ready state, proceeding anyway...');
-            // Give the app a bit more time to process
-            await new Promise(resolve => setTimeout(resolve, 2000));
-        }
-
-        // Additional wait for UI to stabilize
-        await new Promise(resolve => setTimeout(resolve, 500));
-    }
-
-    /**
-     * Capture frames and encode video in streaming mode using FFmpeg stdin
-     * Frames are captured as JPEG buffers and streamed directly to FFmpeg
-     */
-    private async captureAndEncodeFrames(
-        manifest: GenericReplayManifest,
-        config: RenderConfig & typeof DEFAULT_CONFIG,
-        timing: ReplayTiming,
-        onProgress?: (progress: RenderProgress) => void
-    ): Promise<void> {
-        // Use encoder based on configuration
-        const encoder = config.useHardwareAcceleration ? 'h264_nvenc' : 'libx264';
-        console.log(`📼 Using encoder: ${encoder} (Hardware acceleration: ${config.useHardwareAcceleration ? 'enabled' : 'disabled'})`);
-
-        await this.captureAndEncodeFramesWithEncoder(manifest, config, timing, encoder, onProgress);
-    }
-
-    /**
-     * Internal method to capture and encode with a specific encoder
-     */
-    private async captureAndEncodeFramesWithEncoder(
-        manifest: GenericReplayManifest,
-        config: RenderConfig & typeof DEFAULT_CONFIG,
-        timing: ReplayTiming,
-        encoder: 'h264_nvenc' | 'libx264',
-        onProgress?: (progress: RenderProgress) => void
-    ): Promise<void> {
-        if (!this.page) throw new Error('Browser not initialized');
-
-        const { mode, totalFrames, durationSeconds } = timing;
-        
-        // For age-based mode, calculate ages and frames per age
-        const timelineLength = Array.isArray(manifest.timeline) ? manifest.timeline.length : 1;
-        const totalAges = timelineLength;
-        const framesPerAge = Math.ceil(config.secondsPerAge * config.fps);
-
-        console.log(`📼 Replay mode: ${mode}, duration: ${durationSeconds.toFixed(2)}s, frames: ${totalFrames}`);
-        console.log(`📼 Using encoder: ${encoder}`);
-
-        // Launch FFmpeg process with stdin input
-        const ffmpeg = this.launchFFmpeg(encoder, config);
-
-        let frameIndex = 0;
-        let ffmpegStderr = '';
-
-        // Create promise for waiting FFmpeg completion
-        let ffmpegResolve: (() => void) | null = null;
-        let ffmpegReject: ((err: Error) => void) | null = null;
-        let ffmpegClosed = false;
-        let encodingTimeout: NodeJS.Timeout | null = null;
-
-        // Create promise that will be resolved when FFmpeg finishes
-        const encodingPromise = new Promise<void>((resolve, reject) => {
-            ffmpegResolve = resolve;
-            ffmpegReject = reject;
-
-            // Set timeout to prevent hanging (5 minutes should be enough for encoding)
-            encodingTimeout = setTimeout(() => {
-                if (!ffmpegClosed) {
-                    ffmpegClosed = true;
-                    const timeoutError = new Error(`FFmpeg encoding timeout after 5 minutes. Process may be stuck.`);
-                    reject(timeoutError);
-                    // Force kill if still running
-                    if (ffmpeg && !ffmpeg.killed) {
-                        ffmpeg.kill('SIGKILL');
-                    }
-                }
-            }, 5 * 60 * 1000); // 5 minutes timeout
-        });
-
-        // Handle FFmpeg stderr output (for logging and debugging)
-        ffmpeg.stderr?.on('data', (data: Buffer) => {
-            const output = data.toString();
-            ffmpegStderr += output;
-        });
-
-        // Handle FFmpeg process errors
-        ffmpeg.on('error', (err: Error) => {
-            if (ffmpegReject && !ffmpegClosed) {
-                ffmpegClosed = true;
-                if (encodingTimeout) clearTimeout(encodingTimeout);
-                ffmpegReject(new Error(`Failed to start FFmpeg: ${err.message}`));
-            }
-        });
-
-        // Set up close event handler BEFORE starting capture
-        ffmpeg.on('close', (code: number) => {
-            if (ffmpegClosed) return; // Prevent multiple calls
-            ffmpegClosed = true;
-
-            if (encodingTimeout) {
-                clearTimeout(encodingTimeout);
-            }
-
-            if (code === 0) {
-                if (ffmpegResolve) {
-                    ffmpegResolve();
-                }
-            } else {
-                const error = new Error(`FFmpeg exited with code ${code}. Stderr: ${ffmpegStderr}`);
-                if (ffmpegReject) {
-                    ffmpegReject(error);
-                }
-            }
-        });
-
-        // Wait for FFmpeg stdin to be ready
-        if (!ffmpeg.stdin) {
-            throw new Error('FFmpeg stdin not available');
-        }
-
-        try {
-            // Frame interval in ms (for continuous mode timing)
-            const frameIntervalMs = 1000 / config.fps;
-
-            if (mode === 'continuous') {
-                // CONTINUOUS MODE: Capture frames at regular intervals
-                // Game runs in real-time, we just capture screenshots
-                console.log(`📼 Continuous mode: capturing ${totalFrames} frames over ${durationSeconds.toFixed(2)}s`);
-
-                for (let f = 0; f < totalFrames; f++) {
-                    // Check if FFmpeg process has closed unexpectedly
-                    if (ffmpegClosed) {
-                        throw new Error('FFmpeg process closed unexpectedly');
-                    }
-
-                    // Capture screenshot as JPEG buffer
-                    const screenshotBuffer = await this.page.screenshot({
-                        type: 'jpeg',
-                        quality: 85,
-                        encoding: 'binary',
-                    }) as Buffer;
-
-                    // Write frame to FFmpeg stdin
-                    if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
-                        const writeSuccess = ffmpeg.stdin.write(screenshotBuffer);
-                        if (!writeSuccess) {
-                            // Wait for drain event if buffer is full
-                            await new Promise<void>((resolve) => {
-                                if (ffmpeg.stdin) {
-                                    ffmpeg.stdin.once('drain', resolve);
-                                } else {
-                                    resolve();
-                                }
-                            });
-                        }
-                    }
-
-                    frameIndex++;
-
-                    // Update progress every second
-                    if (frameIndex % config.fps === 0) {
-                        this.reportProgress(
-                            onProgress,
-                            'capture',
-                            frameIndex,
-                            totalFrames,
-                            `Capturing & encoding frame ${frameIndex}/${totalFrames} (${encoder})`
-                        );
-                    }
-
-                    // Wait for next frame interval (allow game to advance simulation)
-                    await new Promise(resolve => setTimeout(resolve, frameIntervalMs));
-                }
-            } else {
-                // AGE-BASED MODE: Dispatch replay-next-age for each age
-                // Used by LifeRestart and similar games
-                console.log(`📼 Age-based mode: ${totalAges} ages, ${framesPerAge} frames per age`);
-
-                for (let ageIndex = 0; ageIndex < totalAges; ageIndex++) {
-                    // Advance to next age in the game
-                    await this.page.evaluate(() => {
-                        window.dispatchEvent(new CustomEvent('replay-next-age'));
-                    });
-
-                    // Wait a bit for animation
-                    await new Promise(resolve => setTimeout(resolve, 100));
-
-                    // Capture frames for this age
-                    for (let f = 0; f < framesPerAge; f++) {
-                        // Check if FFmpeg process has closed unexpectedly
-                        if (ffmpegClosed) {
-                            throw new Error('FFmpeg process closed unexpectedly');
-                        }
-
-                        // Capture screenshot as JPEG buffer
-                        const screenshotBuffer = await this.page.screenshot({
-                            type: 'jpeg',
-                            quality: 85,
-                            encoding: 'binary',
-                        }) as Buffer;
-
-                        // Write frame to FFmpeg stdin
-                        if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
-                            const writeSuccess = ffmpeg.stdin.write(screenshotBuffer);
-                            if (!writeSuccess) {
-                                // Wait for drain event if buffer is full
-                                await new Promise<void>((resolve) => {
-                                    if (ffmpeg.stdin) {
-                                        ffmpeg.stdin.once('drain', resolve);
-                                    } else {
-                                        resolve();
-                                    }
-                                });
-                            }
-                        }
-
-                        frameIndex++;
-
-                        // Update progress
-                        if (frameIndex % config.fps === 0) {
-                            this.reportProgress(
-                                onProgress,
-                                'capture',
-                                frameIndex,
-                                totalFrames,
-                                `Capturing & encoding frame ${frameIndex}/${totalFrames} (${encoder})`
-                            );
-                        }
-
-                        // Small delay between frames
-                        await new Promise(resolve => setTimeout(resolve, frameIntervalMs / 2));
-                    }
-                }
-            }
-
-            // Close stdin to signal end of input
-            if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
-                ffmpeg.stdin.end();
-            }
-
-            // Wait for FFmpeg to finish encoding
-            // The promise was created earlier with event handlers already set up
-            await encodingPromise;
-        } catch (error) {
-            // Cleanup on error
-            if (!ffmpegClosed) {
-                ffmpegClosed = true;
-                if (encodingTimeout) {
-                    clearTimeout(encodingTimeout);
-                }
-                if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
-                    ffmpeg.stdin.destroy();
-                }
-                if (ffmpeg && !ffmpeg.killed) {
-                    ffmpeg.kill('SIGKILL');
-                }
-            }
-            throw error;
-        }
-    }
-
-    /**
-     * Launch FFmpeg process with appropriate parameters
-     */
-    private launchFFmpeg(
-        encoder: 'h264_nvenc' | 'libx264',
-        config: RenderConfig & typeof DEFAULT_CONFIG
-    ): ChildProcess {
-        const args = [
-            '-y', // Overwrite output
-            '-f', 'image2pipe', // Read from stdin
-            '-vcodec', 'mjpeg', // Input format: JPEG
-            '-r', String(config.fps), // Input framerate
-            '-i', '-', // Read from stdin
-            '-c:v', encoder, // Video encoder
-            '-preset', 'fast', // Encoding preset
-            '-crf', '28', // Constant Rate Factor (higher = smaller file, lower quality)
-            '-pix_fmt', 'yuv420p', // Pixel format for compatibility
-            '-r', String(config.fps), // Output framerate
-            '-movflags', '+faststart', // Enable fast start for web playback
-            config.outputPath,
-        ];
-
-        // Add NVENC-specific parameters if using hardware encoder
-        if (encoder === 'h264_nvenc') {
-            // Insert NVENC-specific options before output path
-            args.splice(args.length - 1, 0, '-rc', 'vbr', '-cq', '28');
-        }
-
-        return spawn('ffmpeg', args);
-    }
-
-    private async closeBrowser(): Promise<void> {
-        if (this.browser) {
-            await this.browser.close();
-            this.browser = null;
-            this.page = null;
-        }
-    }
-
-    private reportProgress(
-        callback: ((progress: RenderProgress) => void) | undefined,
-        stage: RenderProgress['stage'],
-        current: number,
-        total: number,
-        message: string
-    ): void {
-        if (callback) {
-            callback({ stage, current, total, message });
-        }
-    }
-}

package/src/renderer/index.ts

@@ -1 +0,0 @@
-export * from './PuppeteerRenderer.js';