@agent-foundry/replay-server 1.0.0

package/src/renderer/PuppeteerRenderer.ts

@@ -0,0 +1,512 @@
+/**
+ * Puppeteer-based Video Renderer
+ * 
+ * This renderer:
+ * 1. Launches a headless browser
+ * 2. Opens the game at a special replay URL
+ * 3. Injects the replay manifest
+ * 4. Captures screenshots as JPEG buffers and streams directly to FFmpeg via stdin
+ * 5. Encodes video using libx264 (CPU) by default, or h264_nvenc if explicitly enabled
+ */
+
+import puppeteer, { Browser, Page } from 'puppeteer';
+import { spawn, ChildProcess } from 'child_process';
+import * as path from 'path';
+import type { ReplayManifestV1 } from '@agent-foundry/replay';
+
+export interface RenderConfig {
+    /** URL of the game (can be local file:// or http://) */
+    gameUrl: string;
+
+    /** Output video path */
+    outputPath: string;
+
+    /** Video dimensions */
+    width?: number;
+    height?: number;
+
+    /** Frames per second */
+    fps?: number;
+
+    /** Seconds to display each age */
+    secondsPerAge?: number;
+
+    /** Enable hardware acceleration (NVENC). Default: false (uses CPU libx264) */
+    useHardwareAcceleration?: boolean;
+
+    /** Temporary directory for screenshots (deprecated, no longer used) */
+    tempDir?: string;
+
+    /** Whether to keep temp files after rendering (deprecated, no longer used) */
+    keepTempFiles?: boolean;
+
+    /** Callback for progress updates */
+    onProgress?: (progress: RenderProgress) => void;
+}
+
+export interface RenderProgress {
+    stage: 'init' | 'capture' | 'encode' | 'complete' | 'error';
+    current: number;
+    total: number;
+    message: string;
+}
+
+export interface RenderResult {
+    success: boolean;
+    outputPath?: string;
+    duration?: number;
+    error?: string;
+}
+
+const DEFAULT_CONFIG = {
+    width: 720,
+    height: 1280,
+    fps: 16,
+    secondsPerAge: 1.2,
+    keepTempFiles: false,
+    useHardwareAcceleration: false, // Default to CPU encoding
+};
+
+export class PuppeteerRenderer {
+    private browser: Browser | null = null;
+    private page: Page | null = null;
+    private pageErrors: string[] = [];
+    private failedRequests: Array<{ url: string; errorText?: string }> = [];
+
+    /**
+     * Render a replay manifest to video
+     */
+    async render(manifest: ReplayManifestV1, config: RenderConfig): Promise<RenderResult> {
+        const startTime = Date.now();
+        const fullConfig: RenderConfig & typeof DEFAULT_CONFIG = {
+            ...DEFAULT_CONFIG,
+            gameUrl: config.gameUrl,
+            outputPath: config.outputPath,
+            width: config.width ?? DEFAULT_CONFIG.width,
+            height: config.height ?? DEFAULT_CONFIG.height,
+            fps: config.fps ?? DEFAULT_CONFIG.fps,
+            secondsPerAge: config.secondsPerAge ?? DEFAULT_CONFIG.secondsPerAge,
+            useHardwareAcceleration: config.useHardwareAcceleration ?? DEFAULT_CONFIG.useHardwareAcceleration,
+            onProgress: config.onProgress,
+        };
+
+        try {
+            // Initialize browser
+            this.reportProgress(config.onProgress, 'init', 0, 1, 'Launching browser...');
+            await this.initBrowser(fullConfig);
+
+            // Navigate to game
+            this.reportProgress(config.onProgress, 'init', 0, 1, 'Loading game...');
+            await this.loadGame(fullConfig.gameUrl, manifest);
+
+            // Capture and encode frames in streaming mode
+            const frameCount = manifest.timeline.length * fullConfig.secondsPerAge * fullConfig.fps;
+            this.reportProgress(config.onProgress, 'capture', 0, frameCount, 'Starting capture & encode...');
+            await this.captureAndEncodeFrames(manifest, fullConfig, config.onProgress);
+
+            const duration = (Date.now() - startTime) / 1000;
+            this.reportProgress(config.onProgress, 'complete', 1, 1, `Completed in ${duration.toFixed(1)}s`);
+
+            return {
+                success: true,
+                outputPath: fullConfig.outputPath,
+                duration,
+            };
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            this.reportProgress(config.onProgress, 'error', 0, 1, errorMessage);
+
+            return {
+                success: false,
+                error: errorMessage,
+            };
+        } finally {
+            await this.closeBrowser();
+        }
+    }
+
+    private async initBrowser(config: RenderConfig & typeof DEFAULT_CONFIG): Promise<void> {
+        this.browser = await puppeteer.launch({
+            executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium',
+            headless: true,
+            timeout: 120_000, // 2 minutes for cold starts
+            dumpio: true, // CRITICAL: logs chromium stderr/stdout
+            args: [
+                '--no-sandbox',
+                '--disable-setuid-sandbox',
+                '--disable-dev-shm-usage', // /dev/shm is small in containers
+                '--no-zygote', // no zygote process in containers
+                '--single-process', // safer in containerized environments
+                '--disable-gpu',
+                '--disable-software-rasterizer',
+                '--disable-background-networking',
+                '--disable-default-apps',
+                '--disable-extensions',
+                '--disable-sync',
+                '--disable-translate',
+                '--metrics-recording-only',
+                '--mute-audio',
+                '--no-first-run',
+                '--safebrowsing-disable-auto-update',
+                `--window-size=${config.width},${config.height}`,
+            ],
+        });
+
+        this.page = await this.browser.newPage();
+        await this.page.setViewport({
+            width: config.width,
+            height: config.height,
+            deviceScaleFactor: 1,
+        });
+
+        // Set up error tracking
+        this.pageErrors = [];
+        this.failedRequests = [];
+
+        this.page.on('pageerror', (error) => {
+            this.pageErrors.push(error.message);
+            console.error('📼 Page error:', error.message);
+        });
+
+        this.page.on('requestfailed', (request) => {
+            const failure = request.failure();
+            this.failedRequests.push({
+                url: request.url(),
+                errorText: failure?.errorText,
+            });
+            console.error('📼 Request failed:', request.url(), failure?.errorText);
+        });
+    }
+
+    private async loadGame(gameUrl: string, manifest: ReplayManifestV1): Promise<void> {
+        if (!this.page) throw new Error('Browser not initialized');
+
+        // Navigate to game with replay mode query parameter
+        const url = new URL(gameUrl);
+        url.searchParams.set('mode', 'replay');
+
+        console.log(`📼 Navigating to ${url.toString()}`);
+
+        await this.page.goto(url.toString(), {
+            waitUntil: 'networkidle0',
+            timeout: 30000,
+        });
+
+        console.log('📼 Page loaded, waiting for React to initialize...');
+
+        // Wait for React to initialize (check for the app container)
+        try {
+            await this.page.waitForFunction(() => {
+                // Wait for the app to be mounted and initial loading to complete
+                const app = document.querySelector('ion-app');
+                const loading = document.querySelector('ion-spinner');
+                return app !== null && loading === null;
+            }, { timeout: 15000 });
+        } catch (error) {
+            // Log errors before failing
+            if (this.pageErrors.length > 0) {
+                console.error('📼 Page errors detected:', this.pageErrors);
+            }
+            if (this.failedRequests.length > 0) {
+                console.error('📼 Failed requests:', this.failedRequests);
+            }
+            throw error;
+        }
+
+        // Log any errors that occurred (but didn't prevent initialization)
+        if (this.pageErrors.length > 0) {
+            console.warn('📼 Page errors detected (non-fatal):', this.pageErrors);
+        }
+        if (this.failedRequests.length > 0) {
+            console.warn('📼 Failed requests (non-fatal):', this.failedRequests);
+        }
+
+        console.log('📼 React initialized, injecting manifest...');
+
+        // Inject the replay manifest with retry logic
+        const maxRetries = 3;
+        let retryCount = 0;
+        let success = false;
+
+        while (retryCount < maxRetries && !success) {
+            // Inject the replay manifest
+            await this.page.evaluate((manifestJson: string) => {
+                // Store manifest in window for the game to read
+                (window as unknown as { __REPLAY_MANIFEST__: unknown }).__REPLAY_MANIFEST__ = JSON.parse(manifestJson);
+
+                console.log('📼 Manifest injected, dispatching event...');
+
+                // Dispatch event to notify the game
+                window.dispatchEvent(new CustomEvent('replay-manifest-loaded', {
+                    detail: JSON.parse(manifestJson)
+                }));
+            }, JSON.stringify(manifest));
+
+            // Wait for the replay ready indicator
+            try {
+                await this.page.waitForSelector('[data-replay-ready="true"]', {
+                    timeout: 5000,
+                });
+                success = true;
+                console.log('📼 Replay mode ready!');
+            } catch {
+                retryCount++;
+                console.log(`📼 Retry ${retryCount}/${maxRetries}: Waiting for replay ready...`);
+
+                if (retryCount < maxRetries) {
+                    // Small delay before retry
+                    await new Promise(resolve => setTimeout(resolve, 500));
+                }
+            }
+        }
+
+        if (!success) {
+            console.warn('📼 Warning: Could not confirm replay ready state, proceeding anyway...');
+            // Give the app a bit more time to process
+            await new Promise(resolve => setTimeout(resolve, 2000));
+        }
+
+        // Additional wait for UI to stabilize
+        await new Promise(resolve => setTimeout(resolve, 500));
+    }
+
+    /**
+     * Capture frames and encode video in streaming mode using FFmpeg stdin
+     * Frames are captured as JPEG buffers and streamed directly to FFmpeg
+     */
+    private async captureAndEncodeFrames(
+        manifest: ReplayManifestV1,
+        config: RenderConfig & typeof DEFAULT_CONFIG,
+        onProgress?: (progress: RenderProgress) => void
+    ): Promise<void> {
+        // Use encoder based on configuration
+        const encoder = config.useHardwareAcceleration ? 'h264_nvenc' : 'libx264';
+        console.log(`📼 Using encoder: ${encoder} (Hardware acceleration: ${config.useHardwareAcceleration ? 'enabled' : 'disabled'})`);
+
+        await this.captureAndEncodeFramesWithEncoder(manifest, config, encoder, onProgress);
+    }
+
+    /**
+     * Internal method to capture and encode with a specific encoder
+     */
+    private async captureAndEncodeFramesWithEncoder(
+        manifest: ReplayManifestV1,
+        config: RenderConfig & typeof DEFAULT_CONFIG,
+        encoder: 'h264_nvenc' | 'libx264',
+        onProgress?: (progress: RenderProgress) => void
+    ): Promise<void> {
+        if (!this.page) throw new Error('Browser not initialized');
+
+        const totalAges = manifest.timeline.length;
+        const framesPerAge = config.secondsPerAge * config.fps;
+        const totalFrames = totalAges * framesPerAge;
+
+        console.log(`📼 Using encoder: ${encoder}`);
+
+        // Launch FFmpeg process with stdin input
+        const ffmpeg = this.launchFFmpeg(encoder, config);
+
+        let frameIndex = 0;
+        let ffmpegStderr = '';
+
+        // Create promise for waiting FFmpeg completion
+        let ffmpegResolve: (() => void) | null = null;
+        let ffmpegReject: ((err: Error) => void) | null = null;
+        let ffmpegClosed = false;
+        let encodingTimeout: NodeJS.Timeout | null = null;
+
+        // Create promise that will be resolved when FFmpeg finishes
+        const encodingPromise = new Promise<void>((resolve, reject) => {
+            ffmpegResolve = resolve;
+            ffmpegReject = reject;
+
+            // Set timeout to prevent hanging (5 minutes should be enough for encoding)
+            encodingTimeout = setTimeout(() => {
+                if (!ffmpegClosed) {
+                    ffmpegClosed = true;
+                    const timeoutError = new Error(`FFmpeg encoding timeout after 5 minutes. Process may be stuck.`);
+                    reject(timeoutError);
+                    // Force kill if still running
+                    if (ffmpeg && !ffmpeg.killed) {
+                        ffmpeg.kill('SIGKILL');
+                    }
+                }
+            }, 5 * 60 * 1000); // 5 minutes timeout
+        });
+
+        // Handle FFmpeg stderr output (for logging and debugging)
+        ffmpeg.stderr?.on('data', (data: Buffer) => {
+            const output = data.toString();
+            ffmpegStderr += output;
+        });
+
+        // Handle FFmpeg process errors
+        ffmpeg.on('error', (err: Error) => {
+            if (ffmpegReject && !ffmpegClosed) {
+                ffmpegClosed = true;
+                if (encodingTimeout) clearTimeout(encodingTimeout);
+                ffmpegReject(new Error(`Failed to start FFmpeg: ${err.message}`));
+            }
+        });
+
+        // Set up close event handler BEFORE starting capture
+        ffmpeg.on('close', (code: number) => {
+            if (ffmpegClosed) return; // Prevent multiple calls
+            ffmpegClosed = true;
+
+            if (encodingTimeout) {
+                clearTimeout(encodingTimeout);
+            }
+
+            if (code === 0) {
+                if (ffmpegResolve) {
+                    ffmpegResolve();
+                }
+            } else {
+                const error = new Error(`FFmpeg exited with code ${code}. Stderr: ${ffmpegStderr}`);
+                if (ffmpegReject) {
+                    ffmpegReject(error);
+                }
+            }
+        });
+
+        // Wait for FFmpeg stdin to be ready
+        if (!ffmpeg.stdin) {
+            throw new Error('FFmpeg stdin not available');
+        }
+
+        try {
+            // Capture and stream frames
+            for (let ageIndex = 0; ageIndex < totalAges; ageIndex++) {
+                // Advance to next age in the game
+                await this.page.evaluate(() => {
+                    window.dispatchEvent(new CustomEvent('replay-next-age'));
+                });
+
+                // Wait a bit for animation
+                await new Promise(resolve => setTimeout(resolve, 100));
+
+                // Capture frames for this age
+                for (let f = 0; f < framesPerAge; f++) {
+                    // Check if FFmpeg process has closed unexpectedly
+                    if (ffmpegClosed) {
+                        throw new Error('FFmpeg process closed unexpectedly');
+                    }
+
+                    // Capture screenshot as JPEG buffer
+                    const screenshotBuffer = await this.page.screenshot({
+                        type: 'jpeg',
+                        quality: 85,
+                        encoding: 'binary',
+                    }) as Buffer;
+
+                    // Write frame to FFmpeg stdin
+                    if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
+                        const writeSuccess = ffmpeg.stdin.write(screenshotBuffer);
+                        if (!writeSuccess) {
+                            // Wait for drain event if buffer is full
+                            await new Promise<void>((resolve) => {
+                                if (ffmpeg.stdin) {
+                                    ffmpeg.stdin.once('drain', resolve);
+                                } else {
+                                    resolve();
+                                }
+                            });
+                        }
+                    }
+
+                    frameIndex++;
+
+                    // Update progress
+                    if (frameIndex % config.fps === 0) {
+                        this.reportProgress(
+                            onProgress,
+                            'capture',
+                            frameIndex,
+                            totalFrames,
+                            `Capturing & encoding frame ${frameIndex}/${totalFrames} (${encoder})`
+                        );
+                    }
+
+                    // Small delay between frames
+                    await new Promise(resolve => setTimeout(resolve, 1000 / config.fps / 2));
+                }
+            }
+
+            // Close stdin to signal end of input
+            if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
+                ffmpeg.stdin.end();
+            }
+
+            // Wait for FFmpeg to finish encoding
+            // The promise was created earlier with event handlers already set up
+            await encodingPromise;
+        } catch (error) {
+            // Cleanup on error
+            if (!ffmpegClosed) {
+                ffmpegClosed = true;
+                if (encodingTimeout) {
+                    clearTimeout(encodingTimeout);
+                }
+                if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
+                    ffmpeg.stdin.destroy();
+                }
+                if (ffmpeg && !ffmpeg.killed) {
+                    ffmpeg.kill('SIGKILL');
+                }
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Launch FFmpeg process with appropriate parameters
+     */
+    private launchFFmpeg(
+        encoder: 'h264_nvenc' | 'libx264',
+        config: RenderConfig & typeof DEFAULT_CONFIG
+    ): ChildProcess {
+        const args = [
+            '-y', // Overwrite output
+            '-f', 'image2pipe', // Read from stdin
+            '-vcodec', 'mjpeg', // Input format: JPEG
+            '-r', String(config.fps), // Input framerate
+            '-i', '-', // Read from stdin
+            '-c:v', encoder, // Video encoder
+            '-preset', 'fast', // Encoding preset
+            '-crf', '28', // Constant Rate Factor (higher = smaller file, lower quality)
+            '-pix_fmt', 'yuv420p', // Pixel format for compatibility
+            '-r', String(config.fps), // Output framerate
+            '-movflags', '+faststart', // Enable fast start for web playback
+            config.outputPath,
+        ];
+
+        // Add NVENC-specific parameters if using hardware encoder
+        if (encoder === 'h264_nvenc') {
+            // Insert NVENC-specific options before output path
+            args.splice(args.length - 1, 0, '-rc', 'vbr', '-cq', '28');
+        }
+
+        return spawn('ffmpeg', args);
+    }
+
+    private async closeBrowser(): Promise<void> {
+        if (this.browser) {
+            await this.browser.close();
+            this.browser = null;
+            this.page = null;
+        }
+    }
+
+    private reportProgress(
+        callback: ((progress: RenderProgress) => void) | undefined,
+        stage: RenderProgress['stage'],
+        current: number,
+        total: number,
+        message: string
+    ): void {
+        if (callback) {
+            callback({ stage, current, total, message });
+        }
+    }
+}

package/src/renderer/index.ts

@@ -0,0 +1 @@
+export * from './PuppeteerRenderer.js';