agent-browser 0.5.0 → 0.7.0

package/dist/browser.js

@@ -1,14 +1,19 @@
 import { chromium, firefox, webkit, devices, } from 'playwright-core';
 import path from 'node:path';
 import os from 'node:os';
+import { existsSync, mkdirSync, rmSync } from 'node:fs';
 import { getEnhancedSnapshot, parseRef } from './snapshot.js';
 /**
  * Manages the Playwright browser lifecycle with multiple tabs/windows
  */
 export class BrowserManager {
     browser = null;
-    cdpPort = null;
+    cdpEndpoint = null; // stores port number or full URL
     isPersistentContext = false;
+    browserbaseSessionId = null;
+    browserbaseApiKey = null;
+    browserUseSessionId = null;
+    browserUseApiKey = null;
     contexts = [];
     pages = [];
     activePageIndex = 0;
@@ -28,6 +33,11 @@ export class BrowserManager {
     screencastSessionId = 0;
     frameCallback = null;
     screencastFrameHandler = null;
+    // Video recording (Playwright native)
+    recordingContext = null;
+    recordingPage = null;
+    recordingOutputPath = '';
+    recordingTempDir = '';
     /**
      * Check if browser is launched
      */
@@ -507,27 +517,174 @@ export class BrowserManager {
     /**
      * Check if CDP connection needs to be re-established
      */
-    needsCdpReconnect(cdpPort) {
+    needsCdpReconnect(cdpEndpoint) {
         if (!this.browser?.isConnected())
             return true;
-        if (this.cdpPort !== cdpPort)
+        if (this.cdpEndpoint !== cdpEndpoint)
             return true;
         if (!this.isCdpConnectionAlive())
             return true;
         return false;
     }
+    /**
+     * Close a Browserbase session via API
+     */
+    async closeBrowserbaseSession(sessionId, apiKey) {
+        await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}`, {
+            method: 'DELETE',
+            headers: {
+                'X-BB-API-Key': apiKey,
+            },
+        });
+    }
+    /**
+     * Close a Browser Use session via API
+     */
+    async closeBrowserUseSession(sessionId, apiKey) {
+        const response = await fetch(`https://api.browser-use.com/api/v2/browsers/${sessionId}`, {
+            method: 'PATCH',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-Browser-Use-API-Key': apiKey,
+            },
+            body: JSON.stringify({ action: 'stop' }),
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to close Browser Use session: ${response.statusText}`);
+        }
+    }
+    /**
+     * Connect to Browserbase remote browser via CDP.
+     * Requires BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID environment variables.
+     */
+    async connectToBrowserbase() {
+        const browserbaseApiKey = process.env.BROWSERBASE_API_KEY;
+        const browserbaseProjectId = process.env.BROWSERBASE_PROJECT_ID;
+        if (!browserbaseApiKey || !browserbaseProjectId) {
+            throw new Error('BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID are required when using browserbase as a provider');
+        }
+        const response = await fetch('https://api.browserbase.com/v1/sessions', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-BB-API-Key': browserbaseApiKey,
+            },
+            body: JSON.stringify({
+                projectId: browserbaseProjectId,
+            }),
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to create Browserbase session: ${response.statusText}`);
+        }
+        const session = (await response.json());
+        const browser = await chromium.connectOverCDP(session.connectUrl).catch(() => {
+            throw new Error('Failed to connect to Browserbase session via CDP');
+        });
+        try {
+            const contexts = browser.contexts();
+            if (contexts.length === 0) {
+                throw new Error('No browser context found in Browserbase session');
+            }
+            const context = contexts[0];
+            const pages = context.pages();
+            const page = pages[0] ?? (await context.newPage());
+            this.browserbaseSessionId = session.id;
+            this.browserbaseApiKey = browserbaseApiKey;
+            this.browser = browser;
+            context.setDefaultTimeout(10000);
+            this.contexts.push(context);
+            this.pages.push(page);
+            this.activePageIndex = 0;
+            this.setupPageTracking(page);
+        }
+        catch (error) {
+            await this.closeBrowserbaseSession(session.id, browserbaseApiKey).catch((sessionError) => {
+                console.error('Failed to close Browserbase session during cleanup:', sessionError);
+            });
+            throw error;
+        }
+    }
+    /**
+     * Connect to Browser Use remote browser via CDP.
+     * Requires BROWSER_USE_API_KEY environment variable.
+     */
+    async connectToBrowserUse() {
+        const browserUseApiKey = process.env.BROWSER_USE_API_KEY;
+        if (!browserUseApiKey) {
+            throw new Error('BROWSER_USE_API_KEY is required when using browseruse as a provider');
+        }
+        const response = await fetch('https://api.browser-use.com/api/v2/browsers', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-Browser-Use-API-Key': browserUseApiKey,
+            },
+            body: JSON.stringify({}),
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to create Browser Use session: ${response.statusText}`);
+        }
+        let session;
+        try {
+            session = (await response.json());
+        }
+        catch (error) {
+            throw new Error(`Failed to parse Browser Use session response: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if (!session.id || !session.cdpUrl) {
+            throw new Error(`Invalid Browser Use session response: missing ${!session.id ? 'id' : 'cdpUrl'}`);
+        }
+        const browser = await chromium.connectOverCDP(session.cdpUrl).catch(() => {
+            throw new Error('Failed to connect to Browser Use session via CDP');
+        });
+        try {
+            const contexts = browser.contexts();
+            let context;
+            let page;
+            if (contexts.length === 0) {
+                context = await browser.newContext();
+                page = await context.newPage();
+            }
+            else {
+                context = contexts[0];
+                const pages = context.pages();
+                page = pages[0] ?? (await context.newPage());
+            }
+            this.browserUseSessionId = session.id;
+            this.browserUseApiKey = browserUseApiKey;
+            this.browser = browser;
+            context.setDefaultTimeout(60000);
+            this.contexts.push(context);
+            this.pages.push(page);
+            this.activePageIndex = 0;
+            this.setupPageTracking(page);
+            this.setupContextTracking(context);
+        }
+        catch (error) {
+            await this.closeBrowserUseSession(session.id, browserUseApiKey).catch((sessionError) => {
+                console.error('Failed to close Browser Use session during cleanup:', sessionError);
+            });
+            throw error;
+        }
+    }
     /**
      * Launch the browser with the specified options
      * If already launched, this is a no-op (browser stays open)
      */
     async launch(options) {
-        const cdpPort = options.cdpPort;
+        // Determine CDP endpoint: prefer cdpUrl over cdpPort for flexibility
+        const cdpEndpoint = options.cdpUrl ?? (options.cdpPort ? String(options.cdpPort) : undefined);
         const hasExtensions = !!options.extensions?.length;
-        if (hasExtensions && cdpPort) {
+        const hasProfile = !!options.profile;
+        if (hasExtensions && cdpEndpoint) {
             throw new Error('Extensions cannot be used with CDP connection');
         }
+        if (hasProfile && cdpEndpoint) {
+            throw new Error('Profile cannot be used with CDP connection');
+        }
         if (this.isLaunched()) {
-            const needsRelaunch = (!cdpPort && this.cdpPort !== null) || (!!cdpPort && this.needsCdpReconnect(cdpPort));
+            const needsRelaunch = (!cdpEndpoint && this.cdpEndpoint !== null) ||
+                (!!cdpEndpoint && this.needsCdpReconnect(cdpEndpoint));
             if (needsRelaunch) {
                 await this.close();
             }
@@ -535,8 +692,19 @@ export class BrowserManager {
                 return;
             }
         }
-        if (cdpPort) {
-            await this.connectViaCDP(cdpPort);
+        if (cdpEndpoint) {
+            await this.connectViaCDP(cdpEndpoint);
+            return;
+        }
+        // Cloud browser providers require explicit opt-in via -p flag or AGENT_BROWSER_PROVIDER env var
+        // -p flag takes precedence over env var
+        const provider = options.provider ?? process.env.AGENT_BROWSER_PROVIDER;
+        if (provider === 'browserbase') {
+            await this.connectToBrowserbase();
+            return;
+        }
+        if (provider === 'browseruse') {
+            await this.connectToBrowserUse();
             return;
         }
         const browserType = options.browser ?? 'chromium';
@@ -547,26 +715,51 @@ export class BrowserManager {
         const viewport = options.viewport ?? { width: 1280, height: 720 };
         let context;
         if (hasExtensions) {
+            // Extensions require persistent context in a temp directory
             const extPaths = options.extensions.join(',');
             const session = process.env.AGENT_BROWSER_SESSION || 'default';
+            // Combine extension args with custom args
+            const extArgs = [`--disable-extensions-except=${extPaths}`, `--load-extension=${extPaths}`];
+            const allArgs = options.args ? [...extArgs, ...options.args] : extArgs;
             context = await launcher.launchPersistentContext(path.join(os.tmpdir(), `agent-browser-ext-${session}`), {
                 headless: false,
                 executablePath: options.executablePath,
-                args: [`--disable-extensions-except=${extPaths}`, `--load-extension=${extPaths}`],
+                args: allArgs,
+                viewport,
+                extraHTTPHeaders: options.headers,
+                userAgent: options.userAgent,
+                ...(options.proxy && { proxy: options.proxy }),
+            });
+            this.isPersistentContext = true;
+        }
+        else if (hasProfile) {
+            // Profile uses persistent context for durable cookies/storage
+            // Expand ~ to home directory since it won't be shell-expanded
+            const profilePath = options.profile.replace(/^~\//, os.homedir() + '/');
+            context = await launcher.launchPersistentContext(profilePath, {
+                headless: options.headless ?? true,
+                executablePath: options.executablePath,
                 viewport,
                 extraHTTPHeaders: options.headers,
             });
             this.isPersistentContext = true;
         }
         else {
+            // Regular ephemeral browser
             this.browser = await launcher.launch({
                 headless: options.headless ?? true,
                 executablePath: options.executablePath,
+                args: options.args,
+            });
+            this.cdpEndpoint = null;
+            context = await this.browser.newContext({
+                viewport,
+                extraHTTPHeaders: options.headers,
+                userAgent: options.userAgent,
+                ...(options.proxy && { proxy: options.proxy }),
             });
-            this.cdpPort = null;
-            context = await this.browser.newContext({ viewport, extraHTTPHeaders: options.headers });
         }
-        context.setDefaultTimeout(10000);
+        context.setDefaultTimeout(60000);
         this.contexts.push(context);
         const page = context.pages()[0] ?? (await context.newPage());
         this.pages.push(page);
@@ -575,14 +768,36 @@ export class BrowserManager {
     }
     /**
      * Connect to a running browser via CDP (Chrome DevTools Protocol)
+     * @param cdpEndpoint Either a port number (as string) or a full WebSocket URL (ws:// or wss://)
      */
-    async connectViaCDP(cdpPort) {
-        if (!cdpPort) {
-            throw new Error('cdpPort is required for CDP connection');
+    async connectViaCDP(cdpEndpoint) {
+        if (!cdpEndpoint) {
+            throw new Error('CDP endpoint is required for CDP connection');
         }
-        const browser = await chromium.connectOverCDP(`http://localhost:${cdpPort}`).catch(() => {
-            throw new Error(`Failed to connect via CDP on port ${cdpPort}. ` +
-                `Make sure the app is running with --remote-debugging-port=${cdpPort}`);
+        // Determine the connection URL:
+        // - If it starts with ws://, wss://, http://, or https://, use it directly
+        // - If it's a numeric string (e.g., "9222"), treat as port for localhost
+        // - Otherwise, treat it as a port number for localhost
+        let cdpUrl;
+        if (cdpEndpoint.startsWith('ws://') ||
+            cdpEndpoint.startsWith('wss://') ||
+            cdpEndpoint.startsWith('http://') ||
+            cdpEndpoint.startsWith('https://')) {
+            cdpUrl = cdpEndpoint;
+        }
+        else if (/^\d+$/.test(cdpEndpoint)) {
+            // Numeric string - treat as port number (handles JSON serialization quirks)
+            cdpUrl = `http://localhost:${cdpEndpoint}`;
+        }
+        else {
+            // Unknown format - still try as port for backward compatibility
+            cdpUrl = `http://localhost:${cdpEndpoint}`;
+        }
+        const browser = await chromium.connectOverCDP(cdpUrl).catch(() => {
+            throw new Error(`Failed to connect via CDP to ${cdpUrl}. ` +
+                (cdpUrl.includes('localhost')
+                    ? `Make sure the app is running with --remote-debugging-port=${cdpEndpoint}`
+                    : 'Make sure the remote browser is accessible and the URL is correct.'));
         });
         // Validate and set up state, cleaning up browser connection if anything fails
         try {
@@ -590,13 +805,14 @@ export class BrowserManager {
             if (contexts.length === 0) {
                 throw new Error('No browser context found. Make sure the app has an open window.');
             }
-            const allPages = contexts.flatMap((context) => context.pages());
+            // Filter out pages with empty URLs, which can cause Playwright to hang
+            const allPages = contexts.flatMap((context) => context.pages()).filter((page) => page.url());
             if (allPages.length === 0) {
                 throw new Error('No page found. Make sure the app has loaded content.');
             }
             // All validation passed - commit state
             this.browser = browser;
-            this.cdpPort = cdpPort;
+            this.cdpEndpoint = cdpEndpoint;
             for (const context of contexts) {
                 this.contexts.push(context);
                 this.setupContextTracking(context);
@@ -676,7 +892,7 @@ export class BrowserManager {
         const context = await this.browser.newContext({
             viewport: viewport ?? { width: 1280, height: 720 },
         });
-        context.setDefaultTimeout(10000);
+        context.setDefaultTimeout(60000);
         this.contexts.push(context);
         const page = await context.newPage();
         this.pages.push(page);
@@ -888,10 +1104,177 @@ export class BrowserManager {
             modifiers: params.modifiers ?? 0,
         });
     }
+    /**
+     * Check if video recording is currently active
+     */
+    isRecording() {
+        return this.recordingContext !== null;
+    }
+    /**
+     * Start recording to a video file using Playwright's native video recording.
+     * Creates a fresh browser context with video recording enabled.
+     * Automatically captures current URL and transfers cookies/storage if no URL provided.
+     *
+     * @param outputPath - Path to the output video file (will be .webm)
+     * @param url - Optional URL to navigate to (defaults to current page URL)
+     */
+    async startRecording(outputPath, url) {
+        if (this.recordingContext) {
+            throw new Error("Recording already in progress. Run 'record stop' first, or use 'record restart' to stop and start a new recording.");
+        }
+        if (!this.browser) {
+            throw new Error('Browser not launched. Call launch first.');
+        }
+        // Check if output file already exists
+        if (existsSync(outputPath)) {
+            throw new Error(`Output file already exists: ${outputPath}`);
+        }
+        // Validate output path is .webm (Playwright native format)
+        if (!outputPath.endsWith('.webm')) {
+            throw new Error('Playwright native recording only supports WebM format. Please use a .webm extension.');
+        }
+        // Auto-capture current URL if none provided
+        const currentPage = this.pages.length > 0 ? this.pages[this.activePageIndex] : null;
+        const currentContext = this.contexts.length > 0 ? this.contexts[0] : null;
+        if (!url && currentPage) {
+            const currentUrl = currentPage.url();
+            if (currentUrl && currentUrl !== 'about:blank') {
+                url = currentUrl;
+            }
+        }
+        // Capture state from current context (cookies + storage)
+        let storageState;
+        if (currentContext) {
+            try {
+                storageState = await currentContext.storageState();
+            }
+            catch {
+                // Ignore errors - context might be closed or invalid
+            }
+        }
+        // Create a temp directory for video recording
+        const session = process.env.AGENT_BROWSER_SESSION || 'default';
+        this.recordingTempDir = path.join(os.tmpdir(), `agent-browser-recording-${session}-${Date.now()}`);
+        mkdirSync(this.recordingTempDir, { recursive: true });
+        this.recordingOutputPath = outputPath;
+        // Create a new context with video recording enabled and restored state
+        const viewport = { width: 1280, height: 720 };
+        this.recordingContext = await this.browser.newContext({
+            viewport,
+            recordVideo: {
+                dir: this.recordingTempDir,
+                size: viewport,
+            },
+            storageState,
+        });
+        this.recordingContext.setDefaultTimeout(10000);
+        // Create a page in the recording context
+        this.recordingPage = await this.recordingContext.newPage();
+        // Add the recording context and page to our managed lists
+        this.contexts.push(this.recordingContext);
+        this.pages.push(this.recordingPage);
+        this.activePageIndex = this.pages.length - 1;
+        // Set up page tracking
+        this.setupPageTracking(this.recordingPage);
+        // Invalidate CDP session since we switched pages
+        await this.invalidateCDPSession();
+        // Navigate to URL if provided or captured
+        if (url) {
+            await this.recordingPage.goto(url, { waitUntil: 'load' });
+        }
+    }
+    /**
+     * Stop recording and save the video file
+     * @returns Recording result with path
+     */
+    async stopRecording() {
+        if (!this.recordingContext || !this.recordingPage) {
+            return { path: '', frames: 0, error: 'No recording in progress' };
+        }
+        const outputPath = this.recordingOutputPath;
+        try {
+            // Get the video object before closing the page
+            const video = this.recordingPage.video();
+            // Remove recording page/context from our managed lists before closing
+            const pageIndex = this.pages.indexOf(this.recordingPage);
+            if (pageIndex !== -1) {
+                this.pages.splice(pageIndex, 1);
+            }
+            const contextIndex = this.contexts.indexOf(this.recordingContext);
+            if (contextIndex !== -1) {
+                this.contexts.splice(contextIndex, 1);
+            }
+            // Close the page to finalize the video
+            await this.recordingPage.close();
+            // Save the video to the desired output path
+            if (video) {
+                await video.saveAs(outputPath);
+            }
+            // Clean up temp directory
+            if (this.recordingTempDir) {
+                rmSync(this.recordingTempDir, { recursive: true, force: true });
+            }
+            // Close the recording context
+            await this.recordingContext.close();
+            // Reset recording state
+            this.recordingContext = null;
+            this.recordingPage = null;
+            this.recordingOutputPath = '';
+            this.recordingTempDir = '';
+            // Adjust active page index
+            if (this.pages.length > 0) {
+                this.activePageIndex = Math.min(this.activePageIndex, this.pages.length - 1);
+            }
+            else {
+                this.activePageIndex = 0;
+            }
+            // Invalidate CDP session since we may have switched pages
+            await this.invalidateCDPSession();
+            return { path: outputPath, frames: 0 }; // Playwright doesn't expose frame count
+        }
+        catch (error) {
+            // Clean up temp directory on error
+            if (this.recordingTempDir) {
+                rmSync(this.recordingTempDir, { recursive: true, force: true });
+            }
+            // Reset state on error
+            this.recordingContext = null;
+            this.recordingPage = null;
+            this.recordingOutputPath = '';
+            this.recordingTempDir = '';
+            const message = error instanceof Error ? error.message : String(error);
+            return { path: outputPath, frames: 0, error: message };
+        }
+    }
+    /**
+     * Restart recording - stops current recording (if any) and starts a new one.
+     * Convenience method that combines stopRecording and startRecording.
+     *
+     * @param outputPath - Path to the output video file (must be .webm)
+     * @param url - Optional URL to navigate to (defaults to current page URL)
+     * @returns Result from stopping the previous recording (if any)
+     */
+    async restartRecording(outputPath, url) {
+        let previousPath;
+        let stopped = false;
+        // Stop current recording if active
+        if (this.recordingContext) {
+            const result = await this.stopRecording();
+            previousPath = result.path;
+            stopped = true;
+        }
+        // Start new recording
+        await this.startRecording(outputPath, url);
+        return { previousPath, stopped };
+    }
     /**
      * Close the browser and clean up
      */
     async close() {
+        // Stop recording if active (saves video)
+        if (this.recordingContext) {
+            await this.stopRecording();
+        }
         // Stop screencast if active
         if (this.screencastActive) {
             await this.stopScreencast();
@@ -901,8 +1284,20 @@ export class BrowserManager {
             await this.cdpSession.detach().catch(() => { });
             this.cdpSession = null;
         }
-        // CDP: only disconnect, don't close external app's pages
-        if (this.cdpPort !== null) {
+        if (this.browserbaseSessionId && this.browserbaseApiKey) {
+            await this.closeBrowserbaseSession(this.browserbaseSessionId, this.browserbaseApiKey).catch((error) => {
+                console.error('Failed to close Browserbase session:', error);
+            });
+            this.browser = null;
+        }
+        else if (this.browserUseSessionId && this.browserUseApiKey) {
+            await this.closeBrowserUseSession(this.browserUseSessionId, this.browserUseApiKey).catch((error) => {
+                console.error('Failed to close Browser Use session:', error);
+            });
+            this.browser = null;
+        }
+        else if (this.cdpEndpoint !== null) {
+            // CDP: only disconnect, don't close external app's pages
             if (this.browser) {
                 await this.browser.close().catch(() => { });
                 this.browser = null;
@@ -923,7 +1318,11 @@ export class BrowserManager {
         }
         this.pages = [];
         this.contexts = [];
-        this.cdpPort = null;
+        this.cdpEndpoint = null;
+        this.browserbaseSessionId = null;
+        this.browserbaseApiKey = null;
+        this.browserUseSessionId = null;
+        this.browserUseApiKey = null;
         this.isPersistentContext = false;
         this.activePageIndex = 0;
         this.refMap = {};