@mastra/agent-browser 0.0.0

package/dist/index.d.ts

@@ -0,0 +1,654 @@
+import { BrowserConfig as BrowserConfig$1, ThreadManager, ThreadManagerConfig, ThreadSession, MastraBrowser, BrowserToolError, BrowserState, BrowserTabState, ScreencastOptions, ScreencastStream, MouseEventParams, KeyboardEventParams } from '@mastra/core/browser';
+import { Tool } from '@mastra/core/tools';
+import { BrowserManager } from 'agent-browser';
+import { Page } from 'playwright-core';
+import { z } from 'zod';
+
+/**
+ * AgentBrowser Tool Schemas
+ *
+ * Flat schemas for browser tools. Each tool has a single-purpose schema
+ * without discriminated unions, making them easier for LLMs to understand.
+ *
+ * Tools:
+ * - Core: goto, snapshot, click, type, press, select, scroll, close
+ * - Extended: hover, back, dialog, wait, tabs, drag
+ * - Escape Hatch: evaluate
+ */
+
+/**
+ * browser_goto - Navigate to a URL
+ */
+declare const gotoInputSchema: z.ZodObject<{
+    url: z.ZodString;
+    waitUntil: z.ZodOptional<z.ZodEnum<{
+        load: "load";
+        domcontentloaded: "domcontentloaded";
+        networkidle: "networkidle";
+    }>>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+type GotoInput = z.output<typeof gotoInputSchema>;
+/**
+ * browser_snapshot - Get accessibility tree snapshot
+ */
+declare const snapshotInputSchema: z.ZodObject<{
+    interactiveOnly: z.ZodOptional<z.ZodBoolean>;
+    maxDepth: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+type SnapshotInput = z.output<typeof snapshotInputSchema>;
+/**
+ * browser_click - Click an element
+ */
+declare const clickInputSchema: z.ZodObject<{
+    ref: z.ZodString;
+    button: z.ZodOptional<z.ZodEnum<{
+        left: "left";
+        right: "right";
+        middle: "middle";
+    }>>;
+    clickCount: z.ZodOptional<z.ZodNumber>;
+    modifiers: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        Alt: "Alt";
+        Control: "Control";
+        Meta: "Meta";
+        Shift: "Shift";
+    }>>>;
+}, z.core.$strip>;
+type ClickInput = z.output<typeof clickInputSchema>;
+/**
+ * browser_type - Type text into an element
+ */
+declare const typeInputSchema: z.ZodObject<{
+    ref: z.ZodString;
+    text: z.ZodString;
+    clear: z.ZodOptional<z.ZodBoolean>;
+    delay: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+type TypeInput = z.output<typeof typeInputSchema>;
+/**
+ * browser_press - Press a keyboard key
+ */
+declare const pressInputSchema: z.ZodObject<{
+    key: z.ZodString;
+    modifiers: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        Alt: "Alt";
+        Control: "Control";
+        Meta: "Meta";
+        Shift: "Shift";
+    }>>>;
+}, z.core.$strip>;
+type PressInput = z.output<typeof pressInputSchema>;
+/**
+ * browser_select - Select option from dropdown
+ */
+declare const selectInputSchema: z.ZodObject<{
+    ref: z.ZodString;
+    value: z.ZodOptional<z.ZodString>;
+    label: z.ZodOptional<z.ZodString>;
+    index: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+type SelectInput = z.output<typeof selectInputSchema>;
+/**
+ * browser_scroll - Scroll the page or element
+ */
+declare const scrollInputSchema: z.ZodObject<{
+    direction: z.ZodEnum<{
+        left: "left";
+        right: "right";
+        up: "up";
+        down: "down";
+    }>;
+    amount: z.ZodOptional<z.ZodNumber>;
+    ref: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type ScrollInput = z.output<typeof scrollInputSchema>;
+/**
+ * browser_close - Close the browser
+ */
+declare const closeInputSchema: z.ZodObject<{}, z.core.$strip>;
+type CloseInput = z.output<typeof closeInputSchema>;
+/**
+ * browser_hover - Hover over an element
+ */
+declare const hoverInputSchema: z.ZodObject<{
+    ref: z.ZodString;
+}, z.core.$strip>;
+type HoverInput = z.output<typeof hoverInputSchema>;
+/**
+ * browser_back - Go back in browser history
+ */
+declare const backInputSchema: z.ZodObject<{}, z.core.$strip>;
+type BackInput = z.output<typeof backInputSchema>;
+/**
+ * browser_dialog - Click an element that triggers a dialog and handle it
+ */
+declare const dialogInputSchema: z.ZodObject<{
+    triggerRef: z.ZodString;
+    action: z.ZodEnum<{
+        accept: "accept";
+        dismiss: "dismiss";
+    }>;
+    text: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type DialogInput = z.output<typeof dialogInputSchema>;
+/**
+ * browser_wait - Wait for an element or condition
+ */
+declare const waitInputSchema: z.ZodObject<{
+    ref: z.ZodOptional<z.ZodString>;
+    state: z.ZodOptional<z.ZodEnum<{
+        visible: "visible";
+        hidden: "hidden";
+        attached: "attached";
+        detached: "detached";
+    }>>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+type WaitInput = z.output<typeof waitInputSchema>;
+/**
+ * browser_tabs - Manage browser tabs
+ */
+declare const tabsInputSchema: z.ZodObject<{
+    action: z.ZodEnum<{
+        list: "list";
+        new: "new";
+        switch: "switch";
+        close: "close";
+    }>;
+    index: z.ZodOptional<z.ZodNumber>;
+    url: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type TabsInput = z.output<typeof tabsInputSchema>;
+/**
+ * browser_drag - Drag an element to another element
+ */
+declare const dragInputSchema: z.ZodObject<{
+    sourceRef: z.ZodOptional<z.ZodString>;
+    targetRef: z.ZodOptional<z.ZodString>;
+    sourceSelector: z.ZodOptional<z.ZodString>;
+    targetSelector: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type DragInput = z.output<typeof dragInputSchema>;
+/**
+ * browser_evaluate - Execute JavaScript in the browser
+ */
+declare const evaluateInputSchema: z.ZodObject<{
+    script: z.ZodString;
+    arg: z.ZodOptional<z.ZodUnknown>;
+}, z.core.$strip>;
+type EvaluateInput = z.output<typeof evaluateInputSchema>;
+declare const browserSchemas: {
+    readonly goto: z.ZodObject<{
+        url: z.ZodString;
+        waitUntil: z.ZodOptional<z.ZodEnum<{
+            load: "load";
+            domcontentloaded: "domcontentloaded";
+            networkidle: "networkidle";
+        }>>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>;
+    readonly snapshot: z.ZodObject<{
+        interactiveOnly: z.ZodOptional<z.ZodBoolean>;
+        maxDepth: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>;
+    readonly click: z.ZodObject<{
+        ref: z.ZodString;
+        button: z.ZodOptional<z.ZodEnum<{
+            left: "left";
+            right: "right";
+            middle: "middle";
+        }>>;
+        clickCount: z.ZodOptional<z.ZodNumber>;
+        modifiers: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            Alt: "Alt";
+            Control: "Control";
+            Meta: "Meta";
+            Shift: "Shift";
+        }>>>;
+    }, z.core.$strip>;
+    readonly type: z.ZodObject<{
+        ref: z.ZodString;
+        text: z.ZodString;
+        clear: z.ZodOptional<z.ZodBoolean>;
+        delay: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>;
+    readonly press: z.ZodObject<{
+        key: z.ZodString;
+        modifiers: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            Alt: "Alt";
+            Control: "Control";
+            Meta: "Meta";
+            Shift: "Shift";
+        }>>>;
+    }, z.core.$strip>;
+    readonly select: z.ZodObject<{
+        ref: z.ZodString;
+        value: z.ZodOptional<z.ZodString>;
+        label: z.ZodOptional<z.ZodString>;
+        index: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>;
+    readonly scroll: z.ZodObject<{
+        direction: z.ZodEnum<{
+            left: "left";
+            right: "right";
+            up: "up";
+            down: "down";
+        }>;
+        amount: z.ZodOptional<z.ZodNumber>;
+        ref: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+    readonly close: z.ZodObject<{}, z.core.$strip>;
+    readonly hover: z.ZodObject<{
+        ref: z.ZodString;
+    }, z.core.$strip>;
+    readonly back: z.ZodObject<{}, z.core.$strip>;
+    readonly dialog: z.ZodObject<{
+        triggerRef: z.ZodString;
+        action: z.ZodEnum<{
+            accept: "accept";
+            dismiss: "dismiss";
+        }>;
+        text: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+    readonly wait: z.ZodObject<{
+        ref: z.ZodOptional<z.ZodString>;
+        state: z.ZodOptional<z.ZodEnum<{
+            visible: "visible";
+            hidden: "hidden";
+            attached: "attached";
+            detached: "detached";
+        }>>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>;
+    readonly tabs: z.ZodObject<{
+        action: z.ZodEnum<{
+            list: "list";
+            new: "new";
+            switch: "switch";
+            close: "close";
+        }>;
+        index: z.ZodOptional<z.ZodNumber>;
+        url: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+    readonly drag: z.ZodObject<{
+        sourceRef: z.ZodOptional<z.ZodString>;
+        targetRef: z.ZodOptional<z.ZodString>;
+        sourceSelector: z.ZodOptional<z.ZodString>;
+        targetSelector: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+    readonly evaluate: z.ZodObject<{
+        script: z.ZodString;
+        arg: z.ZodOptional<z.ZodUnknown>;
+    }, z.core.$strip>;
+};
+
+/**
+ * Configuration options for AgentBrowser.
+ * Type alias for BaseBrowserConfig.
+ */
+type BrowserConfig = BrowserConfig$1;
+
+/**
+ * AgentBrowserThreadManager - Thread isolation for AgentBrowser
+ *
+ * Manages thread-scoped browser sessions using agent-browser's
+ * BrowserManager capabilities (newWindow, switchTo, closeTab).
+ */
+
+/**
+ * Extended session info for AgentBrowser.
+ */
+interface AgentBrowserSession extends ThreadSession {
+    /** For 'thread' scope: dedicated browser manager instance */
+    manager?: BrowserManager;
+}
+/**
+ * Configuration for AgentBrowserThreadManager.
+ */
+interface AgentBrowserThreadManagerConfig extends ThreadManagerConfig {
+    /** Browser configuration for launching new instances */
+    browserConfig: BrowserConfig;
+    /** Function to resolve CDP URL (may be async) */
+    resolveCdpUrl?: (cdpUrl: string | (() => string | Promise<string>)) => Promise<string>;
+    /** Callback when a new browser manager is created for a thread */
+    onBrowserCreated?: (manager: BrowserManager, threadId: string) => void;
+}
+/**
+ * Thread manager implementation for AgentBrowser.
+ *
+ * Supports two scope modes:
+ * - 'shared': All threads share the shared browser manager
+ * - 'thread': Each thread gets a dedicated browser manager instance
+ */
+declare class AgentBrowserThreadManager extends ThreadManager<BrowserManager> {
+    private sharedManager;
+    private readonly browserConfig;
+    private readonly resolveCdpUrl?;
+    private readonly onBrowserCreated?;
+    /** Map of thread ID to dedicated browser manager (for 'thread' scope) */
+    private readonly threadBrowsers;
+    constructor(config: AgentBrowserThreadManagerConfig);
+    /**
+     * Set the shared browser manager (called after browser launch).
+     */
+    setSharedManager(manager: BrowserManager): void;
+    /**
+     * Clear the shared browser manager (called when browser disconnects).
+     */
+    clearSharedManager(): void;
+    /**
+     * Get the shared browser manager.
+     */
+    protected getSharedManager(): BrowserManager;
+    /**
+     * Create a new session for a thread.
+     */
+    protected createSession(threadId: string): Promise<AgentBrowserSession>;
+    /**
+     * Restore browser state (multiple tabs) to a browser manager.
+     */
+    private restoreBrowserState;
+    /**
+     * Switch to an existing session.
+     * For 'thread' scope, no switching needed - each thread has its own manager.
+     * For 'shared' scope, nothing to switch.
+     */
+    protected switchToSession(_session: AgentBrowserSession): Promise<void>;
+    /**
+     * Get the browser manager for a specific session.
+     */
+    protected getManagerForSession(session: AgentBrowserSession): BrowserManager;
+    /**
+     * Destroy a session and clean up resources.
+     */
+    protected doDestroySession(session: AgentBrowserSession): Promise<void>;
+    /**
+     * Destroy all sessions (called during browser close).
+     */
+    destroyAllSessions(): Promise<void>;
+    /**
+     * Check if any thread browsers are still running.
+     */
+    hasActiveThreadBrowsers(): boolean;
+    /**
+     * Get the browser manager for an existing thread session without creating a new one.
+     * Returns null if no session exists for the thread.
+     */
+    getExistingManagerForThread(threadId: string): BrowserManager | null;
+    /**
+     * Clear all session tracking without closing browsers.
+     * Used when browsers have been externally closed and we just need to reset state.
+     */
+    clearAllSessions(): void;
+    /**
+     * Clear a specific thread's session without closing the browser.
+     * Used when a thread's browser has been externally closed.
+     * Preserves the browser state for potential restoration.
+     * @param threadId - The thread ID to clear
+     */
+    clearSession(threadId: string): void;
+}
+
+/**
+ * AgentBrowser - Browser automation using agent-browser (vercel-labs/agent-browser)
+ *
+ * Uses snapshot + refs pattern for LLM-friendly element targeting.
+ */
+declare class AgentBrowser extends MastraBrowser {
+    readonly id: string;
+    readonly name = "AgentBrowser";
+    readonly provider = "vercel-labs/agent-browser";
+    /** Primary browser manager (for 'none' mode, also used as fallback) */
+    private browserManager;
+    private defaultTimeout;
+    /** Active screencast streams per thread (for triggering reconnects on tab changes) */
+    private activeScreencastStreams;
+    /** Default key for shared scope */
+    private static readonly SHARED_STREAM_KEY;
+    /** Thread manager - narrowed type from base class */
+    protected threadManager: AgentBrowserThreadManager;
+    constructor(config?: BrowserConfig);
+    /**
+     * Ensure browser is ready and thread session exists.
+     * Creates a new page/context for the current thread if needed.
+     *
+     * For 'browser' isolation, we need to create the thread session BEFORE
+     * calling super.ensureReady() because the base class's ensureReady() will
+     * call checkBrowserAlive(), which needs at least one thread browser to exist.
+     */
+    ensureReady(): Promise<void>;
+    /**
+     * Get the browser manager for the current thread.
+     * Delegates to ThreadManager for isolation handling.
+     */
+    getManagerForThread(threadId?: string): Promise<BrowserManager>;
+    /**
+     * Get the page for a specific thread.
+     * For thread-isolated modes, ensures we're on the correct context/page.
+     */
+    getPageForThread(threadId?: string): Promise<Page>;
+    /**
+     * Close a specific thread's browser session.
+     * Delegates to ThreadManager and notifies registered callbacks.
+     */
+    closeThreadSession(threadId: string): Promise<void>;
+    protected doLaunch(): Promise<void>;
+    /**
+     * Set up close event listeners for 'none' isolation shared browser.
+     * This handles the case where the shared browser is closed externally.
+     */
+    private setupCloseListenerForNoneIsolation;
+    protected doClose(): Promise<void>;
+    /**
+     * Check if the browser is still alive by verifying the page is connected.
+     * Called by base class ensureReady() to detect externally closed browsers.
+     */
+    protected checkBrowserAlive(): Promise<boolean>;
+    /**
+     * Get the browser tools for this provider.
+     * Returns 17 flat tools for browser automation.
+     */
+    getTools(): Record<string, Tool<any, any>>;
+    /**
+     * Get the page for the current thread.
+     * Uses thread scope if enabled, otherwise returns the shared page.
+     * @param explicitThreadId - Optional thread ID to use instead of getCurrentThread()
+     *                           Use this to avoid race conditions in concurrent tool calls.
+     */
+    private getPage;
+    /**
+     * Handle browser disconnection by clearing internal state.
+     * For 'thread' scope, only notifies the specific thread's callbacks.
+     * For 'shared' scope, notifies all callbacks.
+     */
+    handleBrowserDisconnected(): void;
+    /**
+     * Set up close event listener for a thread's browser manager.
+     * This handles the case where a thread's browser is closed externally.
+     */
+    private setupCloseListenerForThread;
+    /**
+     * Handle browser disconnection for a specific thread.
+     * Called when a thread's browser is closed externally.
+     */
+    private handleThreadBrowserDisconnected;
+    /**
+     * Create an error response from an exception.
+     * Extends base class to add agent-browser specific error handling.
+     */
+    protected createErrorFromException(error: unknown, context: string): BrowserToolError;
+    private requireLocator;
+    private getScrollInfo;
+    /**
+     * Get the current page URL without launching the browser.
+     * @param threadId - Optional thread ID for thread-isolated browsers
+     * @returns The current URL string, or null if browser is not running
+     */
+    getCurrentUrl(threadId?: string): Promise<string | null>;
+    /**
+     * Navigate to a URL (simple form). Used internally for restoring state on relaunch.
+     */
+    navigateTo(url: string): Promise<void>;
+    /**
+     * Get the current browser state (all tabs and active tab index).
+     */
+    getBrowserState(threadId?: string): Promise<BrowserState | null>;
+    /**
+     * Get browser state from a specific manager instance.
+     */
+    private getBrowserStateForManager;
+    /**
+     * Get all open tabs with their URLs and titles.
+     */
+    getTabState(threadId?: string): Promise<BrowserTabState[]>;
+    /**
+     * Get the active tab index.
+     */
+    getActiveTabIndex(threadId?: string): Promise<number>;
+    /**
+     * Update the browser state in the thread session.
+     * Called on navigation, tab open/close to keep state fresh.
+     */
+    private updateSessionBrowserState;
+    goto(input: GotoInput, threadId?: string): Promise<{
+        success: true;
+        url: string;
+        title: string;
+        hint: string;
+    } | BrowserToolError>;
+    snapshot(input: SnapshotInput, threadId?: string): Promise<{
+        success: true;
+        snapshot: string;
+        url: string;
+        title: string;
+        elementCount: number;
+        scroll: string;
+        hint?: string;
+    } | BrowserToolError>;
+    click(input: ClickInput, threadId?: string): Promise<{
+        success: true;
+        url: string;
+        hint: string;
+    } | BrowserToolError>;
+    type(input: TypeInput, threadId?: string): Promise<{
+        success: true;
+        value: string;
+        url: string;
+        hint: string;
+    } | BrowserToolError>;
+    press(input: PressInput, threadId?: string): Promise<{
+        success: true;
+        url: string;
+        hint: string;
+    } | BrowserToolError>;
+    select(input: SelectInput, threadId?: string): Promise<{
+        success: true;
+        selected: string[];
+        url: string;
+        hint: string;
+    } | BrowserToolError>;
+    scroll(input: ScrollInput, threadId?: string): Promise<{
+        success: true;
+        position: {
+            x: number;
+            y: number;
+        };
+        scroll: string;
+        hint: string;
+    } | BrowserToolError>;
+    hover(input: HoverInput, threadId?: string): Promise<{
+        success: true;
+        url: string;
+        hint: string;
+    } | BrowserToolError>;
+    back(threadId?: string): Promise<{
+        success: true;
+        url: string;
+        title: string;
+        hint: string;
+    } | BrowserToolError>;
+    dialog(input: DialogInput, threadId?: string): Promise<{
+        success: true;
+        action: 'accept' | 'dismiss';
+        dialogType: string;
+        message: string;
+        hint: string;
+    } | BrowserToolError>;
+    wait(input: WaitInput, threadId?: string): Promise<{
+        success: true;
+        hint: string;
+    } | BrowserToolError>;
+    tabs(input: TabsInput, threadId?: string): Promise<{
+        success: true;
+        tabs?: unknown[];
+        index?: number;
+        url?: string;
+        title?: string;
+        remaining?: number;
+        hint: string;
+    } | BrowserToolError>;
+    drag(input: DragInput, threadId?: string): Promise<{
+        success: true;
+        url: string;
+        hint: string;
+    } | BrowserToolError>;
+    evaluate(input: EvaluateInput, threadId?: string): Promise<{
+        success: true;
+        result: unknown;
+        hint: string;
+    } | BrowserToolError>;
+    closeBrowser(): Promise<{
+        success: true;
+        hint: string;
+    } | BrowserToolError>;
+    /**
+     * Get the stream key for a thread (or shared key for shared scope).
+     */
+    private getStreamKey;
+    /**
+     * Trigger a screencast reconnect after tab changes.
+     * Called internally when tabs are switched or closed.
+     */
+    private reconnectScreencast;
+    startScreencast(_options?: ScreencastOptions): Promise<ScreencastStream>;
+    injectMouseEvent(event: MouseEventParams, threadId?: string): Promise<void>;
+    injectKeyboardEvent(event: KeyboardEventParams, threadId?: string): Promise<void>;
+}
+
+/**
+ * Browser Tool Constants
+ */
+declare const BROWSER_TOOLS: {
+    readonly GOTO: "browser_goto";
+    readonly SNAPSHOT: "browser_snapshot";
+    readonly CLICK: "browser_click";
+    readonly TYPE: "browser_type";
+    readonly PRESS: "browser_press";
+    readonly SELECT: "browser_select";
+    readonly SCROLL: "browser_scroll";
+    readonly CLOSE: "browser_close";
+    readonly HOVER: "browser_hover";
+    readonly BACK: "browser_back";
+    readonly DIALOG: "browser_dialog";
+    readonly WAIT: "browser_wait";
+    readonly TABS: "browser_tabs";
+    readonly DRAG: "browser_drag";
+    readonly EVALUATE: "browser_evaluate";
+};
+type BrowserToolName = (typeof BROWSER_TOOLS)[keyof typeof BROWSER_TOOLS];
+
+/**
+ * AgentBrowser Tools
+ *
+ * Creates browser tools bound to an AgentBrowser instance.
+ * Each tool is defined in its own file for maintainability.
+ */
+
+/**
+ * Creates all browser tools bound to an AgentBrowser instance.
+ * The browser is lazily initialized on first tool use.
+ */
+declare function createAgentBrowserTools(browser: AgentBrowser): Record<string, Tool<any, any>>;
+
+export { AgentBrowser, BROWSER_TOOLS, type BackInput, type BrowserConfig, type BrowserToolName, type ClickInput, type CloseInput, type DialogInput, type DragInput, type EvaluateInput, type GotoInput, type HoverInput, type PressInput, type ScrollInput, type SelectInput, type SnapshotInput, type TabsInput, type TypeInput, type WaitInput, backInputSchema, browserSchemas, clickInputSchema, closeInputSchema, createAgentBrowserTools, dialogInputSchema, dragInputSchema, evaluateInputSchema, gotoInputSchema, hoverInputSchema, pressInputSchema, scrollInputSchema, selectInputSchema, snapshotInputSchema, tabsInputSchema, typeInputSchema, waitInputSchema };