@browserbasehq/orca 3.0.0-preview.0

package/dist/lib/v3/types/stagehand.d.ts

@@ -0,0 +1,113 @@
+import { Client } from "@modelcontextprotocol/sdk/dist/esm/client";
+import { ToolSet } from "ai";
+import { z } from "zod/v3";
+import { AgentProviderType } from "./agent";
+import { ModelConfiguration } from "./model";
+export interface ActOptions {
+    action: string;
+    model?: ModelConfiguration;
+    variables?: Record<string, string>;
+    domSettleTimeoutMs?: number;
+    timeoutMs?: number;
+    iframes?: boolean;
+    frameId?: string;
+}
+export interface ActResult {
+    success: boolean;
+    message: string;
+    actionDescription: string;
+    actions: Action[];
+}
+export interface ExtractOptions<T extends z.AnyZodObject> {
+    instruction?: string;
+    schema?: T;
+    model?: ModelConfiguration;
+    domSettleTimeoutMs?: number;
+    /**
+     * @deprecated The `useTextExtract` parameter has no effect in this version of Stagehand and will be removed in later versions.
+     */
+    useTextExtract?: boolean;
+    selector?: string;
+    iframes?: boolean;
+    frameId?: string;
+}
+export type ExtractResult<T extends z.AnyZodObject> = z.infer<T>;
+export interface ObserveOptions {
+    instruction?: string;
+    model?: ModelConfiguration;
+    domSettleTimeoutMs?: number;
+    returnAction?: boolean;
+    selector?: string;
+    /**
+     * @deprecated The `onlyVisible` parameter has no effect in this version of Stagehand and will be removed in later versions.
+     */
+    onlyVisible?: boolean;
+    drawOverlay?: boolean;
+    iframes?: boolean;
+    frameId?: string;
+}
+export interface Action {
+    selector: string;
+    description: string;
+    backendNodeId?: number;
+    method?: string;
+    arguments?: string[];
+}
+/**
+ * Configuration for agent functionality
+ */
+export interface AgentConfig {
+    /**
+     * The provider to use for agent functionality
+     */
+    provider?: AgentProviderType;
+    /**
+     * The model to use for agent functionality
+     */
+    model?: string;
+    /**
+     * The model to use for tool execution (observe/act calls within agent tools).
+     * If not specified, inherits from the main model configuration.
+     * Format: "provider/model" (e.g., "openai/gpt-4o-mini", "google/gemini-2.0-flash-exp")
+     */
+    executionModel?: string;
+    /**
+     * Custom instructions to provide to the agent
+     */
+    instructions?: string;
+    /**
+     * Additional options to pass to the agent client
+     */
+    options?: Record<string, unknown>;
+    /**
+     * MCP integrations - Array of Client objects
+     */
+    integrations?: (Client | string)[];
+    /**
+     * Tools passed to the agent client
+     */
+    tools?: ToolSet;
+}
+export interface HistoryEntry {
+    method: "act" | "extract" | "observe" | "navigate";
+    parameters: unknown;
+    result: unknown;
+    timestamp: string;
+}
+/**
+ * Represents a path through a Zod schema from the root object down to a
+ * particular field. The `segments` array describes the chain of keys/indices.
+ *
+ * - **String** segments indicate object property names.
+ * - **Number** segments indicate array indices.
+ *
+ * For example, `["users", 0, "homepage"]` might describe reaching
+ * the `homepage` field in `schema.users[0].homepage`.
+ */
+export interface ZodPathSegments {
+    /**
+     * The ordered list of keys/indices leading from the schema root
+     * to the targeted field.
+     */
+    segments: Array<string | number>;
+}

package/dist/lib/v3/types/stagehandApiErrors.d.ts

@@ -0,0 +1,18 @@
+export declare class StagehandAPIError extends Error {
+    constructor(message: string);
+}
+export declare class StagehandAPIUnauthorizedError extends StagehandAPIError {
+    constructor(message?: string);
+}
+export declare class StagehandHttpError extends StagehandAPIError {
+    constructor(message: string);
+}
+export declare class StagehandServerError extends StagehandAPIError {
+    constructor(message: string);
+}
+export declare class StagehandResponseBodyError extends StagehandAPIError {
+    constructor();
+}
+export declare class StagehandResponseParseError extends StagehandAPIError {
+    constructor(message: string);
+}

package/dist/lib/v3/types/stagehandErrors.d.ts

@@ -0,0 +1,104 @@
+import { ZodError } from "zod/v3";
+export declare class StagehandError extends Error {
+    constructor(message: string);
+}
+export declare class StagehandDefaultError extends StagehandError {
+    constructor(error?: unknown);
+}
+export declare class StagehandEnvironmentError extends StagehandError {
+    constructor(currentEnvironment: string, requiredEnvironment: string, feature: string);
+}
+export declare class MissingEnvironmentVariableError extends StagehandError {
+    constructor(missingEnvironmentVariable: string, feature: string);
+}
+export declare class UnsupportedModelError extends StagehandError {
+    constructor(supportedModels: string[], feature?: string);
+}
+export declare class UnsupportedModelProviderError extends StagehandError {
+    constructor(supportedProviders: string[], feature?: string);
+}
+export declare class UnsupportedAISDKModelProviderError extends StagehandError {
+    constructor(provider: string, supportedProviders: string[]);
+}
+export declare class InvalidAISDKModelFormatError extends StagehandError {
+    constructor(modelName: string);
+}
+export declare class StagehandNotInitializedError extends StagehandError {
+    constructor(prop: string);
+}
+export declare class BrowserbaseSessionNotFoundError extends StagehandError {
+    constructor();
+}
+export declare class CaptchaTimeoutError extends StagehandError {
+    constructor();
+}
+export declare class MissingLLMConfigurationError extends StagehandError {
+    constructor();
+}
+export declare class HandlerNotInitializedError extends StagehandError {
+    constructor(handlerType: string);
+}
+export declare class StagehandInvalidArgumentError extends StagehandError {
+    constructor(message: string);
+}
+export declare class StagehandElementNotFoundError extends StagehandError {
+    constructor(xpaths: string[]);
+}
+export declare class AgentScreenshotProviderError extends StagehandError {
+    constructor(message: string);
+}
+export declare class StagehandMissingArgumentError extends StagehandError {
+    constructor(message: string);
+}
+export declare class CreateChatCompletionResponseError extends StagehandError {
+    constructor(message: string);
+}
+export declare class StagehandEvalError extends StagehandError {
+    constructor(message: string);
+}
+export declare class StagehandDomProcessError extends StagehandError {
+    constructor(message: string);
+}
+export declare class StagehandClickError extends StagehandError {
+    constructor(message: string, selector: string);
+}
+export declare class LLMResponseError extends StagehandError {
+    constructor(primitive: string, message: string);
+}
+export declare class StagehandIframeError extends StagehandError {
+    constructor(frameUrl: string, message: string);
+}
+export declare class ContentFrameNotFoundError extends StagehandError {
+    constructor(selector: string);
+}
+export declare class XPathResolutionError extends StagehandError {
+    constructor(xpath: string);
+}
+export declare class ExperimentalApiConflictError extends StagehandError {
+    constructor();
+}
+export declare class ExperimentalNotConfiguredError extends StagehandError {
+    constructor(featureName: string);
+}
+export declare class ZodSchemaValidationError extends Error {
+    readonly received: unknown;
+    readonly issues: ReturnType<ZodError["format"]>;
+    constructor(received: unknown, issues: ReturnType<ZodError["format"]>);
+}
+export declare class StagehandInitError extends StagehandError {
+    constructor(message: string);
+}
+export declare class MCPConnectionError extends StagehandError {
+    readonly serverUrl: string;
+    readonly originalError: unknown;
+    constructor(serverUrl: string, originalError: unknown);
+}
+export declare class StagehandShadowRootMissingError extends StagehandError {
+    constructor(detail?: string);
+}
+export declare class StagehandShadowSegmentEmptyError extends StagehandError {
+    constructor();
+}
+export declare class StagehandShadowSegmentNotFoundError extends StagehandError {
+    constructor(segment: string, hint?: string);
+}

package/dist/lib/v3/types.d.ts

@@ -0,0 +1,176 @@
+import { LaunchedChrome } from "chrome-launcher";
+import Browserbase from "@browserbasehq/sdk";
+import { Page } from "./understudy/page";
+import { ModelConfiguration } from "../v3/types/model";
+import { LLMClient } from "./llm/LLMClient";
+import { z } from "zod/v3";
+import type { ZodTypeAny } from "zod/v3";
+import type { LogLine } from "../v3/types/log";
+export type V3Env = "LOCAL" | "BROWSERBASE";
+/** Local launch options for V3 (chrome-launcher + CDP).
+ * Matches v2 shape where feasible; unsupported fields are accepted but ignored.
+ */
+export interface LocalBrowserLaunchOptions {
+    args?: string[];
+    executablePath?: string;
+    userDataDir?: string;
+    preserveUserDataDir?: boolean;
+    headless?: boolean;
+    devtools?: boolean;
+    chromiumSandbox?: boolean;
+    ignoreDefaultArgs?: boolean | string[];
+    proxy?: {
+        server: string;
+        bypass?: string;
+        username?: string;
+        password?: string;
+    };
+    locale?: string;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    deviceScaleFactor?: number;
+    hasTouch?: boolean;
+    ignoreHTTPSErrors?: boolean;
+    cdpUrl?: string;
+    connectTimeoutMs?: number;
+    downloadsPath?: string;
+    acceptDownloads?: boolean;
+}
+/** Constructor options for V3 */
+export interface V3Options {
+    env: V3Env;
+    apiKey?: string;
+    projectId?: string;
+    /**
+     * Optional: fine-tune Browserbase session creation or resume an existing session.
+     */
+    browserbaseSessionCreateParams?: Omit<Browserbase.Sessions.SessionCreateParams, "projectId"> & {
+        projectId?: string;
+    };
+    browserbaseSessionID?: string;
+    localBrowserLaunchOptions?: LocalBrowserLaunchOptions;
+    model?: ModelConfiguration;
+    llmClient?: LLMClient;
+    systemPrompt?: string;
+    logInferenceToFile?: boolean;
+    experimental?: boolean;
+    verbose?: 0 | 1 | 2;
+    selfHeal?: boolean;
+    /** Disable pino logging backend (useful for tests or minimal environments). */
+    disablePino?: boolean;
+    /** Optional external logger hook for integrating with host apps. */
+    logger?: (line: LogLine) => void;
+    /** Show a visual cursor overlay that follows our mouse events. */
+    includeCursor?: boolean;
+    /** Directory used to persist cached actions for act(). */
+    cacheDir?: string;
+    domSettleTimeout?: number;
+}
+/** Narrow shape we rely on from Browserbase session creation */
+export interface BrowserbaseSession {
+    id: string;
+    connectUrl: string;
+}
+/** Narrow shape we rely on from /json/version */
+export interface JsonVersionResponse {
+    webSocketDebuggerUrl: string;
+}
+export type InitState = {
+    kind: "UNINITIALIZED";
+} | {
+    kind: "LOCAL";
+    chrome: LaunchedChrome;
+    ws: string;
+    userDataDir?: string;
+    createdTempProfile?: boolean;
+    preserveUserDataDir?: boolean;
+} | {
+    kind: "BROWSERBASE";
+    bb: Browserbase;
+    sessionId: string;
+    ws: string;
+};
+export type PlaywrightPage = import("playwright-core").Page;
+export type PatchrightPage = import("patchright-core").Page;
+export type PuppeteerPage = import("puppeteer-core").Page;
+export type AnyPage = PlaywrightPage | PuppeteerPage | PatchrightPage | Page;
+export interface ActOptions {
+    model?: ModelConfiguration;
+    variables?: Record<string, string>;
+    timeout?: number;
+    page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page;
+}
+export interface ActHandlerParams {
+    instruction: string;
+    model?: ModelConfiguration;
+    variables?: Record<string, string>;
+    timeout?: number;
+    page: Page;
+}
+export interface ExtractOptions {
+    model?: ModelConfiguration;
+    timeout?: number;
+    selector?: string;
+    page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page;
+}
+export interface ExtractHandlerParams<T extends ZodTypeAny> {
+    instruction?: string;
+    schema?: T;
+    model?: ModelConfiguration;
+    timeout?: number;
+    selector?: string;
+    page: Page;
+}
+export declare const defaultExtractSchema: z.ZodObject<{
+    extraction: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    extraction?: string;
+}, {
+    extraction?: string;
+}>;
+export declare const pageTextSchema: z.ZodObject<{
+    pageText: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    pageText?: string;
+}, {
+    pageText?: string;
+}>;
+export interface ObserveOptions {
+    model?: ModelConfiguration;
+    timeout?: number;
+    selector?: string;
+    page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page;
+}
+export interface ObserveHandlerParams {
+    instruction?: string;
+    model?: ModelConfiguration;
+    timeout?: number;
+    selector?: string;
+    page: Page;
+}
+export type LoadState = "load" | "domcontentloaded" | "networkidle";
+export interface V3Metrics {
+    actPromptTokens: number;
+    actCompletionTokens: number;
+    actInferenceTimeMs: number;
+    extractPromptTokens: number;
+    extractCompletionTokens: number;
+    extractInferenceTimeMs: number;
+    observePromptTokens: number;
+    observeCompletionTokens: number;
+    observeInferenceTimeMs: number;
+    agentPromptTokens: number;
+    agentCompletionTokens: number;
+    agentInferenceTimeMs: number;
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    totalInferenceTimeMs: number;
+}
+export declare enum V3FunctionName {
+    ACT = "ACT",
+    EXTRACT = "EXTRACT",
+    OBSERVE = "OBSERVE",
+    AGENT = "AGENT"
+}

package/dist/lib/v3/understudy/a11y/snapshot.d.ts

@@ -0,0 +1,71 @@
+import { Page } from "../page";
+/**
+ * a11y/snapshot
+ *
+ * Purpose:
+ * Build a **hybrid DOM + Accessibility** snapshot for a V3 `Page`, suitable for
+ * act/extract/observe handlers. The snapshot includes:
+ *  - A merged, human-readable outline (Accessibility tree) for the page
+ *  - EncodedId → XPath map (DOM)
+ *  - EncodedId → URL map (from AX properties)
+ *
+ * Design highlights:
+ *  - EncodedId is frame-aware and compact: `${frameOrdinal}-${backendNodeId}` where
+ *    the frame ordinal is provided by `Page.getOrdinal(frameId)`.
+ *  - Each frame (main, same-process iframe, OOPIF) is processed against **its owning session**,
+ *    so DOM and A11y ownership are correct for that document.
+ *  - We compute an **absolute iframe XPath prefix** for each child frame by asking
+ *    the **parent session** for the `<iframe>` owner node (via `DOM.getFrameOwner`).
+ *  - No global inventory imports; we rely solely on Page’s registry-backed APIs.
+ */
+export type SnapshotOptions = {
+    /**
+     * Filter the snapshot to a specific element/subtree using a selector that can cross iframes.
+     * Supports:
+     *  - XPath: strings starting with 'xpath=' or '/'
+     *  - CSS with iframe hops via '>>': e.g., 'section iframe >> #email'
+     */
+    focusSelector?: string;
+    /** Pierce shadow DOM in DOM.getDocument (default: true). */
+    pierceShadow?: boolean;
+    /** Experimental behaviours flag. */
+    experimental?: boolean;
+};
+export type HybridSnapshot = {
+    /** Merged/stitched outline across frames. */
+    combinedTree: string;
+    /** EncodedId (ordinal-backendId) → absolute XPath (across iframes). */
+    combinedXpathMap: Record<string, string>;
+    /** EncodedId → URL (from AX properties). */
+    combinedUrlMap: Record<string, string>;
+    /** Per-frame details for debugging/consumers. */
+    perFrame?: Array<{
+        frameId: string;
+        outline: string;
+        xpathMap: Record<string, string>;
+        urlMap: Record<string, string>;
+    }>;
+};
+/**
+ * Resolve deepest node for a page coordinate and compute its absolute XPath across frames.
+ * More efficient than building a full hybrid snapshot when only a single node’s XPath is needed.
+ */
+export declare function resolveXpathForLocation(page: Page, x: number, y: number): Promise<{
+    frameId: string;
+    backendNodeId: number;
+    absoluteXPath: string;
+} | null>;
+/**
+ * Compute the absolute XPath for the currently focused element.
+ * - Detects which frame has focus via document.hasFocus().
+ * - Finds the deepest activeElement (dives into shadow DOM).
+ * - Builds an absolute, cross-frame XPath by prefixing iframe hosts.
+ */
+export declare function computeActiveElementXpath(page: Page): Promise<string | null>;
+export declare function captureHybridSnapshot(page: Page, options?: SnapshotOptions): Promise<HybridSnapshot>;
+/**
+ * Return the lines that appear in `nextTree` but not in `prevTree`.
+ * Comparison is done line-by-line, ignoring leading whitespace in both trees.
+ * The returned block is re-indented so the minimal indent becomes column 0.
+ */
+export declare function diffCombinedTrees(prevTree: string, nextTree: string): string;

package/dist/lib/v3/understudy/cdp.d.ts

@@ -0,0 +1,58 @@
+/**
+ * CDP transport & session multiplexer
+ *
+ * Owns the browser WebSocket and multiplexes flattened Target sessions.
+ * Tracks inflight CDP calls, routes responses to the right session, and forwards events.
+ *
+ * This does not interpret Page/DOM/Runtime semantics — callers own that logic.
+ */
+export interface CDPSessionLike {
+    send<R = unknown>(method: string, params?: object): Promise<R>;
+    on<P = unknown>(event: string, handler: (params: P) => void): void;
+    off<P = unknown>(event: string, handler: (params: P) => void): void;
+    close(): Promise<void>;
+    readonly id: string | null;
+}
+type EventHandler = (params: unknown) => void;
+export declare class CdpConnection implements CDPSessionLike {
+    private ws;
+    private nextId;
+    private inflight;
+    private eventHandlers;
+    private sessions;
+    readonly id: string | null;
+    private transportCloseHandlers;
+    onTransportClosed(handler: (why: string) => void): void;
+    offTransportClosed(handler: (why: string) => void): void;
+    private emitTransportClosed;
+    private constructor();
+    static connect(wsUrl: string): Promise<CdpConnection>;
+    enableAutoAttach(): Promise<void>;
+    send<R = unknown>(method: string, params?: object): Promise<R>;
+    on<P = unknown>(event: string, handler: (params: P) => void): void;
+    off<P = unknown>(event: string, handler: (params: P) => void): void;
+    close(): Promise<void>;
+    getSession(sessionId: string): CdpSession | undefined;
+    attachToTarget(targetId: string): Promise<CdpSession>;
+    getTargets(): Promise<Array<{
+        targetId: string;
+        type: string;
+        url: string;
+    }>>;
+    private onMessage;
+    _sendViaSession<R = unknown>(sessionId: string, method: string, params?: object): Promise<R>;
+    _onSessionEvent(sessionId: string, event: string, handler: EventHandler): void;
+    _offSessionEvent(sessionId: string, event: string, handler: EventHandler): void;
+    _dispatchToSession(sessionId: string, event: string, params: unknown): void;
+}
+export declare class CdpSession implements CDPSessionLike {
+    private readonly root;
+    readonly id: string;
+    constructor(root: CdpConnection, id: string);
+    send<R = unknown>(method: string, params?: object): Promise<R>;
+    on<P = unknown>(event: string, handler: (params: P) => void): void;
+    off<P = unknown>(event: string, handler: (params: P) => void): void;
+    close(): Promise<void>;
+    dispatch(event: string, params: unknown): void;
+}
+export {};

package/dist/lib/v3/understudy/context.d.ts

@@ -0,0 +1,120 @@
+import type { Protocol } from "devtools-protocol";
+import { CdpConnection } from "./cdp";
+import { Page } from "./page";
+/**
+ * V3Context
+ *
+ * Owns the root CDP connection and wires Target/Page events into Page.
+ * Maintains one Page per top-level target, adopts OOPIF child sessions into the owner Page,
+ * and tracks target→page and (root) frame→target mappings for lookups.
+ *
+ * IMPORTANT: FrameId → session ownership is managed inside Page (via its FrameRegistry).
+ * Context never “guesses” owners; it simply forwards events (with the emitting session)
+ * so Page can record the correct owner at event time.
+ */
+export declare class V3Context {
+    readonly conn: CdpConnection;
+    private readonly includeCursor;
+    private readonly env;
+    private constructor();
+    private readonly _piercerInstalled;
+    private _lastPopupSignalAt;
+    private sessionKey;
+    private readonly _sessionInit;
+    private pagesByTarget;
+    private mainFrameToTarget;
+    private sessionOwnerPage;
+    private frameOwnerPage;
+    private pendingOopifByMainFrame;
+    private createdAtByTarget;
+    private typeByTarget;
+    private _pageOrder;
+    /**
+     * Create a Context for a given CDP websocket URL and bootstrap target wiring.
+     */
+    static create(wsUrl: string, opts?: {
+        includeCursor?: boolean;
+        env?: "LOCAL" | "BROWSERBASE";
+    }): Promise<V3Context>;
+    /**
+     * Wait until at least one top-level Page has been created and registered.
+     * We poll internal maps that bootstrap/onAttachedToTarget populate.
+     */
+    private waitForFirstTopLevelPage;
+    private ensurePiercer;
+    /** Mark a page target as the most-recent one (active). */
+    private _pushActive;
+    /** Remove a page target from the recency list (used on close). */
+    private _removeFromOrder;
+    /** Return the current active Page (most-recent page that still exists). */
+    activePage(): Page | undefined;
+    /**
+     * Return top-level `Page`s (oldest → newest). OOPIF targets are not included.
+     */
+    pages(): Page[];
+    /**
+     * Resolve an owning `Page` by the **top-level main frame id**.
+     * Note: child (OOPIF) roots are intentionally not present in this mapping.
+     */
+    resolvePageByMainFrameId(frameId: string): Page | undefined;
+    /**
+     * Serialize the full frame tree for a given top-level main frame id.
+     */
+    getFullFrameTreeByMainFrameId(rootMainFrameId: string): Promise<Protocol.Page.FrameTree>;
+    /**
+     * Create a new top-level page (tab) with the given URL and return its Page object.
+     * Waits until the target is attached and registered.
+     */
+    newPage(url?: string): Promise<Page>;
+    /**
+     * Close CDP and clear all mappings. Best-effort cleanup.
+     */
+    close(): Promise<void>;
+    /**
+     * Bootstrap target lifecycle:
+     * - Attach to existing targets.
+     * - Attach on `Target.targetCreated` (fallback for OOPIFs).
+     * - Handle auto-attach events.
+     * - Clean up on detach/destroy.
+     */
+    private bootstrap;
+    /**
+     * Handle a newly attached target (top-level or potential OOPIF):
+     * - Enable Page domain and lifecycle events.
+     * - If top-level → create Page, wire listeners, resume.
+     * - Else → probe child root frame id via `Page.getFrameTree` and adopt immediately
+     *   if the parent is known; otherwise stage until parent `frameAttached`.
+     * - Resume the target only after listeners are wired.
+     */
+    private onAttachedToTarget;
+    /**
+     * Detach handler:
+     * - Remove child session ownership and prune its subtree.
+     * - If a top-level target, cleanup its `Page` and mappings.
+     * - Drop any staged child for this session.
+     */
+    private onDetachedFromTarget;
+    /**
+     * Cleanup a top-level Page by target id, removing its root and staged children.
+     */
+    private cleanupByTarget;
+    /**
+     * Wire Page-domain frame events for a session into the owning Page & mappings.
+     * We forward the *emitting session* with every event so Page can stamp ownership precisely.
+     */
+    private installFrameEventBridges;
+    /**
+     * Register that a session belongs to a Page (used by event routing).
+     */
+    private wireSessionToOwnerPage;
+    /**
+     * Utility: reverse-lookup the top-level target id that owns a given Page.
+     */
+    private findTargetIdByPage;
+    private _notePopupSignal;
+    /**
+     * Await the current active page, waiting briefly if a popup/open was just triggered.
+     * Normal path returns immediately; popup path waits up to timeoutMs for the new page.
+     */
+    awaitActivePage(timeoutMs?: number): Promise<Page>;
+}