@browserbasehq/orca 3.0.0-preview.0 → 3.0.0-preview.1

package/dist/index.d.ts

@@ -1355,6 +1355,7 @@ declare class V3 {
     private llmProvider;
     private readonly domSettleTimeoutMs?;
     private _isClosing;
+    browserbaseSessionId?: string;
     private _onCdpClosed;
     readonly experimental: boolean;
     readonly logInferenceToFile: boolean;
@@ -1397,6 +1398,7 @@ declare class V3 {
     init(): Promise<void>;
     /** Apply post-connect local browser options that require CDP. */
     private _applyPostConnectLocalOptions;
+    private _ensureBrowserbaseDownloadsEnabled;
     /**
      * Run an "act" instruction through the ActHandler.
      *

package/dist/index.js

@@ -1253,7 +1253,7 @@ function decorateRoles(nodes, opts) {
     const domIsScrollable = encodedId ? opts.scrollableMap[encodedId] === true : false;
     const tag = encodedId ? opts.tagNameMap[encodedId] : void 0;
     const isHtmlElement = tag === "html";
-    if (domIsScrollable || isHtmlElement) {
+    if ((domIsScrollable || isHtmlElement) && tag !== "#document") {
       const tagLabel = tag && tag.startsWith("#") ? tag.slice(1) : tag;
       role = tagLabel ? `scrollable, ${tagLabel}` : `scrollable${role ? `, ${role}` : ""}`;
     }
@@ -6660,7 +6660,7 @@ init_deepLocator();
 init_logger2();
 
 // lib/version.ts
-var STAGEHAND_VERSION = "3.0.0-preview.0";
+var STAGEHAND_VERSION = "3.0.0-preview.1";
 
 // lib/v3/types/stagehandErrors.ts
 var StagehandError = class extends Error {
@@ -7738,12 +7738,30 @@ function transformSchema(schema, currentPath) {
   return [schema, []];
 }
 function injectUrls(obj, path5, idToUrlMapping) {
-  var _a15;
+  var _a15, _b;
   if (path5.length === 0) return;
+  const toId = (value) => {
+    if (typeof value === "number") {
+      return String(value);
+    }
+    if (typeof value === "string" && ID_PATTERN.test(value)) {
+      return value;
+    }
+    return void 0;
+  };
   const [key, ...rest] = path5;
   if (key === "*") {
     if (Array.isArray(obj)) {
-      for (const item of obj) injectUrls(item, rest, idToUrlMapping);
+      if (rest.length === 0) {
+        for (let i = 0; i < obj.length; i += 1) {
+          const id = toId(obj[i]);
+          if (id !== void 0) {
+            obj[i] = (_a15 = idToUrlMapping[id]) != null ? _a15 : "";
+          }
+        }
+      } else {
+        for (const item of obj) injectUrls(item, rest, idToUrlMapping);
+      }
     }
     return;
   }
@@ -7751,9 +7769,9 @@ function injectUrls(obj, path5, idToUrlMapping) {
     const record = obj;
     if (path5.length === 1) {
       const fieldValue = record[key];
-      const id = typeof fieldValue === "number" ? String(fieldValue) : typeof fieldValue === "string" && ID_PATTERN.test(fieldValue) ? fieldValue : void 0;
+      const id = toId(fieldValue);
       if (id !== void 0) {
-        record[key] = (_a15 = idToUrlMapping[id]) != null ? _a15 : "";
+        record[key] = (_b = idToUrlMapping[id]) != null ? _b : "";
       }
     } else {
       injectUrls(record[key], rest, idToUrlMapping);
@@ -27455,6 +27473,7 @@ var _V3 = class _V3 {
             createdTempProfile: createdTemp,
             preserveUserDataDir: !!lbo.preserveUserDataDir
           };
+          this.browserbaseSessionId = void 0;
           yield this._applyPostConnectLocalOptions(lbo);
           return;
         }
@@ -27472,6 +27491,8 @@ var _V3 = class _V3 {
           });
           this.ctx.conn.onTransportClosed(this._onCdpClosed);
           this.state = { kind: "BROWSERBASE", sessionId, ws, bb };
+          this.browserbaseSessionId = sessionId;
+          yield this._ensureBrowserbaseDownloadsEnabled();
           try {
             const resumed = !!this.opts.browserbaseSessionID;
             let debugUrl;
@@ -27527,6 +27548,21 @@ var _V3 = class _V3 {
       }
     });
   }
+  _ensureBrowserbaseDownloadsEnabled() {
+    return __async(this, null, function* () {
+      var _a15;
+      const conn = (_a15 = this.ctx) == null ? void 0 : _a15.conn;
+      if (!conn) return;
+      try {
+        yield conn.send("Browser.setDownloadBehavior", {
+          behavior: "allow",
+          downloadPath: "downloads",
+          eventsEnabled: true
+        });
+      } catch (e) {
+      }
+    });
+  }
   act(input, options) {
     return __async(this, null, function* () {
       return yield withInstanceLogContext(this.instanceId, () => __async(this, null, function* () {
@@ -27829,6 +27865,7 @@ var _V3 = class _V3 {
         this.state = { kind: "UNINITIALIZED" };
         this.ctx = null;
         this._isClosing = false;
+        this.browserbaseSessionId = void 0;
         try {
           unbindInstanceLogger(this.instanceId);
         } catch (e) {

package/dist/lib/StagehandContext.d.ts

@@ -0,0 +1,25 @@
+import type { BrowserContext as PlaywrightContext, Page as PlaywrightPage } from "playwright";
+import { Stagehand } from "./index";
+import { StagehandPage } from "./StagehandPage";
+import { EnhancedContext } from "../types/context";
+export declare class StagehandContext {
+    private readonly stagehand;
+    private readonly intContext;
+    private pageMap;
+    private activeStagehandPage;
+    private readonly frameIdMap;
+    private constructor();
+    private createStagehandPage;
+    static init(context: PlaywrightContext, stagehand: Stagehand): Promise<StagehandContext>;
+    get frameIdLookup(): ReadonlyMap<string, StagehandPage>;
+    registerFrameId(frameId: string, page: StagehandPage): void;
+    unregisterFrameId(frameId: string): void;
+    getStagehandPageByFrameId(frameId: string): StagehandPage | undefined;
+    get context(): EnhancedContext;
+    getStagehandPage(page: PlaywrightPage): Promise<StagehandPage>;
+    getStagehandPages(): Promise<StagehandPage[]>;
+    setActivePage(page: StagehandPage): void;
+    getActivePage(): StagehandPage | null;
+    private handleNewPlaywrightPage;
+    private attachFrameNavigatedListener;
+}

package/dist/lib/StagehandPage.d.ts

@@ -0,0 +1,103 @@
+import type { CDPSession, Page as PlaywrightPage, Frame } from "playwright";
+import { z } from "zod/v3";
+import { Page, defaultExtractSchema } from "../types/page";
+import { ExtractOptions, ExtractResult, ObserveOptions, ObserveResult } from "../types/stagehand";
+import { StagehandAPI } from "./api";
+import { ActOptions, ActResult, Stagehand } from "./index";
+import { LLMClient } from "./llm/LLMClient";
+import { StagehandContext } from "./StagehandContext";
+import { EncodedId, EnhancedContext } from "../types/context";
+export declare class StagehandPage {
+    private stagehand;
+    private rawPage;
+    private intPage;
+    private intContext;
+    private actHandler;
+    private extractHandler;
+    private observeHandler;
+    private llmClient;
+    private cdpClient;
+    private api;
+    private userProvidedInstructions?;
+    private waitForCaptchaSolves;
+    private initialized;
+    private readonly cdpClients;
+    private fidOrdinals;
+    private rootFrameId;
+    get frameId(): string;
+    updateRootFrameId(newId: string): void;
+    constructor(page: PlaywrightPage, stagehand: Stagehand, context: StagehandContext, llmClient: LLMClient, userProvidedInstructions?: string, api?: StagehandAPI, waitForCaptchaSolves?: boolean);
+    ordinalForFrameId(fid: string | undefined): number;
+    encodeWithFrameId(fid: string | undefined, backendId: number): EncodedId;
+    resetFrameOrdinals(): void;
+    private ensureStagehandScript;
+    /** Register the custom selector engine that pierces open/closed shadow roots. */
+    private ensureStagehandSelectorEngine;
+    /**
+     * Waits for a captcha to be solved when using Browserbase environment.
+     *
+     * @param timeoutMs - Optional timeout in milliseconds. If provided, the promise will reject if the captcha solving hasn't started within the given time.
+     * @throws StagehandEnvironmentError if called in a LOCAL environment
+     * @throws CaptchaTimeoutError if the timeout is reached before captcha solving starts
+     * @returns Promise that resolves when the captcha is solved
+     */
+    waitForCaptchaSolve(timeoutMs?: number): Promise<void>;
+    init(): Promise<StagehandPage>;
+    get page(): Page;
+    get context(): EnhancedContext;
+    /**
+     * `_waitForSettledDom` waits until the DOM is settled, and therefore is
+     * ready for actions to be taken.
+     *
+     * **Definition of "settled"**
+     *   • No in-flight network requests (except WebSocket / Server-Sent-Events).
+     *   • That idle state lasts for at least **500 ms** (the "quiet-window").
+     *
+     * **How it works**
+     *   1.  Subscribes to CDP Network and Page events for the main target and all
+     *       out-of-process iframes (via `Target.setAutoAttach { flatten:true }`).
+     *   2.  Every time `Network.requestWillBeSent` fires, the request ID is added
+     *       to an **`inflight`** `Set`.
+     *   3.  When the request finishes—`loadingFinished`, `loadingFailed`,
+     *       `requestServedFromCache`, or a *data:* response—the request ID is
+     *       removed.
+     *   4.  *Document* requests are also mapped **frameId → requestId**; when
+     *       `Page.frameStoppedLoading` fires the corresponding Document request is
+     *       removed immediately (covers iframes whose network events never close).
+     *   5.  A **stalled-request sweep timer** runs every 500 ms.  If a *Document*
+     *       request has been open for ≥ 2 s it is forcibly removed; this prevents
+     *       ad/analytics iframes from blocking the wait forever.
+     *   6.  When `inflight` becomes empty the helper starts a 500 ms timer.
+     *       If no new request appears before the timer fires, the promise
+     *       resolves → **DOM is considered settled**.
+     *   7.  A global guard (`timeoutMs` or `stagehand.domSettleTimeoutMs`,
+     *       default ≈ 30 s) ensures we always resolve; if it fires we log how many
+     *       requests were still outstanding.
+     *
+     * @param timeoutMs – Optional hard cap (ms).  Defaults to
+     *                    `this.stagehand.domSettleTimeoutMs`.
+     */
+    _waitForSettledDom(timeoutMs?: number): Promise<void>;
+    act(actionOrOptions: string | ActOptions | ObserveResult): Promise<ActResult>;
+    extract<T extends z.AnyZodObject = typeof defaultExtractSchema>(instructionOrOptions?: string | ExtractOptions<T>): Promise<ExtractResult<T>>;
+    observe(instructionOrOptions?: string | ObserveOptions): Promise<ObserveResult[]>;
+    /**
+     * Get or create a CDP session for the given target.
+     * @param target  The Page or (OOPIF) Frame you want to talk to.
+     */
+    getCDPClient(target?: PlaywrightPage | Frame): Promise<CDPSession>;
+    /**
+     * Send a CDP command to the chosen DevTools target.
+     *
+     * @param method  Any valid CDP method, e.g. `"DOM.getDocument"`.
+     * @param params  Command parameters (optional).
+     * @param target  A `Page` or OOPIF `Frame`. Defaults to the main page.
+     *
+     * @typeParam T  Expected result shape (defaults to `unknown`).
+     */
+    sendCDP<T = unknown>(method: string, params?: Record<string, unknown>, target?: PlaywrightPage | Frame): Promise<T>;
+    /** Enable a CDP domain (e.g. `"Network"` or `"DOM"`) on the chosen target. */
+    enableCDP(domain: string, target?: PlaywrightPage | Frame): Promise<void>;
+    /** Disable a CDP domain on the chosen target. */
+    disableCDP(domain: string, target?: PlaywrightPage | Frame): Promise<void>;
+}

package/dist/lib/a11y/utils.d.ts

@@ -0,0 +1,144 @@
+import { AccessibilityNode, TreeResult, BackendIdMaps, CombinedA11yResult, EncodedId } from "../../types/context";
+import { StagehandPage } from "../StagehandPage";
+import { LogLine } from "../../types/log";
+import { Frame } from "playwright";
+/**
+ * Clean a string by removing private-use unicode characters, normalizing whitespace,
+ * and trimming the result.
+ *
+ * @param input - The text to clean, potentially containing PUA and NBSP characters.
+ * @returns A cleaned string with PUA characters removed, NBSP variants collapsed,
+ *          consecutive spaces merged, and leading/trailing whitespace trimmed.
+ */
+export declare function cleanText(input: string): string;
+/**
+ * Generate a human-readable, indented outline of an accessibility node tree.
+ *
+ * @param node - The accessibility node to format, optionally with an encodedId.
+ * @param level - The current depth level for indentation (used internally).
+ * @returns A string representation of the node and its descendants, with one node per line.
+ */
+export declare function formatSimplifiedTree(node: AccessibilityNode & {
+    encodedId?: EncodedId;
+}, level?: number): string;
+/**
+ * Build mappings from CDP backendNodeIds to HTML tag names and relative XPaths.
+ *
+ * @param experimental - Whether to use experimental behaviour.
+ * @param sp - The StagehandPage wrapper for Playwright and CDP calls.
+ * @param targetFrame - Optional Playwright.Frame whose DOM subtree to map; defaults to main frame.
+ * @returns A Promise resolving to BackendIdMaps containing tagNameMap and xpathMap.
+ */
+export declare function buildBackendIdMaps(experimental: boolean, sp: StagehandPage, targetFrame?: Frame): Promise<BackendIdMaps>;
+/**
+ * Convert a flat array of AccessibilityNodes into a cleaned, hierarchical tree.
+ * Nodes are pruned, structural wrappers removed, and each kept node is stamped
+ * with its EncodedId for later lookup or subtree injection.
+ *
+ * @param nodes - Raw flat list of AX nodes retrieved via CDP.
+ * @param tagNameMap - Mapping of EncodedId to HTML tag names for structural decisions.
+ * @param logger - Optional function for logging diagnostic messages.
+ * @param xpathMap - Optional mapping of EncodedId to relative XPath for element lookup.
+ * @returns A Promise resolving to a TreeResult with cleaned tree, simplified text outline,
+ *          iframe list, URL map, and inherited xpathMap.
+ */
+export declare function buildHierarchicalTree(nodes: AccessibilityNode[], tagNameMap: Record<EncodedId, string>, logger?: (l: LogLine) => void, xpathMap?: Record<EncodedId, string>): Promise<TreeResult>;
+/**
+ * Resolve the CDP frame identifier for a Playwright Frame, handling same-process and OOPIF.
+ *
+ * @param sp - The StagehandPage instance for issuing CDP commands.
+ * @param frame - The target Playwright.Frame; undefined or main frame yields undefined.
+ * @returns A Promise resolving to the CDP frameId string, or undefined for main document.
+ */
+export declare function getCDPFrameId(sp: StagehandPage, frame?: Frame): Promise<string | undefined>;
+/**
+ * Retrieve and build a cleaned accessibility tree for a document or specific iframe.
+ * Prunes, formats, and optionally filters by XPath, including scrollable role decoration.
+ *
+ * @param stagehandPage - The StagehandPage instance for Playwright and CDP interaction.
+ * @param logger - Logging function for diagnostics and performance metrics.
+ * @param selector - Optional XPath to filter the AX tree to a specific subtree.
+ * @param targetFrame - Optional Playwright.Frame to scope the AX tree retrieval.
+ * @returns A Promise resolving to a TreeResult with the hierarchical AX tree and related metadata.
+ */
+export declare function getAccessibilityTree(experimental: boolean, stagehandPage: StagehandPage, logger: (log: LogLine) => void, selector?: string, targetFrame?: Frame): Promise<TreeResult>;
+/**
+ * Get the backendNodeId of the iframe element that contains a given Playwright.Frame.
+ *
+ * @param sp - The StagehandPage instance for issuing CDP commands.
+ * @param frame - The Playwright.Frame whose host iframe element to locate.
+ * @returns A Promise resolving to the backendNodeId of the iframe element, or null if not applicable.
+ */
+export declare function getFrameRootBackendNodeId(sp: StagehandPage, frame: Frame | undefined): Promise<number | null>;
+/**
+ * Compute the absolute XPath for the iframe element hosting a given Playwright.Frame.
+ *
+ * @param frame - The Playwright.Frame whose iframe element to locate.
+ * @returns A Promise resolving to the XPath of the iframe element, or "/" if no frame provided.
+ */
+export declare function getFrameRootXpathWithShadow(frame: Frame | undefined): Promise<string>;
+export declare function getFrameRootXpath(frame: Frame | undefined): Promise<string>;
+/**
+ * Inject simplified subtree outlines into the main frame outline for nested iframes.
+ * Walks the main tree text, looks for EncodedId labels, and inserts matching subtrees.
+ *
+ * @param tree - The indented AX outline of the main frame.
+ * @param idToTree - Map of EncodedId to subtree outlines for nested frames.
+ * @returns A single combined text outline with iframe subtrees injected.
+ */
+export declare function injectSubtrees(tree: string, idToTree: Map<EncodedId, string>): string;
+/**
+ * Retrieve and merge accessibility trees for the main document and nested iframes.
+ * Walks through frame chains if a root XPath is provided, then stitches subtree outlines.
+ *
+ * @param stagehandPage - The StagehandPage instance for Playwright and CDP interaction.
+ * @param logger - Logging function for diagnostics and performance.
+ * @param rootXPath - Optional absolute XPath to focus the crawl on a subtree across frames.
+ * @returns A Promise resolving to CombinedA11yResult with combined tree text, xpath map, and URL map.
+ */
+export declare function getAccessibilityTreeWithFrames(experimental: boolean, stagehandPage: StagehandPage, logger: (l: LogLine) => void, rootXPath?: string): Promise<CombinedA11yResult>;
+/**
+ * `findScrollableElementIds` is a function that identifies elements in
+ * the browser that are deemed "scrollable". At a high level, it does the
+ * following:
+ * - Calls the browser-side `window.getScrollableElementXpaths()` function,
+ *   which returns a list of XPaths for scrollable containers.
+ * - Iterates over the returned list of XPaths, locating each element in the DOM
+ *   using `stagehandPage.sendCDP(...)`
+ *     - During each iteration, we call `Runtime.evaluate` to run `document.evaluate(...)`
+ *       with each XPath, obtaining a `RemoteObject` reference if it exists.
+ *     - Then, for each valid object reference, we call `DOM.describeNode` to retrieve
+ *       the element’s `backendNodeId`.
+ * - Collects all resulting `backendNodeId`s in a Set and returns them.
+ *
+ * @param stagehandPage - A StagehandPage instance with built-in CDP helpers.
+ * @returns A Promise that resolves to a Set of unique `backendNodeId`s corresponding
+ *          to scrollable elements in the DOM.
+ */
+export declare function findScrollableElementIds(stagehandPage: StagehandPage, targetFrame?: Frame): Promise<Set<number>>;
+/**
+ * Resolve an XPath to a Chrome-DevTools-Protocol (CDP) remote-object ID.
+ *
+ * @param page     A StagehandPage (or Playwright.Page with .sendCDP)
+ * @param xpath    An absolute or relative XPath
+ * @returns        The remote objectId for the matched node, or null
+ */
+export declare function resolveObjectIdForXPath(page: StagehandPage, xpath: string, targetFrame?: Frame): Promise<string | null>;
+/**
+ * Resolve a chain of iframe frames from an absolute XPath, returning the frame sequence and inner XPath.
+ *
+ * This helper walks an XPath expression containing iframe steps (e.g., '/html/body/iframe[2]/...'),
+ * descending into each matching iframe element to build a frame chain, and returns the leftover
+ * XPath segment to evaluate within the context of the last iframe.
+ *
+ * @param sp - The StagehandPage instance for evaluating XPath and locating frames.
+ * @param absPath - An absolute XPath expression starting with '/', potentially including iframe steps.
+ * @returns An object containing:
+ *   frames: Array of Frame objects representing each iframe in the chain.
+ *   rest: The remaining XPath string to evaluate inside the final iframe.
+ * @throws Error if an iframe cannot be found or the final XPath cannot be resolved.
+ */
+export declare function resolveFrameChain(sp: StagehandPage, absPath: string): Promise<{
+    frames: Frame[];
+    rest: string;
+}>;

package/dist/lib/agent/AgentClient.d.ts

@@ -0,0 +1,20 @@
+import { AgentAction, AgentResult, AgentType, AgentExecutionOptions } from "@/types/agent";
+import { ToolSet } from "ai/dist";
+/**
+ * Abstract base class for agent clients
+ * This provides a common interface for all agent implementations
+ */
+export declare abstract class AgentClient {
+    type: AgentType;
+    modelName: string;
+    clientOptions: Record<string, unknown>;
+    userProvidedInstructions?: string;
+    constructor(type: AgentType, modelName: string, userProvidedInstructions?: string);
+    abstract execute(options: AgentExecutionOptions): Promise<AgentResult>;
+    abstract captureScreenshot(options?: Record<string, unknown>): Promise<unknown>;
+    abstract setViewport(width: number, height: number): void;
+    abstract setCurrentUrl(url: string): void;
+    abstract setScreenshotProvider(provider: () => Promise<string>): void;
+    abstract setActionHandler(handler: (action: AgentAction) => Promise<void>): void;
+    abstract setTools(tools: ToolSet): void;
+}

package/dist/lib/agent/AgentProvider.d.ts

@@ -0,0 +1,19 @@
+import { AgentType } from "@/types/agent";
+import { LogLine } from "@/types/log";
+import { ToolSet } from "ai/dist";
+import { AgentClient } from "./AgentClient";
+export declare const modelToAgentProviderMap: Record<string, AgentType>;
+/**
+ * Provider for agent clients
+ * This class is responsible for creating the appropriate agent client
+ * based on the provider type
+ */
+export declare class AgentProvider {
+    private logger;
+    /**
+     * Create a new agent provider
+     */
+    constructor(logger: (message: LogLine) => void);
+    getClient(modelName: string, clientOptions?: Record<string, unknown>, userProvidedInstructions?: string, tools?: ToolSet): AgentClient;
+    static getAgentProvider(modelName: string): AgentType;
+}

package/dist/lib/agent/AnthropicCUAClient.d.ts

@@ -0,0 +1,56 @@
+import { AgentAction, AgentExecutionOptions, AgentResult, AgentType, AnthropicContentBlock, AnthropicMessage, AnthropicToolResult, ToolUseItem } from "@/types/agent";
+import { LogLine } from "@/types/log";
+import { ToolSet } from "ai";
+import { AgentClient } from "./AgentClient";
+export type ResponseInputItem = AnthropicMessage | AnthropicToolResult;
+/**
+ * Client for Anthropic's Computer Use API
+ * This implementation uses the official Anthropic Messages API for Computer Use
+ */
+export declare class AnthropicCUAClient extends AgentClient {
+    private apiKey;
+    private baseURL?;
+    private client;
+    lastMessageId?: string;
+    private currentViewport;
+    private currentUrl?;
+    private screenshotProvider?;
+    private actionHandler?;
+    private thinkingBudget;
+    private tools?;
+    constructor(type: AgentType, modelName: string, userProvidedInstructions?: string, clientOptions?: Record<string, unknown>, tools?: ToolSet);
+    setViewport(width: number, height: number): void;
+    setCurrentUrl(url: string): void;
+    setScreenshotProvider(provider: () => Promise<string>): void;
+    setActionHandler(handler: (action: AgentAction) => Promise<void>): void;
+    setTools(tools: ToolSet): void;
+    /**
+     * Execute a task with the Anthropic CUA
+     * This is the main entry point for the agent
+     * @implements AgentClient.execute
+     */
+    execute(executionOptions: AgentExecutionOptions): Promise<AgentResult>;
+    executeStep(inputItems: ResponseInputItem[], logger: (message: LogLine) => void): Promise<{
+        actions: AgentAction[];
+        message: string;
+        completed: boolean;
+        nextInputItems: ResponseInputItem[];
+        usage: {
+            input_tokens: number;
+            output_tokens: number;
+            inference_time_ms: number;
+        };
+    }>;
+    private createInitialInputItems;
+    getAction(inputItems: ResponseInputItem[]): Promise<{
+        content: AnthropicContentBlock[];
+        id: string;
+        usage: Record<string, number>;
+    }>;
+    takeAction(toolUseItems: ToolUseItem[], logger: (message: LogLine) => void): Promise<AnthropicToolResult[]>;
+    private convertToolUseToAction;
+    captureScreenshot(options?: {
+        base64Image?: string;
+        currentUrl?: string;
+    }): Promise<string>;
+}

package/dist/lib/agent/GoogleCUAClient.d.ts

@@ -0,0 +1,63 @@
+import { LogLine } from "../../types/log";
+import { AgentAction, AgentResult, AgentType, AgentExecutionOptions } from "@/types/agent";
+import { AgentClient } from "./AgentClient";
+/**
+ * Client for Google's Computer Use Assistant API
+ * This implementation uses the Google Generative AI SDK for Computer Use
+ */
+export declare class GoogleCUAClient extends AgentClient {
+    private apiKey;
+    private client;
+    private currentViewport;
+    private currentUrl?;
+    private screenshotProvider?;
+    private actionHandler?;
+    private history;
+    private environment;
+    private generateContentConfig;
+    constructor(type: AgentType, modelName: string, userProvidedInstructions?: string, clientOptions?: Record<string, unknown>);
+    setViewport(width: number, height: number): void;
+    setCurrentUrl(url: string): void;
+    setScreenshotProvider(provider: () => Promise<string>): void;
+    setActionHandler(handler: (action: AgentAction) => Promise<void>): void;
+    setTools(): void;
+    /**
+     * Execute a task with the Google CUA
+     * This is the main entry point for the agent
+     * @implements AgentClient.execute
+     */
+    execute(executionOptions: AgentExecutionOptions): Promise<AgentResult>;
+    /**
+     * Initialize conversation history with the initial instruction
+     */
+    private initializeHistory;
+    /**
+     * Execute a single step of the agent
+     */
+    executeStep(logger: (message: LogLine) => void): Promise<{
+        actions: AgentAction[];
+        message: string;
+        completed: boolean;
+        usage: {
+            input_tokens: number;
+            output_tokens: number;
+            inference_time_ms: number;
+        };
+    }>;
+    /**
+     * Process the response from Google's API
+     */
+    private processResponse;
+    /**
+     * Convert Google function call to Stagehand action
+     */
+    private convertFunctionCallToAction;
+    /**
+     * Normalize coordinates from Google's 0-1000 range to actual viewport dimensions
+     */
+    private normalizeCoordinates;
+    captureScreenshot(options?: {
+        base64Image?: string;
+        currentUrl?: string;
+    }): Promise<string>;
+}

package/dist/lib/agent/OpenAICUAClient.d.ts

@@ -0,0 +1,65 @@
+import { LogLine } from "../../types/log";
+import { AgentAction, AgentResult, AgentType, AgentExecutionOptions, ResponseInputItem, ResponseItem } from "@/types/agent";
+import { AgentClient } from "./AgentClient";
+import { ToolSet } from "ai/dist";
+/**
+ * Client for OpenAI's Computer Use Assistant API
+ * This implementation uses the official OpenAI Responses API for Computer Use
+ */
+export declare class OpenAICUAClient extends AgentClient {
+    private apiKey;
+    private organization?;
+    private baseURL;
+    private client;
+    lastResponseId?: string;
+    private currentViewport;
+    private currentUrl?;
+    private screenshotProvider?;
+    private actionHandler?;
+    private reasoningItems;
+    private environment;
+    private tools?;
+    constructor(type: AgentType, modelName: string, userProvidedInstructions?: string, clientOptions?: Record<string, unknown>, tools?: ToolSet);
+    setViewport(width: number, height: number): void;
+    setCurrentUrl(url: string): void;
+    setScreenshotProvider(provider: () => Promise<string>): void;
+    setActionHandler(handler: (action: AgentAction) => Promise<void>): void;
+    setTools(tools: ToolSet): void;
+    /**
+     * Execute a task with the OpenAI CUA
+     * This is the main entry point for the agent
+     * @implements AgentClient.execute
+     */
+    execute(executionOptions: AgentExecutionOptions): Promise<AgentResult>;
+    /**
+     * Execute a single step of the agent
+     * This coordinates the flow: Request → Get Action → Execute Action
+     */
+    executeStep(inputItems: ResponseInputItem[], previousResponseId: string | undefined, logger: (message: LogLine) => void): Promise<{
+        actions: AgentAction[];
+        message: string;
+        completed: boolean;
+        nextInputItems: ResponseInputItem[];
+        responseId: string;
+        usage: {
+            input_tokens: number;
+            output_tokens: number;
+            inference_time_ms: number;
+        };
+    }>;
+    private isComputerCallItem;
+    private isFunctionCallItem;
+    private createInitialInputItems;
+    getAction(inputItems: ResponseInputItem[], previousResponseId?: string): Promise<{
+        output: ResponseItem[];
+        responseId: string;
+        usage: Record<string, number>;
+    }>;
+    takeAction(output: ResponseItem[], logger: (message: LogLine) => void): Promise<ResponseInputItem[]>;
+    private convertComputerCallToAction;
+    private convertFunctionCallToAction;
+    captureScreenshot(options?: {
+        base64Image?: string;
+        currentUrl?: string;
+    }): Promise<string>;
+}

package/dist/lib/agent/StagehandAgent.d.ts

@@ -0,0 +1,15 @@
+import { LogLine } from "@/types/log";
+import { AgentExecuteOptions, AgentResult } from "@/types/agent";
+import { AgentClient } from "./AgentClient";
+/**
+ * Main interface for agent operations in Stagehand
+ * This class provides methods for executing tasks with an agent
+ */
+export declare class StagehandAgent {
+    private client;
+    private logger;
+    constructor(client: AgentClient, logger: (message: LogLine) => void);
+    execute(optionsOrInstruction: AgentExecuteOptions | string): Promise<AgentResult>;
+    getModelName(): string;
+    getAgentType(): string;
+}