@browserbasehq/orca 3.0.0-preview.0 → 3.0.0-preview.1

package/dist/lib/llm/LLMClient.d.ts

@@ -0,0 +1,99 @@
+import { LLMTool } from "@/types/llm";
+import { embed, embedMany, experimental_generateImage, experimental_generateSpeech, experimental_transcribe, generateObject, generateText, LanguageModel, streamObject, streamText } from "ai";
+import { ZodType } from "zod/v3";
+import { LogLine } from "../../types/log";
+import { AvailableModel, ClientOptions } from "../../types/model";
+export interface ChatMessage {
+    role: "system" | "user" | "assistant";
+    content: ChatMessageContent;
+}
+export type ChatMessageContent = string | (ChatMessageImageContent | ChatMessageTextContent)[];
+export interface ChatMessageImageContent {
+    type: string;
+    image_url?: {
+        url: string;
+    };
+    text?: string;
+    source?: {
+        type: string;
+        media_type: string;
+        data: string;
+    };
+}
+export interface ChatMessageTextContent {
+    type: string;
+    text: string;
+}
+export declare const AnnotatedScreenshotText = "This is a screenshot of the current page state with the elements annotated on it. Each element id is annotated with a number to the top left of it. Duplicate annotations at the same location are under each other vertically.";
+export interface ChatCompletionOptions {
+    messages: ChatMessage[];
+    temperature?: number;
+    top_p?: number;
+    frequency_penalty?: number;
+    presence_penalty?: number;
+    image?: {
+        buffer: Buffer;
+        description?: string;
+    };
+    response_model?: {
+        name: string;
+        schema: ZodType;
+    };
+    tools?: LLMTool[];
+    tool_choice?: "auto" | "none" | "required";
+    maxTokens?: number;
+    requestId?: string;
+}
+export type LLMResponse = {
+    id: string;
+    object: string;
+    created: number;
+    model: string;
+    choices: {
+        index: number;
+        message: {
+            role: string;
+            content: string | null;
+            tool_calls: {
+                id: string;
+                type: string;
+                function: {
+                    name: string;
+                    arguments: string;
+                };
+            }[];
+        };
+        finish_reason: string;
+    }[];
+    usage: {
+        prompt_tokens: number;
+        completion_tokens: number;
+        total_tokens: number;
+    };
+};
+export interface CreateChatCompletionOptions {
+    options: ChatCompletionOptions;
+    logger: (message: LogLine) => void;
+    retries?: number;
+}
+export declare abstract class LLMClient {
+    type: "openai" | "anthropic" | "cerebras" | "groq" | (string & {});
+    modelName: AvailableModel | (string & {});
+    hasVision: boolean;
+    clientOptions: ClientOptions;
+    userProvidedInstructions?: string;
+    constructor(modelName: AvailableModel, userProvidedInstructions?: string);
+    abstract createChatCompletion<T = LLMResponse & {
+        usage?: LLMResponse["usage"];
+    }>(options: CreateChatCompletionOptions): Promise<T>;
+    generateObject: typeof generateObject;
+    generateText: typeof generateText;
+    streamText: typeof streamText;
+    streamObject: typeof streamObject;
+    generateImage: typeof experimental_generateImage;
+    embed: typeof embed;
+    embedMany: typeof embedMany;
+    transcribe: typeof experimental_transcribe;
+    generateSpeech: typeof experimental_generateSpeech;
+    getLanguageModel?(): LanguageModel;
+}

package/dist/lib/llm/LLMProvider.d.ts

@@ -0,0 +1,13 @@
+import { LogLine } from "../../types/log";
+import { AvailableModel, ClientOptions, ModelProvider } from "../../types/model";
+import { LLMClient } from "./LLMClient";
+export declare function getAISDKLanguageModel(subProvider: string, subModelName: string, apiKey?: string, baseURL?: string): import("ai/dist").LanguageModelV1;
+export declare class LLMProvider {
+    private logger;
+    private enableCaching;
+    private cache;
+    constructor(logger: (message: LogLine) => void, enableCaching: boolean);
+    cleanRequestCache(requestId: string): void;
+    getClient(modelName: AvailableModel, clientOptions?: ClientOptions): LLMClient;
+    static getModelProvider(modelName: AvailableModel): ModelProvider;
+}

package/dist/lib/llm/OpenAIClient.d.ts

@@ -0,0 +1,20 @@
+import { ClientOptions } from "openai";
+import { LogLine } from "../../types/log";
+import { AvailableModel } from "../../types/model";
+import { LLMCache } from "../cache/LLMCache";
+import { CreateChatCompletionOptions, LLMClient, LLMResponse } from "./LLMClient";
+export declare class OpenAIClient extends LLMClient {
+    type: "openai";
+    private client;
+    private cache;
+    private enableCaching;
+    clientOptions: ClientOptions;
+    constructor({ enableCaching, cache, modelName, clientOptions, }: {
+        logger: (message: LogLine) => void;
+        enableCaching?: boolean;
+        cache?: LLMCache;
+        modelName: AvailableModel;
+        clientOptions?: ClientOptions;
+    });
+    createChatCompletion<T = LLMResponse>({ options: optionsInitial, logger, retries, }: CreateChatCompletionOptions): Promise<T>;
+}

package/dist/lib/llm/aisdk.d.ts

@@ -0,0 +1,20 @@
+import { LanguageModel } from "ai";
+import { ChatCompletion } from "openai/resources";
+import { LogLine } from "../../types/log";
+import { LLMCache } from "../cache/LLMCache";
+import { CreateChatCompletionOptions, LLMClient } from "./LLMClient";
+export declare class AISdkClient extends LLMClient {
+    type: "aisdk";
+    private model;
+    private logger?;
+    private cache;
+    private enableCaching;
+    constructor({ model, logger, enableCaching, cache, }: {
+        model: LanguageModel;
+        logger?: (message: LogLine) => void;
+        enableCaching?: boolean;
+        cache?: LLMCache;
+    });
+    getLanguageModel(): LanguageModel;
+    createChatCompletion<T = ChatCompletion>({ options, }: CreateChatCompletionOptions): Promise<T>;
+}

package/dist/lib/mcp/connection.d.ts

@@ -0,0 +1,11 @@
+import { Client, ClientOptions } from "@modelcontextprotocol/sdk/client/index.js";
+export interface ConnectToMCPServerOptions {
+    serverUrl: string | URL;
+    clientOptions?: ClientOptions;
+}
+export interface StdioServerConfig {
+    command: string;
+    args?: string[];
+    env?: Record<string, string>;
+}
+export declare const connectToMCPServer: (serverConfig: string | URL | StdioServerConfig | ConnectToMCPServerOptions) => Promise<Client>;

package/dist/lib/mcp/utils.d.ts

@@ -0,0 +1,3 @@
+import { Client } from "@modelcontextprotocol/sdk/dist/esm/client";
+import { ToolSet } from "ai/dist";
+export declare const resolveTools: (clients: (Client | string)[], userTools: ToolSet) => Promise<ToolSet>;

package/dist/lib/v3/tests/downloads.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/lib/v3/tests/v3.bb.config.d.ts

@@ -0,0 +1,4 @@
+import type { V3Options } from "@/lib/v3/types";
+export declare const v3BBTestConfig: V3Options;
+export declare function getV3BBTestConfig(overrides?: Partial<V3Options>): V3Options;
+export default getV3BBTestConfig;

package/dist/lib/v3/v3.d.ts

@@ -36,6 +36,7 @@ export declare class V3 {
     private llmProvider;
     private readonly domSettleTimeoutMs?;
     private _isClosing;
+    browserbaseSessionId?: string;
     private _onCdpClosed;
     readonly experimental: boolean;
     readonly logInferenceToFile: boolean;
@@ -78,6 +79,7 @@ export declare class V3 {
     init(): Promise<void>;
     /** Apply post-connect local browser options that require CDP. */
     private _applyPostConnectLocalOptions;
+    private _ensureBrowserbaseDownloadsEnabled;
     /**
      * Run an "act" instruction through the ActHandler.
      *

package/dist/lib/version.d.ts

@@ -2,4 +2,4 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export declare const STAGEHAND_VERSION: "3.0.0-preview.0";
+export declare const STAGEHAND_VERSION: "3.0.0-preview.1";

package/dist/stagehand.config.d.ts

@@ -0,0 +1,3 @@
+import type { ConstructorParams } from "@/dist";
+declare const StagehandConfig: ConstructorParams;
+export default StagehandConfig;

package/dist/types/act.d.ts

@@ -0,0 +1,50 @@
+import { LLMClient } from "../lib/llm/LLMClient";
+import { Locator } from "playwright";
+import { Logger } from "@/types/log";
+import { StagehandPage } from "@/lib/StagehandPage";
+export interface ActCommandParams {
+    action: string;
+    steps?: string;
+    domElements: string;
+    llmClient: LLMClient;
+    retries?: number;
+    logger: (message: {
+        category?: string;
+        message: string;
+    }) => void;
+    requestId: string;
+    variables?: Record<string, string>;
+    userProvidedInstructions?: string;
+}
+export interface ActCommandResult {
+    method: string;
+    element: number;
+    args: unknown[];
+    completed: boolean;
+    step: string;
+    why?: string;
+}
+export declare enum SupportedPlaywrightAction {
+    CLICK = "click",
+    FILL = "fill",
+    TYPE = "type",
+    PRESS = "press",
+    SCROLL = "scrollTo",
+    NEXT_CHUNK = "nextChunk",
+    PREV_CHUNK = "prevChunk",
+    SELECT_OPTION_FROM_DROPDOWN = "selectOptionFromDropdown"
+}
+/**
+ * A context object to hold all parameters that might be needed by
+ * any of the methods in the `methodHandlerMap`
+ */
+export interface MethodHandlerContext {
+    method: string;
+    locator: Locator;
+    xpath: string;
+    args: unknown[];
+    logger: Logger;
+    stagehandPage: StagehandPage;
+    initialUrl: string;
+    domSettleTimeoutMs?: number;
+}

package/dist/types/agent.d.ts

@@ -0,0 +1,143 @@
+import { LogLine } from "./log";
+import { ObserveResult } from "./stagehand";
+export interface ActToolResult {
+    success: boolean;
+    action?: string;
+    error?: string;
+    isIframe?: boolean;
+    playwrightArguments?: ObserveResult | null;
+}
+export interface AgentAction {
+    type: string;
+    reasoning?: string;
+    taskCompleted?: boolean;
+    action?: string;
+    timeMs?: number;
+    pageText?: string;
+    pageUrl?: string;
+    instruction?: string;
+    playwrightArguments?: ObserveResult | null;
+    [key: string]: unknown;
+}
+export interface AgentResult {
+    success: boolean;
+    message: string;
+    actions: AgentAction[];
+    completed: boolean;
+    metadata?: Record<string, unknown>;
+    usage?: {
+        input_tokens: number;
+        output_tokens: number;
+        inference_time_ms: number;
+    };
+}
+export interface AgentOptions {
+    maxSteps?: number;
+    autoScreenshot?: boolean;
+    waitBetweenActions?: number;
+    context?: string;
+    highlightCursor?: boolean;
+}
+export interface AgentExecuteOptions extends AgentOptions {
+    instruction: string;
+}
+export type AgentProviderType = "openai" | "anthropic" | "google";
+export interface AgentClientOptions {
+    apiKey: string;
+    organization?: string;
+    baseURL?: string;
+    defaultMaxSteps?: number;
+    [key: string]: unknown;
+}
+export type AgentType = "openai" | "anthropic" | "google";
+export interface AgentExecutionOptions {
+    options: AgentExecuteOptions;
+    logger: (message: LogLine) => void;
+    retries?: number;
+}
+export interface AgentHandlerOptions {
+    modelName: string;
+    clientOptions?: Record<string, unknown>;
+    userProvidedInstructions?: string;
+    agentType: AgentType;
+    experimental?: boolean;
+}
+export interface ActionExecutionResult {
+    success: boolean;
+    error?: string;
+    data?: unknown;
+}
+export interface ToolUseItem extends ResponseItem {
+    type: "tool_use";
+    id: string;
+    name: string;
+    input: Record<string, unknown>;
+}
+export interface AnthropicMessage {
+    role: string;
+    content: string | Array<AnthropicContentBlock>;
+}
+export interface AnthropicContentBlock {
+    type: string;
+    [key: string]: unknown;
+}
+export interface AnthropicTextBlock extends AnthropicContentBlock {
+    type: "text";
+    text: string;
+}
+export interface AnthropicToolResult {
+    type: "tool_result";
+    tool_use_id: string;
+    content: string | Array<AnthropicContentBlock>;
+}
+export interface ResponseItem {
+    type: string;
+    id: string;
+    [key: string]: unknown;
+}
+export interface ComputerCallItem extends ResponseItem {
+    type: "computer_call";
+    call_id: string;
+    action: {
+        type: string;
+        [key: string]: unknown;
+    };
+    pending_safety_checks?: Array<{
+        id: string;
+        code: string;
+        message: string;
+    }>;
+}
+export interface FunctionCallItem extends ResponseItem {
+    type: "function_call";
+    call_id: string;
+    name: string;
+    arguments: string;
+}
+export type ResponseInputItem = {
+    role: string;
+    content: string;
+} | {
+    type: "computer_call_output";
+    call_id: string;
+    output: {
+        type: "input_image";
+        image_url: string;
+        current_url?: string;
+        error?: string;
+        [key: string]: unknown;
+    } | string;
+    acknowledged_safety_checks?: Array<{
+        id: string;
+        code: string;
+        message: string;
+    }>;
+} | {
+    type: "function_call_output";
+    call_id: string;
+    output: string;
+};
+export interface AgentInstance {
+    execute: (instructionOrOptions: string | AgentExecuteOptions) => Promise<AgentResult>;
+    setScreenshotCollector?: (collector: unknown) => void;
+}

package/dist/types/api.d.ts

@@ -0,0 +1,40 @@
+import Browserbase from "@browserbasehq/sdk";
+import { LogLine } from "./log";
+export interface StagehandAPIConstructorParams {
+    apiKey: string;
+    projectId: string;
+    logger: (message: LogLine) => void;
+}
+export interface ExecuteActionParams {
+    method: "act" | "extract" | "observe" | "navigate" | "end" | "agentExecute";
+    args?: unknown;
+    params?: unknown;
+}
+export interface StartSessionParams {
+    modelName: string;
+    modelApiKey: string;
+    domSettleTimeoutMs: number;
+    verbose: number;
+    debugDom: boolean;
+    systemPrompt?: string;
+    browserbaseSessionCreateParams?: Omit<Browserbase.Sessions.SessionCreateParams, "projectId"> & {
+        projectId?: string;
+    };
+    selfHeal?: boolean;
+    waitForCaptchaSolves?: boolean;
+    actionTimeoutMs?: number;
+    browserbaseSessionID?: string;
+}
+export interface StartSessionResult {
+    sessionId: string;
+    available?: boolean;
+}
+export interface SuccessResponse<T> {
+    success: true;
+    data: T;
+}
+export interface ErrorResponse {
+    success: false;
+    message: string;
+}
+export type ApiResponse<T> = SuccessResponse<T> | ErrorResponse;

package/dist/types/browser.d.ts

@@ -0,0 +1,10 @@
+import { Browser, BrowserContext } from "./page";
+export interface BrowserResult {
+    env: "LOCAL" | "BROWSERBASE";
+    browser?: Browser;
+    context: BrowserContext;
+    debugUrl?: string;
+    sessionUrl?: string;
+    contextPath?: string;
+    sessionId?: string;
+}

package/dist/types/context.d.ts

@@ -0,0 +1,117 @@
+import type { BrowserContext as PlaywrightContext, Frame } from "playwright";
+import { Page } from "../types/page";
+export interface AXNode {
+    role?: {
+        value: string;
+    };
+    name?: {
+        value: string;
+    };
+    description?: {
+        value: string;
+    };
+    value?: {
+        value: string;
+    };
+    nodeId: string;
+    backendDOMNodeId?: number;
+    parentId?: string;
+    childIds?: string[];
+    properties?: {
+        name: string;
+        value: {
+            type: string;
+            value?: string;
+        };
+    }[];
+}
+export type AccessibilityNode = {
+    role: string;
+    name?: string;
+    description?: string;
+    value?: string;
+    children?: AccessibilityNode[];
+    childIds?: string[];
+    parentId?: string;
+    nodeId?: string;
+    backendDOMNodeId?: number;
+    properties?: {
+        name: string;
+        value: {
+            type: string;
+            value?: string;
+        };
+    }[];
+};
+export interface TreeResult {
+    tree: AccessibilityNode[];
+    simplified: string;
+    iframes?: AccessibilityNode[];
+    idToUrl: Record<EncodedId, string>;
+    xpathMap: Record<EncodedId, string>;
+}
+export type DOMNode = {
+    backendNodeId?: number;
+    nodeName?: string;
+    children?: DOMNode[];
+    shadowRoots?: DOMNode[];
+    contentDocument?: DOMNode;
+    nodeType: number;
+    frameId?: string;
+};
+export type BackendIdMaps = {
+    tagNameMap: Record<number, string>;
+    xpathMap: Record<number, string>;
+    iframeXPath?: string;
+};
+export interface EnhancedContext extends Omit<PlaywrightContext, "newPage" | "pages"> {
+    newPage(): Promise<Page>;
+    pages(): Page[];
+}
+export type FrameId = string;
+export type LoaderId = string;
+export interface CdpFrame {
+    id: FrameId;
+    parentId?: FrameId;
+    loaderId: LoaderId;
+    name?: string;
+    url: string;
+    urlFragment?: string;
+    domainAndRegistry?: string;
+    securityOrigin: string;
+    securityOriginDetails?: Record<string, unknown>;
+    mimeType: string;
+    unreachableUrl?: string;
+    adFrameStatus?: string;
+    secureContextType?: string;
+    crossOriginIsolatedContextType?: string;
+    gatedAPIFeatures?: string[];
+}
+export interface CdpFrameTree {
+    frame: CdpFrame;
+    childFrames?: CdpFrameTree[];
+}
+export interface FrameOwnerResult {
+    backendNodeId?: number;
+}
+export interface CombinedA11yResult {
+    combinedTree: string;
+    combinedXpathMap: Record<EncodedId, string>;
+    combinedUrlMap: Record<EncodedId, string>;
+}
+export interface FrameSnapshot {
+    frame: Frame;
+    tree: string;
+    xpathMap: Record<EncodedId, string>;
+    urlMap: Record<EncodedId, string>;
+    frameXpath: string;
+    backendNodeId: number | null;
+    parentFrame?: Frame;
+    /** CDP frame identifier for this snapshot; used to generate stable EncodedIds. */
+    frameId?: string;
+}
+export type EncodedId = `${number}-${number}`;
+export interface RichNode extends AccessibilityNode {
+    encodedId?: EncodedId;
+}
+export declare const ID_PATTERN: RegExp;

package/dist/types/evals.d.ts

@@ -0,0 +1,94 @@
+import { z } from "zod/v3";
+import type { AvailableModel } from "../types/model";
+import type { LogLine } from "../types/log";
+import type { AgentInstance } from "../types/agent";
+import type { EvalCase } from "braintrust";
+import { Stagehand } from "@/dist";
+import { ConstructorParams } from "@/dist";
+import { EvalLogger } from "@/evals/logger";
+export type StagehandInitResult = {
+    stagehand: Stagehand;
+    logger: EvalLogger;
+    debugUrl: string;
+    sessionUrl: string;
+    stagehandConfig: ConstructorParams;
+    modelName: AvailableModel;
+    agent: AgentInstance;
+};
+export declare enum ErrorType {
+    TIMEOUT = "timeout",
+    NETWORK = "network",
+    AGENT_FAILURE = "agent_failure",
+    EVALUATION_ERROR = "evaluation_error",
+    SETUP_ERROR = "setup_error",
+    PARSING_ERROR = "parsing_error",
+    ANTIBOT = "bot_detected",
+    UNKNOWN = "unknown"
+}
+export interface EvalOutput {
+    _success: boolean;
+    logs: LogLine[];
+    debugUrl: string;
+    sessionUrl: string;
+    error?: unknown;
+    error_type?: ErrorType;
+    error_message?: string;
+    error_stack?: string;
+    execution_time?: number;
+    agent_steps?: number;
+    final_answer?: string;
+    reasoning?: string;
+    observations?: string | unknown;
+    [key: string]: unknown;
+}
+export type EvalFunction = (taskInput: StagehandInitResult & {
+    input: EvalInput;
+}) => Promise<EvalOutput>;
+export declare const EvalCategorySchema: z.ZodEnum<["observe", "act", "combination", "extract", "experimental", "targeted_extract", "regression", "regression_llm_providers", "llm_clients", "agent", "external_agent_benchmarks"]>;
+export type EvalCategory = z.infer<typeof EvalCategorySchema>;
+export interface EvalInput {
+    name: string;
+    modelName: AvailableModel;
+    params?: Record<string, unknown>;
+}
+export interface TestcaseMetadata {
+    model: AvailableModel;
+    test: string;
+    category?: string;
+    dataset?: string;
+    dataset_id?: string;
+    dataset_level?: string | number;
+    dataset_category?: string;
+    [key: string]: unknown;
+}
+export interface Testcase extends EvalCase<EvalInput, unknown, TestcaseMetadata> {
+    input: EvalInput;
+    name: string;
+    tags: string[];
+    metadata: TestcaseMetadata;
+    expected: unknown;
+}
+export interface SummaryResult {
+    input: EvalInput;
+    output: {
+        _success: boolean;
+    };
+    name: string;
+    score: number;
+}
+export interface EvalArgs<TInput, TOutput, TExpected> {
+    input: TInput;
+    output: TOutput;
+    expected: TExpected;
+    metadata?: {
+        model: AvailableModel;
+        test: string;
+    };
+}
+export interface EvalResult {
+    name: string;
+    score: number;
+}
+export type LogLineEval = LogLine & {
+    parsedAuxiliary?: string | object;
+};