@browserbasehq/orca 3.0.9-alpha-4 → 3.1.0-patch.1

package/dist/esm/lib/v3/agent/utils/messageProcessing.js

@@ -0,0 +1,164 @@
+// Vision action tools that include screenshots in their results
+const VISION_ACTION_TOOLS = [
+    "click",
+    "type",
+    "dragAndDrop",
+    "wait",
+    "fillFormVision",
+    "scroll",
+];
+function isToolMessage(message) {
+    return (!!message &&
+        typeof message === "object" &&
+        message.role === "tool" &&
+        Array.isArray(message.content));
+}
+function isScreenshotPart(part) {
+    return (!!part &&
+        typeof part === "object" &&
+        part.toolName === "screenshot");
+}
+function isVisionActionPart(part) {
+    if (!part || typeof part !== "object")
+        return false;
+    const toolName = part.toolName;
+    return typeof toolName === "string" && VISION_ACTION_TOOLS.includes(toolName);
+}
+function isVisionPart(part) {
+    return isScreenshotPart(part) || isVisionActionPart(part);
+}
+function isAriaTreePart(part) {
+    return (!!part &&
+        typeof part === "object" &&
+        part.toolName === "ariaTree");
+}
+/**
+ * Compress old screenshot/ariaTree data in messages in-place.
+ *
+ * Strategy:
+ * - Keep only the 2 most recent vision results (screenshots OR vision action tools like click/type/etc)
+ * - Keep only the 1 most recent ariaTree (replace older ones with placeholder)
+ *
+ * @param messages - The messages array to modify in-place
+ * @returns Number of items compressed
+ */
+export function processMessages(messages) {
+    let compressedCount = 0;
+    // Find indices of all vision-related tool results (screenshots + vision actions)
+    // and ariaTree results
+    const visionIndices = [];
+    const ariaTreeIndices = [];
+    for (let i = 0; i < messages.length; i++) {
+        const message = messages[i];
+        if (isToolMessage(message)) {
+            const content = message.content;
+            if (content.some(isVisionPart)) {
+                visionIndices.push(i);
+            }
+            if (content.some(isAriaTreePart)) {
+                ariaTreeIndices.push(i);
+            }
+        }
+    }
+    // Compress old vision results (keep 2 most recent across all vision tools)
+    if (visionIndices.length > 2) {
+        const toCompress = visionIndices.slice(0, visionIndices.length - 2);
+        for (const index of toCompress) {
+            const message = messages[index];
+            if (isToolMessage(message)) {
+                // Both functions are safe to call - they only modify their respective part types
+                compressScreenshotMessage(message);
+                compressVisionActionMessage(message);
+                compressedCount++;
+            }
+        }
+    }
+    // Compress old ariaTree results (keep 1 most recent)
+    if (ariaTreeIndices.length > 1) {
+        const toCompress = ariaTreeIndices.slice(0, ariaTreeIndices.length - 1);
+        for (const idx of toCompress) {
+            const message = messages[idx];
+            if (isToolMessage(message)) {
+                compressAriaTreeMessage(message);
+                compressedCount++;
+            }
+        }
+    }
+    return compressedCount;
+}
+/**
+ * Check if output has type "content" (array-based value format).
+ * Only outputs with type "content" should have array values.
+ */
+function isContentTypeOutput(output) {
+    return output.type === "content";
+}
+/**
+ * Compress screenshot message content in-place.
+ * Only modifies outputs with type "content" to maintain schema validity.
+ * Replaces entire output object to ensure type/value consistency.
+ */
+function compressScreenshotMessage(message) {
+    for (const part of message.content) {
+        if (isScreenshotPart(part)) {
+            const typedPart = part;
+            // Only compress if output exists and has type "content"
+            if (typedPart.output && isContentTypeOutput(typedPart.output)) {
+                // Replace entire output to ensure type/value consistency
+                typedPart.output = {
+                    type: "content",
+                    value: [{ type: "text", text: "screenshot taken" }],
+                };
+            }
+        }
+    }
+}
+/**
+ * Compress vision action message content in-place by removing the screenshot
+ * but keeping the action result text.
+ * Only modifies outputs with type "content" to maintain schema validity.
+ */
+function compressVisionActionMessage(message) {
+    for (const part of message.content) {
+        if (isVisionActionPart(part)) {
+            const typedPart = part;
+            // Only compress if output is type "content" (array-based value)
+            if (typedPart.output &&
+                isContentTypeOutput(typedPart.output) &&
+                Array.isArray(typedPart.output.value)) {
+                // Filter out media content but keep text results
+                const filteredValue = typedPart.output.value.filter((item) => item && typeof item === "object" && item.type !== "media");
+                // Replace entire output to ensure type/value consistency
+                typedPart.output = {
+                    type: "content",
+                    value: filteredValue,
+                };
+            }
+        }
+    }
+}
+/**
+ * Compress ariaTree message content in-place.
+ * Only modifies outputs with type "content" to maintain schema validity.
+ * Replaces entire output object to ensure type/value consistency.
+ */
+function compressAriaTreeMessage(message) {
+    for (const part of message.content) {
+        if (isAriaTreePart(part)) {
+            const typedPart = part;
+            // Only compress if output exists and has type "content"
+            if (typedPart.output && isContentTypeOutput(typedPart.output)) {
+                typedPart.output = {
+                    type: "content",
+                    value: [
+                        {
+                            type: "text",
+                            text: "ARIA tree extracted for context of page elements",
+                        },
+                    ],
+                };
+            }
+        }
+    }
+}
+//# sourceMappingURL=messageProcessing.js.map

package/dist/esm/lib/v3/agent/utils/messageProcessing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"messageProcessing.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/messageProcessing.ts"],"names":[],"mappings":"AAEA,gEAAgE;AAChE,MAAM,mBAAmB,GAAG;IAC1B,OAAO;IACP,MAAM;IACN,aAAa;IACb,MAAM;IACN,gBAAgB;IAChB,QAAQ;CACT,CAAC;AAEF,SAAS,aAAa,CACpB,OAAgB;IAEhB,OAAO,CACL,CAAC,CAAC,OAAO;QACT,OAAO,OAAO,KAAK,QAAQ;QAC1B,OAA8B,CAAC,IAAI,KAAK,MAAM;QAC/C,KAAK,CAAC,OAAO,CAAE,OAAiC,CAAC,OAAO,CAAC,CAC1D,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa;IACrC,OAAO,CACL,CAAC,CAAC,IAAI;QACN,OAAO,IAAI,KAAK,QAAQ;QACvB,IAA+B,CAAC,QAAQ,KAAK,YAAY,CAC3D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAa;IACvC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACpD,MAAM,QAAQ,GAAI,IAA+B,CAAC,QAAQ,CAAC;IAC3D,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,mBAAmB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,YAAY,CAAC,IAAa;IACjC,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,cAAc,CAAC,IAAa;IACnC,OAAO,CACL,CAAC,CAAC,IAAI;QACN,OAAO,IAAI,KAAK,QAAQ;QACvB,IAA+B,CAAC,QAAQ,KAAK,UAAU,CACzD,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAAC,QAAwB;IACtD,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,iFAAiF;IACjF,uBAAuB;IACvB,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAoB,CAAC;YAC7C,IAAI,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/B,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;gBACjC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,iFAAiF;gBACjF,yBAAyB,CAAC,OAAO,CAAC,CAAC;gBACnC,2BAA2B,CAAC,OAAO,CAAC,CAAC;gBACrC,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACxE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,uBAAuB,CAAC,OAAO,CAAC,CAAC;gBACjC,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAkBD;;;GAGG;AACH,SAAS,mBAAmB,CAAC,MAG5B;IACC,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC;AACnC,CAAC;AAED;;;;GAIG;AACH,SAAS,yBAAyB,CAAC,OAGlC;IACC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,IAAsB,CAAC;YACzC,wDAAwD;YACxD,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9D,yDAAyD;gBACzD,SAAS,CAAC,MAAM,GAAG;oBACjB,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;iBACpD,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,2BAA2B,CAAC,OAGpC;IACC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAsB,CAAC;YAEzC,gEAAgE;YAChE,IACE,SAAS,CAAC,MAAM;gBAChB,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC;gBACrC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EACrC,CAAC;gBACD,iDAAiD;gBACjD,MAAM,aAAa,GACjB,SAAS,CAAC,MAAM,CAAC,KAClB,CAAC,MAAM,CACN,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CACpE,CAAC;gBACF,yDAAyD;gBACzD,SAAS,CAAC,MAAM,GAAG;oBACjB,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,aAAa;iBACrB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,OAGhC;IACC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,IAAsB,CAAC;YACzC,wDAAwD;YACxD,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9D,SAAS,CAAC,MAAM,GAAG;oBACjB,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,kDAAkD;yBACzD;qBACF;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC","sourcesContent":["import type { ModelMessage } from \"ai\";\n\n// Vision action tools that include screenshots in their results\nconst VISION_ACTION_TOOLS = [\n  \"click\",\n  \"type\",\n  \"dragAndDrop\",\n  \"wait\",\n  \"fillFormVision\",\n  \"scroll\",\n];\n\nfunction isToolMessage(\n  message: unknown,\n): message is { role: \"tool\"; content: unknown[] } {\n  return (\n    !!message &&\n    typeof message === \"object\" &&\n    (message as { role?: unknown }).role === \"tool\" &&\n    Array.isArray((message as { content?: unknown }).content)\n  );\n}\n\nfunction isScreenshotPart(part: unknown): boolean {\n  return (\n    !!part &&\n    typeof part === \"object\" &&\n    (part as { toolName?: unknown }).toolName === \"screenshot\"\n  );\n}\n\nfunction isVisionActionPart(part: unknown): boolean {\n  if (!part || typeof part !== \"object\") return false;\n  const toolName = (part as { toolName?: unknown }).toolName;\n  return typeof toolName === \"string\" && VISION_ACTION_TOOLS.includes(toolName);\n}\n\nfunction isVisionPart(part: unknown): boolean {\n  return isScreenshotPart(part) || isVisionActionPart(part);\n}\n\nfunction isAriaTreePart(part: unknown): boolean {\n  return (\n    !!part &&\n    typeof part === \"object\" &&\n    (part as { toolName?: unknown }).toolName === \"ariaTree\"\n  );\n}\n\n/**\n * Compress old screenshot/ariaTree data in messages in-place.\n *\n * Strategy:\n * - Keep only the 2 most recent vision results (screenshots OR vision action tools like click/type/etc)\n * - Keep only the 1 most recent ariaTree (replace older ones with placeholder)\n *\n * @param messages - The messages array to modify in-place\n * @returns Number of items compressed\n */\nexport function processMessages(messages: ModelMessage[]): number {\n  let compressedCount = 0;\n\n  // Find indices of all vision-related tool results (screenshots + vision actions)\n  // and ariaTree results\n  const visionIndices: number[] = [];\n  const ariaTreeIndices: number[] = [];\n\n  for (let i = 0; i < messages.length; i++) {\n    const message = messages[i];\n    if (isToolMessage(message)) {\n      const content = message.content as unknown[];\n      if (content.some(isVisionPart)) {\n        visionIndices.push(i);\n      }\n      if (content.some(isAriaTreePart)) {\n        ariaTreeIndices.push(i);\n      }\n    }\n  }\n\n  // Compress old vision results (keep 2 most recent across all vision tools)\n  if (visionIndices.length > 2) {\n    const toCompress = visionIndices.slice(0, visionIndices.length - 2);\n    for (const index of toCompress) {\n      const message = messages[index];\n      if (isToolMessage(message)) {\n        // Both functions are safe to call - they only modify their respective part types\n        compressScreenshotMessage(message);\n        compressVisionActionMessage(message);\n        compressedCount++;\n      }\n    }\n  }\n\n  // Compress old ariaTree results (keep 1 most recent)\n  if (ariaTreeIndices.length > 1) {\n    const toCompress = ariaTreeIndices.slice(0, ariaTreeIndices.length - 1);\n    for (const idx of toCompress) {\n      const message = messages[idx];\n      if (isToolMessage(message)) {\n        compressAriaTreeMessage(message);\n        compressedCount++;\n      }\n    }\n  }\n\n  return compressedCount;\n}\n\n/**\n * Tool result part structure from AI SDK.\n * The output field uses a discriminated union - type determines value format:\n * - type: \"content\" -> value: Array<{type: \"text\", ...} | {type: \"media\", ...}>\n * - type: \"text\" -> value: string\n * - type: \"json\" -> value: JSONValue\n * - type: \"error-text\" -> value: string\n * - type: \"error-json\" -> value: JSONValue\n */\ninterface ToolResultPart {\n  output?: {\n    type: string;\n    value?: unknown;\n  };\n}\n\n/**\n * Check if output has type \"content\" (array-based value format).\n * Only outputs with type \"content\" should have array values.\n */\nfunction isContentTypeOutput(output: {\n  type: string;\n  value?: unknown;\n}): boolean {\n  return output.type === \"content\";\n}\n\n/**\n * Compress screenshot message content in-place.\n * Only modifies outputs with type \"content\" to maintain schema validity.\n * Replaces entire output object to ensure type/value consistency.\n */\nfunction compressScreenshotMessage(message: {\n  role: \"tool\";\n  content: unknown[];\n}): void {\n  for (const part of message.content) {\n    if (isScreenshotPart(part)) {\n      const typedPart = part as ToolResultPart;\n      // Only compress if output exists and has type \"content\"\n      if (typedPart.output && isContentTypeOutput(typedPart.output)) {\n        // Replace entire output to ensure type/value consistency\n        typedPart.output = {\n          type: \"content\",\n          value: [{ type: \"text\", text: \"screenshot taken\" }],\n        };\n      }\n    }\n  }\n}\n\n/**\n * Compress vision action message content in-place by removing the screenshot\n * but keeping the action result text.\n * Only modifies outputs with type \"content\" to maintain schema validity.\n */\nfunction compressVisionActionMessage(message: {\n  role: \"tool\";\n  content: unknown[];\n}): void {\n  for (const part of message.content) {\n    if (isVisionActionPart(part)) {\n      const typedPart = part as ToolResultPart;\n\n      // Only compress if output is type \"content\" (array-based value)\n      if (\n        typedPart.output &&\n        isContentTypeOutput(typedPart.output) &&\n        Array.isArray(typedPart.output.value)\n      ) {\n        // Filter out media content but keep text results\n        const filteredValue = (\n          typedPart.output.value as Array<{ type?: string }>\n        ).filter(\n          (item) => item && typeof item === \"object\" && item.type !== \"media\",\n        );\n        // Replace entire output to ensure type/value consistency\n        typedPart.output = {\n          type: \"content\",\n          value: filteredValue,\n        };\n      }\n    }\n  }\n}\n\n/**\n * Compress ariaTree message content in-place.\n * Only modifies outputs with type \"content\" to maintain schema validity.\n * Replaces entire output object to ensure type/value consistency.\n */\nfunction compressAriaTreeMessage(message: {\n  role: \"tool\";\n  content: unknown[];\n}): void {\n  for (const part of message.content) {\n    if (isAriaTreePart(part)) {\n      const typedPart = part as ToolResultPart;\n      // Only compress if output exists and has type \"content\"\n      if (typedPart.output && isContentTypeOutput(typedPart.output)) {\n        typedPart.output = {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: \"ARIA tree extracted for context of page elements\",\n            },\n          ],\n        };\n      }\n    }\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/utils/screenshotHandler.d.ts

@@ -0,0 +1,10 @@
+import type { Page } from "../../understudy/page.js";
+/**
+ * Waits for the page to settle and captures a screenshot.
+ * If the screenshot fails (e.g., page closed, navigation in progress),
+ * returns undefined instead of throwing - allowing the action to still succeed.
+ *
+ * @param page - The page to capture
+ * @param delayMs - Delay before capturing (default: 500ms, pass 0 to skip delay)
+ */
+export declare function waitAndCaptureScreenshot(page: Page, delayMs?: number): Promise<string | undefined>;

package/dist/esm/lib/v3/agent/utils/screenshotHandler.js

@@ -0,0 +1,26 @@
+/**
+ * Default delay in milliseconds to wait after vision actions before capturing screenshot.
+ * Allows the page to settle after interactions.
+ */
+const DEFAULT_DELAY_MS = 500;
+/**
+ * Waits for the page to settle and captures a screenshot.
+ * If the screenshot fails (e.g., page closed, navigation in progress),
+ * returns undefined instead of throwing - allowing the action to still succeed.
+ *
+ * @param page - The page to capture
+ * @param delayMs - Delay before capturing (default: 500ms, pass 0 to skip delay)
+ */
+export async function waitAndCaptureScreenshot(page, delayMs = DEFAULT_DELAY_MS) {
+    if (delayMs > 0) {
+        await page.waitForTimeout(delayMs);
+    }
+    try {
+        const buffer = await page.screenshot({ fullPage: false });
+        return buffer.toString("base64");
+    }
+    catch {
+        return undefined;
+    }
+}
+//# sourceMappingURL=screenshotHandler.js.map

package/dist/esm/lib/v3/agent/utils/screenshotHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshotHandler.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/screenshotHandler.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,IAAU,EACV,UAAkB,gBAAgB;IAElC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC","sourcesContent":["import type { Page } from \"../../understudy/page.js\";\n\n/**\n * Default delay in milliseconds to wait after vision actions before capturing screenshot.\n * Allows the page to settle after interactions.\n */\nconst DEFAULT_DELAY_MS = 500;\n\n/**\n * Waits for the page to settle and captures a screenshot.\n * If the screenshot fails (e.g., page closed, navigation in progress),\n * returns undefined instead of throwing - allowing the action to still succeed.\n *\n * @param page - The page to capture\n * @param delayMs - Delay before capturing (default: 500ms, pass 0 to skip delay)\n */\nexport async function waitAndCaptureScreenshot(\n  page: Page,\n  delayMs: number = DEFAULT_DELAY_MS,\n): Promise<string | undefined> {\n  if (delayMs > 0) {\n    await page.waitForTimeout(delayMs);\n  }\n\n  try {\n    const buffer = await page.screenshot({ fullPage: false });\n    return buffer.toString(\"base64\");\n  } catch {\n    return undefined;\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/utils/validateExperimentalFeatures.d.ts

@@ -0,0 +1,26 @@
+import type { AgentConfig, AgentExecuteOptionsBase } from "../../types/public/index.js";
+export interface AgentValidationOptions {
+    /** Whether experimental mode is enabled */
+    isExperimental: boolean;
+    /** Agent config options (integrations, tools, stream, cua, etc.) */
+    agentConfig?: Partial<AgentConfig>;
+    /** Execute options (callbacks, signal, messages, etc.) */
+    executeOptions?: (Partial<AgentExecuteOptionsBase> & {
+        callbacks?: unknown;
+    }) | null;
+    /** Whether this is streaming mode (can be derived from agentConfig.stream) */
+    isStreaming?: boolean;
+}
+/**
+ * Validates agent configuration and experimental feature usage.
+ *
+ * This utility consolidates all validation checks for both CUA and non-CUA agent paths:
+ * - Invalid argument errors for CUA (streaming, abort signal, message continuation, excludeTools, output schema are not supported)
+ * - Experimental feature checks for integrations and tools (both CUA and non-CUA)
+ * - Experimental feature checks for hybrid mode (requires experimental: true)
+ * - Experimental feature checks for non-CUA only (callbacks, signal, messages, streaming, excludeTools, output schema)
+ *
+ * Throws StagehandInvalidArgumentError for invalid/unsupported configurations.
+ * Throws ExperimentalNotConfiguredError if experimental features are used without experimental mode.
+ */
+export declare function validateExperimentalFeatures(options: AgentValidationOptions): void;

package/dist/esm/lib/v3/agent/utils/validateExperimentalFeatures.js

@@ -0,0 +1,79 @@
+import { ExperimentalNotConfiguredError, StagehandInvalidArgumentError, } from "../../types/public/sdkErrors.js";
+/**
+ * Validates agent configuration and experimental feature usage.
+ *
+ * This utility consolidates all validation checks for both CUA and non-CUA agent paths:
+ * - Invalid argument errors for CUA (streaming, abort signal, message continuation, excludeTools, output schema are not supported)
+ * - Experimental feature checks for integrations and tools (both CUA and non-CUA)
+ * - Experimental feature checks for hybrid mode (requires experimental: true)
+ * - Experimental feature checks for non-CUA only (callbacks, signal, messages, streaming, excludeTools, output schema)
+ *
+ * Throws StagehandInvalidArgumentError for invalid/unsupported configurations.
+ * Throws ExperimentalNotConfiguredError if experimental features are used without experimental mode.
+ */
+export function validateExperimentalFeatures(options) {
+    const { isExperimental, agentConfig, executeOptions, isStreaming } = options;
+    // Check if CUA mode is enabled (via mode: "cua" or deprecated cua: true)
+    const isCuaMode = agentConfig?.mode !== undefined
+        ? agentConfig.mode === "cua"
+        : agentConfig?.cua === true;
+    // CUA-specific validation: certain features are not available at all
+    if (isCuaMode) {
+        const unsupportedFeatures = [];
+        if (agentConfig?.stream) {
+            unsupportedFeatures.push("streaming");
+        }
+        if (executeOptions?.signal) {
+            unsupportedFeatures.push("abort signal");
+        }
+        if (executeOptions?.messages) {
+            unsupportedFeatures.push("message continuation");
+        }
+        if (executeOptions?.excludeTools &&
+            executeOptions.excludeTools.length > 0) {
+            unsupportedFeatures.push("excludeTools");
+        }
+        if (executeOptions?.output) {
+            unsupportedFeatures.push("output schema");
+        }
+        if (unsupportedFeatures.length > 0) {
+            throw new StagehandInvalidArgumentError(`${unsupportedFeatures.join(", ")} ${unsupportedFeatures.length === 1 ? "is" : "are"} not supported with CUA (Computer Use Agent) mode.`);
+        }
+    }
+    // Skip experimental checks if already in experimental mode
+    if (isExperimental)
+        return;
+    const features = [];
+    // Check agent config features (check array length to avoid false positives for empty arrays)
+    const hasIntegrations = agentConfig?.integrations && agentConfig.integrations.length > 0;
+    const hasTools = agentConfig?.tools && Object.keys(agentConfig.tools).length > 0;
+    if (hasIntegrations || hasTools) {
+        features.push("MCP integrations and custom tools");
+    }
+    // Check streaming mode (either explicit or derived from config) - only for non-CUA
+    if (!isCuaMode && (isStreaming || agentConfig?.stream)) {
+        features.push("streaming");
+    }
+    // Check execute options features - only for non-CUA
+    if (executeOptions && !isCuaMode) {
+        if (executeOptions.callbacks) {
+            features.push("callbacks");
+        }
+        if (executeOptions.signal) {
+            features.push("abort signal");
+        }
+        if (executeOptions.messages) {
+            features.push("message continuation");
+        }
+        if (executeOptions.excludeTools && executeOptions.excludeTools.length > 0) {
+            features.push("excludeTools");
+        }
+        if (executeOptions.output) {
+            features.push("output schema");
+        }
+    }
+    if (features.length > 0) {
+        throw new ExperimentalNotConfiguredError(`Agent ${features.join(", ")}`);
+    }
+}
+//# sourceMappingURL=validateExperimentalFeatures.js.map

package/dist/esm/lib/v3/agent/utils/validateExperimentalFeatures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validateExperimentalFeatures.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/validateExperimentalFeatures.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,8BAA8B,EAC9B,6BAA6B,GAC9B,MAAM,iCAAiC,CAAC;AAmBzC;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAA+B;IAE/B,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IAE7E,yEAAyE;IACzE,MAAM,SAAS,GACb,WAAW,EAAE,IAAI,KAAK,SAAS;QAC7B,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,KAAK;QAC5B,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,IAAI,CAAC;IAEhC,qEAAqE;IACrE,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,mBAAmB,GAAa,EAAE,CAAC;QAEzC,IAAI,WAAW,EAAE,MAAM,EAAE,CAAC;YACxB,mBAAmB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,cAAc,EAAE,MAAM,EAAE,CAAC;YAC3B,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,cAAc,EAAE,QAAQ,EAAE,CAAC;YAC7B,mBAAmB,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnD,CAAC;QACD,IACE,cAAc,EAAE,YAAY;YAC5B,cAAc,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EACtC,CAAC;YACD,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,cAAc,EAAE,MAAM,EAAE,CAAC;YAC3B,mBAAmB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,6BAA6B,CACrC,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,oDAAoD,CACzI,CAAC;QACJ,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,IAAI,cAAc;QAAE,OAAO;IAE3B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,6FAA6F;IAC7F,MAAM,eAAe,GACnB,WAAW,EAAE,YAAY,IAAI,WAAW,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;IACnE,MAAM,QAAQ,GACZ,WAAW,EAAE,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,eAAe,IAAI,QAAQ,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IACrD,CAAC;IAED,mFAAmF;IACnF,IAAI,CAAC,SAAS,IAAI,CAAC,WAAW,IAAI,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC;QACvD,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7B,CAAC;IAED,oDAAoD;IACpD,IAAI,cAAc,IAAI,CAAC,SAAS,EAAE,CAAC;QACjC,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC;YAC7B,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,cAAc,CAAC,QAAQ,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,cAAc,CAAC,YAAY,IAAI,cAAc,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1E,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,8BAA8B,CAAC,SAAS,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC","sourcesContent":["import {\n  ExperimentalNotConfiguredError,\n  StagehandInvalidArgumentError,\n} from \"../../types/public/sdkErrors.js\";\nimport type {\n  AgentConfig,\n  AgentExecuteOptionsBase,\n} from \"../../types/public/index.js\";\n\nexport interface AgentValidationOptions {\n  /** Whether experimental mode is enabled */\n  isExperimental: boolean;\n  /** Agent config options (integrations, tools, stream, cua, etc.) */\n  agentConfig?: Partial<AgentConfig>;\n  /** Execute options (callbacks, signal, messages, etc.) */\n  executeOptions?:\n    | (Partial<AgentExecuteOptionsBase> & { callbacks?: unknown })\n    | null;\n  /** Whether this is streaming mode (can be derived from agentConfig.stream) */\n  isStreaming?: boolean;\n}\n\n/**\n * Validates agent configuration and experimental feature usage.\n *\n * This utility consolidates all validation checks for both CUA and non-CUA agent paths:\n * - Invalid argument errors for CUA (streaming, abort signal, message continuation, excludeTools, output schema are not supported)\n * - Experimental feature checks for integrations and tools (both CUA and non-CUA)\n * - Experimental feature checks for hybrid mode (requires experimental: true)\n * - Experimental feature checks for non-CUA only (callbacks, signal, messages, streaming, excludeTools, output schema)\n *\n * Throws StagehandInvalidArgumentError for invalid/unsupported configurations.\n * Throws ExperimentalNotConfiguredError if experimental features are used without experimental mode.\n */\nexport function validateExperimentalFeatures(\n  options: AgentValidationOptions,\n): void {\n  const { isExperimental, agentConfig, executeOptions, isStreaming } = options;\n\n  // Check if CUA mode is enabled (via mode: \"cua\" or deprecated cua: true)\n  const isCuaMode =\n    agentConfig?.mode !== undefined\n      ? agentConfig.mode === \"cua\"\n      : agentConfig?.cua === true;\n\n  // CUA-specific validation: certain features are not available at all\n  if (isCuaMode) {\n    const unsupportedFeatures: string[] = [];\n\n    if (agentConfig?.stream) {\n      unsupportedFeatures.push(\"streaming\");\n    }\n    if (executeOptions?.signal) {\n      unsupportedFeatures.push(\"abort signal\");\n    }\n    if (executeOptions?.messages) {\n      unsupportedFeatures.push(\"message continuation\");\n    }\n    if (\n      executeOptions?.excludeTools &&\n      executeOptions.excludeTools.length > 0\n    ) {\n      unsupportedFeatures.push(\"excludeTools\");\n    }\n    if (executeOptions?.output) {\n      unsupportedFeatures.push(\"output schema\");\n    }\n\n    if (unsupportedFeatures.length > 0) {\n      throw new StagehandInvalidArgumentError(\n        `${unsupportedFeatures.join(\", \")} ${unsupportedFeatures.length === 1 ? \"is\" : \"are\"} not supported with CUA (Computer Use Agent) mode.`,\n      );\n    }\n  }\n\n  // Skip experimental checks if already in experimental mode\n  if (isExperimental) return;\n\n  const features: string[] = [];\n\n  // Check agent config features (check array length to avoid false positives for empty arrays)\n  const hasIntegrations =\n    agentConfig?.integrations && agentConfig.integrations.length > 0;\n  const hasTools =\n    agentConfig?.tools && Object.keys(agentConfig.tools).length > 0;\n  if (hasIntegrations || hasTools) {\n    features.push(\"MCP integrations and custom tools\");\n  }\n\n  // Check streaming mode (either explicit or derived from config) - only for non-CUA\n  if (!isCuaMode && (isStreaming || agentConfig?.stream)) {\n    features.push(\"streaming\");\n  }\n\n  // Check execute options features - only for non-CUA\n  if (executeOptions && !isCuaMode) {\n    if (executeOptions.callbacks) {\n      features.push(\"callbacks\");\n    }\n    if (executeOptions.signal) {\n      features.push(\"abort signal\");\n    }\n    if (executeOptions.messages) {\n      features.push(\"message continuation\");\n    }\n    if (executeOptions.excludeTools && executeOptions.excludeTools.length > 0) {\n      features.push(\"excludeTools\");\n    }\n    if (executeOptions.output) {\n      features.push(\"output schema\");\n    }\n  }\n\n  if (features.length > 0) {\n    throw new ExperimentalNotConfiguredError(`Agent ${features.join(\", \")}`);\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/utils/xpath.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Utility functions for XPath handling in agent tools.
+ */
+/**
+ * Ensures a value is properly formatted as an XPath selector.
+ * Returns null if the value is not a valid string.
+ *
+ * @param value - The value to normalize as an XPath
+ * @returns The normalized XPath string prefixed with "xpath=" or null
+ */
+export declare function ensureXPath(value: unknown): string | null;

package/dist/esm/lib/v3/agent/utils/xpath.js

@@ -0,0 +1,19 @@
+/**
+ * Utility functions for XPath handling in agent tools.
+ */
+/**
+ * Ensures a value is properly formatted as an XPath selector.
+ * Returns null if the value is not a valid string.
+ *
+ * @param value - The value to normalize as an XPath
+ * @returns The normalized XPath string prefixed with "xpath=" or null
+ */
+export function ensureXPath(value) {
+    if (typeof value !== "string")
+        return null;
+    const trimmed = value.trim();
+    if (!trimmed)
+        return null;
+    return trimmed.startsWith("xpath=") ? trimmed : `xpath=${trimmed}`;
+}
+//# sourceMappingURL=xpath.js.map

package/dist/esm/lib/v3/agent/utils/xpath.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"xpath.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/xpath.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,OAAO,EAAE,CAAC;AACrE,CAAC","sourcesContent":["/**\n * Utility functions for XPath handling in agent tools.\n */\n\n/**\n * Ensures a value is properly formatted as an XPath selector.\n * Returns null if the value is not a valid string.\n *\n * @param value - The value to normalize as an XPath\n * @returns The normalized XPath string prefixed with \"xpath=\" or null\n */\nexport function ensureXPath(value: unknown): string | null {\n  if (typeof value !== \"string\") return null;\n  const trimmed = value.trim();\n  if (!trimmed) return null;\n  return trimmed.startsWith(\"xpath=\") ? trimmed : `xpath=${trimmed}`;\n}\n"]}

package/dist/esm/lib/v3/api.d.ts

@@ -0,0 +1,86 @@
+import type { Action, ActResult, AgentConfig, AgentExecuteOptions, AgentResult, ExtractResult, LogLine, StagehandMetrics, ActOptions, ExtractOptions, ObserveOptions, Api } from "./types/public/index.js";
+import type { SerializableResponse, AgentCacheTransferPayload } from "./types/private/index.js";
+import type { StagehandZodSchema } from "./zodCompat.js";
+/**
+ * Constructor parameters for StagehandAPIClient
+ */
+interface StagehandAPIConstructorParams {
+    apiKey: string;
+    projectId: string;
+    logger: (message: LogLine) => void;
+}
+/**
+ * Parameters for starting a session via the API client.
+ * Extends Api.SessionStartRequest with client-specific field (modelApiKey).
+ *
+ * Wire format: Api.SessionStartRequest (modelApiKey sent via header, not body)
+ */
+interface ClientSessionStartParams extends Api.SessionStartRequest {
+    /** Model API key - sent via x-model-api-key header, not in request body */
+    modelApiKey: string;
+}
+/**
+ * Client parameters for act() method.
+ * Derives structure from Api.ActRequest but uses SDK's ActOptions (which includes `page`).
+ * Before serialization, `page` is stripped to produce Api.ActRequest wire format.
+ */
+interface ClientActParameters {
+    input: Api.ActRequest["input"];
+    options?: ActOptions;
+    frameId?: Api.ActRequest["frameId"];
+}
+/**
+ * Client parameters for extract() method.
+ * Derives structure from Api.ExtractRequest but uses SDK's ExtractOptions (which includes `page`)
+ * and accepts Zod schema (converted to JSON schema for wire format).
+ */
+interface ClientExtractParameters {
+    instruction?: Api.ExtractRequest["instruction"];
+    schema?: StagehandZodSchema;
+    options?: ExtractOptions;
+    frameId?: Api.ExtractRequest["frameId"];
+}
+/**
+ * Client parameters for observe() method.
+ * Derives structure from Api.ObserveRequest but uses SDK's ObserveOptions (which includes `page`).
+ * Before serialization, `page` is stripped to produce Api.ObserveRequest wire format.
+ */
+interface ClientObserveParameters {
+    instruction?: Api.ObserveRequest["instruction"];
+    options?: ObserveOptions;
+    frameId?: Api.ObserveRequest["frameId"];
+}
+export declare class StagehandAPIClient {
+    private apiKey;
+    private projectId;
+    private sessionId?;
+    private modelApiKey;
+    private modelProvider?;
+    private logger;
+    private fetchWithCookies;
+    private lastFinishedEventData;
+    private latestAgentCacheEntry;
+    constructor({ apiKey, projectId, logger }: StagehandAPIConstructorParams);
+    init({ modelName, modelApiKey, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, browserbaseSessionCreateParams, browserbaseSessionID, }: ClientSessionStartParams): Promise<Api.SessionStartResult>;
+    act({ input, options, frameId, }: ClientActParameters): Promise<ActResult>;
+    extract<T extends StagehandZodSchema>({ instruction, schema: zodSchema, options, frameId, }: ClientExtractParameters): Promise<ExtractResult<T>>;
+    observe({ instruction, options, frameId, }: ClientObserveParameters): Promise<Action[]>;
+    goto(url: string, options?: Api.NavigateRequest["options"], frameId?: string): Promise<SerializableResponse | null>;
+    agentExecute(agentConfig: AgentConfig, executeOptions: AgentExecuteOptions | string, frameId?: string, shouldCache?: boolean): Promise<AgentResult>;
+    consumeLatestAgentCacheEntry(): AgentCacheTransferPayload | null;
+    end(): Promise<Response>;
+    getReplayMetrics(): Promise<StagehandMetrics>;
+    /**
+     * Prepares a model configuration for the API payload by ensuring the `apiKey`
+     * is included. If the model is passed as a string, converts it to an object
+     * with `modelName` and `apiKey`.
+     *
+     * In API mode, we only attempt to load an API key from env vars when the
+     * model provider differs from the one used to init the session.
+     */
+    private prepareModelConfig;
+    private consumeFinishedEventData;
+    private execute;
+    private request;
+}
+export {};