@browserbasehq/orca 3.0.9-alpha-4 → 3.1.0-headers

package/dist/esm/lib/v3/agent/utils/actionMapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actionMapping.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/actionMapping.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,CAAC,kBAAkB,CAAU,CAAC;AAE3D;;GAEG;AACH,SAAS,iBAAiB,CACxB,MAA+B;IAE/B,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IACE,CAAC,oBAAoB,CAAC,QAAQ,CAC5B,GAA4C,CAC7C,EACD,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,EACrC,YAAY,EACZ,UAAU,EACV,IAAI,EACJ,SAAS,GACY;IACrB,QAAQ,YAAY,EAAE,CAAC;QACrB,KAAK,KAAK;YACR,OAAO,gBAAgB,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QACvD,KAAK,UAAU;YACb,OAAO,qBAAqB,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QAC5D;YACE,OAAO,CAAC,oBAAoB,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACvB,UAAmB,EACnB,IAA6B,EAC7B,SAAkB;IAElB,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClD,OAAO,CAAC,oBAAoB,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,MAAM,GAAG,UAAqC,CAAC;IAErD,qDAAqD;IACrD,MAAM,MAAM,GAAI,MAAM,CAAC,MAAkC,IAAI,MAAM,CAAC;IAEpE,6CAA6C;IAC7C,MAAM,MAAM,GAAgB;QAC1B,IAAI,EAAE,KAAK;QACX,SAAS;QACT,aAAa,EAAE,KAAK;QACpB,GAAG,IAAI;KACR,CAAC;IAEF,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,MAAM,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAC1D,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,qBAAqB,CAC5B,UAAmB,EACnB,IAA6B,EAC7B,SAAkB;IAElB,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClD,OAAO,CAAC,oBAAoB,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,UAAqC,CAAC;IAErD,qDAAqD;IACrD,MAAM,MAAM,GAAI,MAAM,CAAC,MAAkC,IAAI,MAAM,CAAC;IAEpE,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC;QAC/D,CAAC,CAAC,MAAM,CAAC,mBAAmB;QAC5B,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,UAAU;QAChB,SAAS;QACT,aAAa,EAAE,KAAK;QACpB,GAAG,IAAI;KACR,CAAC,CAAC;IAEH,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,2BAA2B;YACtC,aAAa,EAAE,KAAK;YACpB,mBAAmB,EAAE,aAAa;SACnC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,oBAAoB,CAC3B,YAAoB,EACpB,UAAmB,EACnB,IAA6B,EAC7B,SAAkB;IAElB,MAAM,MAAM,GAAgB;QAC1B,IAAI,EAAE,YAAY;QAClB,SAAS;QACT,aAAa,EACX,YAAY,KAAK,MAAM,CAAC,CAAC,CAAE,IAAI,EAAE,YAAwB,CAAC,CAAC,CAAC,KAAK;QACnE,GAAG,IAAI;KACR,CAAC;IAEF,qFAAqF;IACrF,gFAAgF;IAChF,IAAI,YAAY,KAAK,YAAY,EAAE,CAAC;QAClC,MAAM,CAAC,MAAM,GAAG,iBAAiB,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,sDAAsD;IACtD,mEAAmE;IACnE,IAAI,YAAY,KAAK,UAAU,IAAI,UAAU,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,UAAkC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAE7B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,aAAa,GAAG,iBAAiB,CACrC,MAAiC,CAClC,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC","sourcesContent":["import { AgentAction } from \"../../types/public/agent.js\";\nimport { ActionMappingOptions } from \"../../types/private/agent.js\";\n\n/**\n * Keys to exclude from tool outputs when mapping to actions.\n * These are large data fields that shouldn't be included in the actions array.\n * Users can access this data through result.messages if needed.\n */\nconst EXCLUDED_OUTPUT_KEYS = [\"screenshotBase64\"] as const;\n\n/**\n * Strips excluded keys (like screenshotBase64) from a tool output object.\n */\nfunction stripExcludedKeys(\n  output: Record<string, unknown>,\n): Record<string, unknown> {\n  const result: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(output)) {\n    if (\n      !EXCLUDED_OUTPUT_KEYS.includes(\n        key as (typeof EXCLUDED_OUTPUT_KEYS)[number],\n      )\n    ) {\n      result[key] = value;\n    }\n  }\n  return result;\n}\n\nexport function mapToolResultToActions({\n  toolCallName,\n  toolResult,\n  args,\n  reasoning,\n}: ActionMappingOptions): AgentAction[] {\n  switch (toolCallName) {\n    case \"act\":\n      return mapActToolResult(toolResult, args, reasoning);\n    case \"fillForm\":\n      return mapFillFormToolResult(toolResult, args, reasoning);\n    default:\n      return [createStandardAction(toolCallName, toolResult, args, reasoning)];\n  }\n}\n\nfunction mapActToolResult(\n  toolResult: unknown,\n  args: Record<string, unknown>,\n  reasoning?: string,\n): AgentAction[] {\n  if (!toolResult || typeof toolResult !== \"object\") {\n    return [createStandardAction(\"act\", toolResult, args, reasoning)];\n  }\n\n  const result = toolResult as Record<string, unknown>;\n\n  // AI SDK wraps the tool result in an output property\n  const output = (result.output as Record<string, unknown>) || result;\n\n  // Extract playwright arguments if they exist\n  const action: AgentAction = {\n    type: \"act\",\n    reasoning,\n    taskCompleted: false,\n    ...args,\n  };\n\n  if (output.playwrightArguments) {\n    action.playwrightArguments = output.playwrightArguments;\n  }\n\n  return [action];\n}\n\nfunction mapFillFormToolResult(\n  toolResult: unknown,\n  args: Record<string, unknown>,\n  reasoning?: string,\n): AgentAction[] {\n  if (!toolResult || typeof toolResult !== \"object\") {\n    return [createStandardAction(\"fillForm\", toolResult, args, reasoning)];\n  }\n\n  const result = toolResult as Record<string, unknown>;\n\n  // AI SDK wraps the tool result in an output property\n  const output = (result.output as Record<string, unknown>) || result;\n\n  const observeResults = Array.isArray(output?.playwrightArguments)\n    ? output.playwrightArguments\n    : [];\n\n  const actions: AgentAction[] = [];\n\n  actions.push({\n    type: \"fillForm\",\n    reasoning,\n    taskCompleted: false,\n    ...args,\n  });\n\n  for (const observeResult of observeResults) {\n    actions.push({\n      type: \"act\",\n      reasoning: \"acting from fillform tool\",\n      taskCompleted: false,\n      playwrightArguments: observeResult,\n    });\n  }\n\n  return actions;\n}\n\nfunction createStandardAction(\n  toolCallName: string,\n  toolResult: unknown,\n  args: Record<string, unknown>,\n  reasoning?: string,\n): AgentAction {\n  const action: AgentAction = {\n    type: toolCallName,\n    reasoning,\n    taskCompleted:\n      toolCallName === \"done\" ? (args?.taskComplete as boolean) : false,\n    ...args,\n  };\n\n  // For screenshot tool, exclude base64 data and just indicate a screenshot was taken,\n  // if somebody really wants the base64 data, they can access it through messages\n  if (toolCallName === \"screenshot\") {\n    action.result = \"screenshotTaken\";\n    return action;\n  }\n\n  // Spread the output from the tool result if it exists\n  // Exclude ariaTree tool result as it is very large and unnecessary\n  if (toolCallName !== \"ariaTree\" && toolResult) {\n    const result = toolResult as { output?: unknown };\n    const output = result.output;\n\n    if (output && typeof output === \"object\" && !Array.isArray(output)) {\n      const cleanedOutput = stripExcludedKeys(\n        output as Record<string, unknown>,\n      );\n      Object.assign(action, cleanedOutput);\n    }\n  }\n\n  return action;\n}\n"]}

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.d.ts

@@ -0,0 +1,13 @@
+import type { V3 } from "../../v3.js";
+export declare function isGoogleProvider(provider?: string): boolean;
+export declare function normalizeGoogleCoordinates(x: number, y: number, viewport: {
+    width: number;
+    height: number;
+}): {
+    x: number;
+    y: number;
+};
+export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3): {
+    x: number;
+    y: number;
+};

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js

@@ -0,0 +1,28 @@
+// Default viewport for advancedStealth mode
+const STEALTH_VIEWPORT = { width: 1288, height: 711 };
+export function isGoogleProvider(provider) {
+    if (!provider)
+        return false;
+    return provider.toLowerCase().includes("google");
+}
+// Google returns coordinates in a 0-1000 range, we need to normalize
+// them to the viewport dimensions
+export function normalizeGoogleCoordinates(x, y, viewport) {
+    const clampedX = Math.min(999, Math.max(0, x));
+    const clampedY = Math.min(999, Math.max(0, y));
+    return {
+        x: Math.floor((clampedX / 1000) * viewport.width),
+        y: Math.floor((clampedY / 1000) * viewport.height),
+    };
+}
+export function processCoordinates(x, y, provider, v3) {
+    if (isGoogleProvider(provider) && v3) {
+        // advancedStealth uses fixed viewport, otherwise use configured viewport
+        const viewport = v3.isAdvancedStealth
+            ? STEALTH_VIEWPORT
+            : v3.configuredViewport;
+        return normalizeGoogleCoordinates(x, y, viewport);
+    }
+    return { x, y };
+}
+//# sourceMappingURL=coordinateNormalization.js.map

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":"AAEA,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AAEtD,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,qEAAqE;AACrE,kCAAkC;AAClC,MAAM,UAAU,0BAA0B,CACxC,CAAS,EACT,CAAS,EACT,QAA2C;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QACjD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,CAAS,EACT,CAAS,EACT,QAAiB,EACjB,EAAO;IAEP,IAAI,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACrC,yEAAyE;QACzE,MAAM,QAAQ,GAAG,EAAE,CAAC,iBAAiB;YACnC,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC;QAC1B,OAAO,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC","sourcesContent":["import type { V3 } from \"../../v3.js\";\n\n// Default viewport for advancedStealth mode\nconst STEALTH_VIEWPORT = { width: 1288, height: 711 };\n\nexport function isGoogleProvider(provider?: string): boolean {\n  if (!provider) return false;\n  return provider.toLowerCase().includes(\"google\");\n}\n\n// Google returns coordinates in a 0-1000 range, we need to normalize\n// them to the viewport dimensions\nexport function normalizeGoogleCoordinates(\n  x: number,\n  y: number,\n  viewport: { width: number; height: number },\n): { x: number; y: number } {\n  const clampedX = Math.min(999, Math.max(0, x));\n  const clampedY = Math.min(999, Math.max(0, y));\n  return {\n    x: Math.floor((clampedX / 1000) * viewport.width),\n    y: Math.floor((clampedY / 1000) * viewport.height),\n  };\n}\n\nexport function processCoordinates(\n  x: number,\n  y: number,\n  provider?: string,\n  v3?: V3,\n): { x: number; y: number } {\n  if (isGoogleProvider(provider) && v3) {\n    // advancedStealth uses fixed viewport, otherwise use configured viewport\n    const viewport = v3.isAdvancedStealth\n      ? STEALTH_VIEWPORT\n      : v3.configuredViewport;\n    return normalizeGoogleCoordinates(x, y, viewport);\n  }\n  return { x, y };\n}\n"]}

package/dist/esm/lib/v3/agent/utils/cuaKeyMapping.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Universal key mapping utility for converting various key representations
+ * to Playwright-compatible key names. Used by all CUA clients and handlers.
+ */
+/**
+ * Maps a key name from various formats to Playwright-compatible format
+ * @param key The key name in any supported format
+ * @returns The Playwright-compatible key name
+ */
+export declare function mapKeyToPlaywright(key: string): string;

package/dist/esm/lib/v3/agent/utils/cuaKeyMapping.js

@@ -0,0 +1,62 @@
+/**
+ * Universal key mapping utility for converting various key representations
+ * to Playwright-compatible key names. Used by all CUA clients and handlers.
+ */
+/**
+ * map of key variations to Playwright key names
+ * This handles keys from both Anthropic and OpenAI CUA APIs
+ */
+const KEY_MAP = {
+    ENTER: "Enter",
+    RETURN: "Enter",
+    ESCAPE: "Escape",
+    ESC: "Escape",
+    BACKSPACE: "Backspace",
+    TAB: "Tab",
+    SPACE: " ",
+    DELETE: "Delete",
+    DEL: "Delete",
+    ARROWUP: "ArrowUp",
+    ARROWDOWN: "ArrowDown",
+    ARROWLEFT: "ArrowLeft",
+    ARROWRIGHT: "ArrowRight",
+    ARROW_UP: "ArrowUp",
+    ARROW_DOWN: "ArrowDown",
+    ARROW_LEFT: "ArrowLeft",
+    ARROW_RIGHT: "ArrowRight",
+    UP: "ArrowUp",
+    DOWN: "ArrowDown",
+    LEFT: "ArrowLeft",
+    RIGHT: "ArrowRight",
+    SHIFT: "Shift",
+    CONTROL: "Control",
+    CTRL: "Control",
+    ALT: "Alt",
+    OPTION: "Alt", // macOS alternative name
+    META: "Meta",
+    COMMAND: "Meta", // macOS
+    CMD: "Meta", // macOS shorthand
+    SUPER: "Meta", // Linux
+    WINDOWS: "Meta", // Windows
+    WIN: "Meta", // Windows shorthand
+    HOME: "Home",
+    END: "End",
+    PAGEUP: "PageUp",
+    PAGEDOWN: "PageDown",
+    PAGE_UP: "PageUp",
+    PAGE_DOWN: "PageDown",
+    PGUP: "PageUp",
+    PGDN: "PageDown",
+};
+/**
+ * Maps a key name from various formats to Playwright-compatible format
+ * @param key The key name in any supported format
+ * @returns The Playwright-compatible key name
+ */
+export function mapKeyToPlaywright(key) {
+    if (!key)
+        return key;
+    const upperKey = key.toUpperCase();
+    return KEY_MAP[upperKey] || key;
+}
+//# sourceMappingURL=cuaKeyMapping.js.map

package/dist/esm/lib/v3/agent/utils/cuaKeyMapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cuaKeyMapping.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/cuaKeyMapping.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;GAGG;AACH,MAAM,OAAO,GAA2B;IACtC,KAAK,EAAE,OAAO;IACd,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,QAAQ;IAChB,GAAG,EAAE,QAAQ;IACb,SAAS,EAAE,WAAW;IACtB,GAAG,EAAE,KAAK;IACV,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,QAAQ;IAChB,GAAG,EAAE,QAAQ;IACb,OAAO,EAAE,SAAS;IAClB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,YAAY;IACxB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,EAAE,EAAE,SAAS;IACb,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,WAAW;IACjB,KAAK,EAAE,YAAY;IACnB,KAAK,EAAE,OAAO;IACd,OAAO,EAAE,SAAS;IAClB,IAAI,EAAE,SAAS;IACf,GAAG,EAAE,KAAK;IACV,MAAM,EAAE,KAAK,EAAE,yBAAyB;IACxC,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,MAAM,EAAE,QAAQ;IACzB,GAAG,EAAE,MAAM,EAAE,kBAAkB;IAC/B,KAAK,EAAE,MAAM,EAAE,QAAQ;IACvB,OAAO,EAAE,MAAM,EAAE,UAAU;IAC3B,GAAG,EAAE,MAAM,EAAE,oBAAoB;IACjC,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,UAAU;IACpB,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,UAAU;IACrB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,UAAU;CACjB,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IACnC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;AAClC,CAAC","sourcesContent":["/**\n * Universal key mapping utility for converting various key representations\n * to Playwright-compatible key names. Used by all CUA clients and handlers.\n */\n\n/**\n * map of key variations to Playwright key names\n * This handles keys from both Anthropic and OpenAI CUA APIs\n */\nconst KEY_MAP: Record<string, string> = {\n  ENTER: \"Enter\",\n  RETURN: \"Enter\",\n  ESCAPE: \"Escape\",\n  ESC: \"Escape\",\n  BACKSPACE: \"Backspace\",\n  TAB: \"Tab\",\n  SPACE: \" \",\n  DELETE: \"Delete\",\n  DEL: \"Delete\",\n  ARROWUP: \"ArrowUp\",\n  ARROWDOWN: \"ArrowDown\",\n  ARROWLEFT: \"ArrowLeft\",\n  ARROWRIGHT: \"ArrowRight\",\n  ARROW_UP: \"ArrowUp\",\n  ARROW_DOWN: \"ArrowDown\",\n  ARROW_LEFT: \"ArrowLeft\",\n  ARROW_RIGHT: \"ArrowRight\",\n  UP: \"ArrowUp\",\n  DOWN: \"ArrowDown\",\n  LEFT: \"ArrowLeft\",\n  RIGHT: \"ArrowRight\",\n  SHIFT: \"Shift\",\n  CONTROL: \"Control\",\n  CTRL: \"Control\",\n  ALT: \"Alt\",\n  OPTION: \"Alt\", // macOS alternative name\n  META: \"Meta\",\n  COMMAND: \"Meta\", // macOS\n  CMD: \"Meta\", // macOS shorthand\n  SUPER: \"Meta\", // Linux\n  WINDOWS: \"Meta\", // Windows\n  WIN: \"Meta\", // Windows shorthand\n  HOME: \"Home\",\n  END: \"End\",\n  PAGEUP: \"PageUp\",\n  PAGEDOWN: \"PageDown\",\n  PAGE_UP: \"PageUp\",\n  PAGE_DOWN: \"PageDown\",\n  PGUP: \"PageUp\",\n  PGDN: \"PageDown\",\n};\n\n/**\n * Maps a key name from various formats to Playwright-compatible format\n * @param key The key name in any supported format\n * @returns The Playwright-compatible key name\n */\nexport function mapKeyToPlaywright(key: string): string {\n  if (!key) return key;\n  const upperKey = key.toUpperCase();\n  return KEY_MAP[upperKey] || key;\n}\n"]}

package/dist/esm/lib/v3/agent/utils/googleCustomToolHandler.d.ts

@@ -0,0 +1,25 @@
+import { Part, FunctionCall, FunctionDeclaration } from "@google/genai";
+import { ToolSet } from "ai";
+import { LogLine } from "../../types/public/logs.js";
+/**
+ * Result of executing a custom tool for Google CUA
+ */
+export interface CustomToolExecutionResult {
+    functionResponse: Part;
+    success: boolean;
+}
+/**
+ * Execute a custom tool and format the response for Google's API
+ * This handles tool execution, result formatting, and error handling
+ * specific to Google's function response format
+ */
+export declare function executeGoogleCustomTool(toolName: string, toolArgs: Record<string, unknown>, tools: ToolSet, functionCall: FunctionCall, logger: (message: LogLine) => void): Promise<CustomToolExecutionResult>;
+/**
+ * Check if a function call is a custom tool
+ */
+export declare function isCustomTool(functionCall: FunctionCall, tools?: ToolSet): boolean;
+/**
+ * Convert ToolSet to Google's FunctionDeclaration array
+ * Handles the conversion of Zod schemas to Google's parameter format
+ */
+export declare function convertToolSetToFunctionDeclarations(tools: ToolSet): FunctionDeclaration[];

package/dist/esm/lib/v3/agent/utils/googleCustomToolHandler.js

@@ -0,0 +1,145 @@
+import { Type } from "@google/genai";
+import { toJsonSchema } from "../../zodCompat.js";
+/**
+ * Execute a custom tool and format the response for Google's API
+ * This handles tool execution, result formatting, and error handling
+ * specific to Google's function response format
+ */
+export async function executeGoogleCustomTool(toolName, toolArgs, tools, functionCall, logger) {
+    try {
+        logger({
+            category: "agent",
+            message: `Executing custom tool: ${toolName} with args: ${JSON.stringify(toolArgs)}`,
+            level: 1,
+        });
+        const tool = tools[toolName];
+        const toolResult = await tool.execute(toolArgs, {
+            toolCallId: `tool_${Date.now()}`,
+            messages: [],
+        });
+        logger({
+            category: "agent",
+            message: `Tool ${toolName} completed successfully. Result: ${JSON.stringify(toolResult)}`,
+            level: 1,
+        });
+        // Create function response with the result
+        const functionResponsePart = {
+            functionResponse: {
+                name: toolName,
+                response: {
+                    result: JSON.stringify(toolResult),
+                },
+            },
+        };
+        return {
+            functionResponse: functionResponsePart,
+            success: true,
+        };
+    }
+    catch (toolError) {
+        const errorMessage = toolError instanceof Error ? toolError.message : String(toolError);
+        logger({
+            category: "agent",
+            message: `Error executing custom tool ${toolName}: ${errorMessage}`,
+            level: 0,
+        });
+        // Create error function response
+        const functionResponsePart = {
+            functionResponse: {
+                name: toolName,
+                response: {
+                    error: errorMessage,
+                },
+            },
+        };
+        return {
+            functionResponse: functionResponsePart,
+            success: false,
+        };
+    }
+}
+/**
+ * Check if a function call is a custom tool
+ */
+export function isCustomTool(functionCall, tools) {
+    return !!(tools && functionCall.name && functionCall.name in tools);
+}
+/**
+ * Convert ToolSet to Google's FunctionDeclaration array
+ * Handles the conversion of Zod schemas to Google's parameter format
+ */
+export function convertToolSetToFunctionDeclarations(tools) {
+    const functionDeclarations = [];
+    for (const [name, tool] of Object.entries(tools)) {
+        const functionDeclaration = convertToolToFunctionDeclaration(name, tool);
+        if (functionDeclaration) {
+            functionDeclarations.push(functionDeclaration);
+        }
+    }
+    return functionDeclarations;
+}
+/**
+ * Convert a single ToolSet tool to Google's FunctionDeclaration format
+ */
+function convertToolToFunctionDeclaration(name, tool) {
+    try {
+        // Convert Zod schema to JSON schema
+        const schema = tool.inputSchema;
+        const jsonSchema = toJsonSchema(schema);
+        const parameters = convertJsonSchemaToGoogleParameters(jsonSchema);
+        return {
+            name,
+            description: tool.description || `Execute ${name}`,
+            parameters,
+        };
+    }
+    catch (error) {
+        console.error(`Error converting tool ${name} to function declaration:`, error);
+        return null;
+    }
+}
+/**
+ * Convert JSON schema to Google's parameter format
+ */
+function convertJsonSchemaToGoogleParameters(schema) {
+    const properties = {};
+    if (schema.properties) {
+        for (const [key, value] of Object.entries(schema.properties)) {
+            const propSchema = value;
+            properties[key] = {
+                type: mapJsonTypeToGoogleType(propSchema.type || "string"),
+                ...(propSchema.description
+                    ? { description: propSchema.description }
+                    : {}),
+            };
+        }
+    }
+    return {
+        type: Type.OBJECT,
+        properties,
+        ...(schema.required && schema.required.length > 0
+            ? { required: schema.required }
+            : {}),
+    };
+}
+/**
+ * Map JSON schema types to Google's Type enum
+ */
+function mapJsonTypeToGoogleType(jsonType) {
+    switch (jsonType.toLowerCase()) {
+        case "string":
+            return Type.STRING;
+        case "number":
+        case "integer":
+            return Type.NUMBER;
+        case "boolean":
+            return Type.BOOLEAN;
+        case "array":
+            return Type.ARRAY;
+        case "object":
+            return Type.OBJECT;
+        default:
+            return Type.STRING;
+    }
+}
+//# sourceMappingURL=googleCustomToolHandler.js.map

package/dist/esm/lib/v3/agent/utils/googleCustomToolHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"googleCustomToolHandler.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/googleCustomToolHandler.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,IAAI,EAAE,MAAM,eAAe,CAAC;AAG9E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAWlD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAgB,EAChB,QAAiC,EACjC,KAAc,EACd,YAA0B,EAC1B,MAAkC;IAElC,IAAI,CAAC;QACH,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,0BAA0B,QAAQ,eAAe,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE;YACpF,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE;YAC9C,UAAU,EAAE,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE;YAChC,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;QAEH,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,QAAQ,QAAQ,oCAAoC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE;YACzF,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,2CAA2C;QAC3C,MAAM,oBAAoB,GAAS;YACjC,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE;oBACR,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;iBACnC;aACF;SACF,CAAC;QAEF,OAAO;YACL,gBAAgB,EAAE,oBAAoB;YACtC,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,SAAS,EAAE,CAAC;QACnB,MAAM,YAAY,GAChB,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,+BAA+B,QAAQ,KAAK,YAAY,EAAE;YACnE,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,iCAAiC;QACjC,MAAM,oBAAoB,GAAS;YACjC,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE;oBACR,KAAK,EAAE,YAAY;iBACpB;aACF;SACF,CAAC;QAEF,OAAO;YACL,gBAAgB,EAAE,oBAAoB;YACtC,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,YAA0B,EAC1B,KAAe;IAEf,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC;AACtE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oCAAoC,CAClD,KAAc;IAEd,MAAM,oBAAoB,GAA0B,EAAE,CAAC;IAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,mBAAmB,GAAG,gCAAgC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACzE,IAAI,mBAAmB,EAAE,CAAC;YACxB,oBAAoB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,SAAS,gCAAgC,CACvC,IAAY,EACZ,IAAoD;IAEpD,IAAI,CAAC;QACH,oCAAoC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAiC,CAAC;QACtD,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAIrC,CAAC;QAEF,MAAM,UAAU,GAAG,mCAAmC,CAAC,UAAU,CAAC,CAAC;QAEnE,OAAO;YACL,IAAI;YACJ,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,WAAW,IAAI,EAAE;YAClD,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,yBAAyB,IAAI,2BAA2B,EACxD,KAAK,CACN,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mCAAmC,CAAC,MAI5C;IAKC,MAAM,UAAU,GAAyD,EAAE,CAAC;IAE5E,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,KAIlB,CAAC;YACF,UAAU,CAAC,GAAG,CAAC,GAAG;gBAChB,IAAI,EAAE,uBAAuB,CAAC,UAAU,CAAC,IAAI,IAAI,QAAQ,CAAC;gBAC1D,GAAG,CAAC,UAAU,CAAC,WAAW;oBACxB,CAAC,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE;oBACzC,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;YAC/C,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE;YAC/B,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,QAAgB;IAC/C,QAAQ,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QAC/B,KAAK,QAAQ;YACX,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,KAAK,SAAS;YACZ,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,KAAK,OAAO;YACV,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,KAAK,QAAQ;YACX,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB;YACE,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;AACH,CAAC","sourcesContent":["import { Part, FunctionCall, FunctionDeclaration, Type } from \"@google/genai\";\nimport { ToolSet } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport { toJsonSchema } from \"../../zodCompat.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\n/**\n * Result of executing a custom tool for Google CUA\n */\nexport interface CustomToolExecutionResult {\n  functionResponse: Part;\n  success: boolean;\n}\n\n/**\n * Execute a custom tool and format the response for Google's API\n * This handles tool execution, result formatting, and error handling\n * specific to Google's function response format\n */\nexport async function executeGoogleCustomTool(\n  toolName: string,\n  toolArgs: Record<string, unknown>,\n  tools: ToolSet,\n  functionCall: FunctionCall,\n  logger: (message: LogLine) => void,\n): Promise<CustomToolExecutionResult> {\n  try {\n    logger({\n      category: \"agent\",\n      message: `Executing custom tool: ${toolName} with args: ${JSON.stringify(toolArgs)}`,\n      level: 1,\n    });\n\n    const tool = tools[toolName];\n    const toolResult = await tool.execute(toolArgs, {\n      toolCallId: `tool_${Date.now()}`,\n      messages: [],\n    });\n\n    logger({\n      category: \"agent\",\n      message: `Tool ${toolName} completed successfully. Result: ${JSON.stringify(toolResult)}`,\n      level: 1,\n    });\n\n    // Create function response with the result\n    const functionResponsePart: Part = {\n      functionResponse: {\n        name: toolName,\n        response: {\n          result: JSON.stringify(toolResult),\n        },\n      },\n    };\n\n    return {\n      functionResponse: functionResponsePart,\n      success: true,\n    };\n  } catch (toolError) {\n    const errorMessage =\n      toolError instanceof Error ? toolError.message : String(toolError);\n\n    logger({\n      category: \"agent\",\n      message: `Error executing custom tool ${toolName}: ${errorMessage}`,\n      level: 0,\n    });\n\n    // Create error function response\n    const functionResponsePart: Part = {\n      functionResponse: {\n        name: toolName,\n        response: {\n          error: errorMessage,\n        },\n      },\n    };\n\n    return {\n      functionResponse: functionResponsePart,\n      success: false,\n    };\n  }\n}\n\n/**\n * Check if a function call is a custom tool\n */\nexport function isCustomTool(\n  functionCall: FunctionCall,\n  tools?: ToolSet,\n): boolean {\n  return !!(tools && functionCall.name && functionCall.name in tools);\n}\n\n/**\n * Convert ToolSet to Google's FunctionDeclaration array\n * Handles the conversion of Zod schemas to Google's parameter format\n */\nexport function convertToolSetToFunctionDeclarations(\n  tools: ToolSet,\n): FunctionDeclaration[] {\n  const functionDeclarations: FunctionDeclaration[] = [];\n\n  for (const [name, tool] of Object.entries(tools)) {\n    const functionDeclaration = convertToolToFunctionDeclaration(name, tool);\n    if (functionDeclaration) {\n      functionDeclarations.push(functionDeclaration);\n    }\n  }\n\n  return functionDeclarations;\n}\n\n/**\n * Convert a single ToolSet tool to Google's FunctionDeclaration format\n */\nfunction convertToolToFunctionDeclaration(\n  name: string,\n  tool: { description?: string; inputSchema: unknown },\n): FunctionDeclaration | null {\n  try {\n    // Convert Zod schema to JSON schema\n    const schema = tool.inputSchema as StagehandZodSchema;\n    const jsonSchema = toJsonSchema(schema) as {\n      properties?: Record<string, unknown>;\n      required?: string[];\n      type?: string;\n    };\n\n    const parameters = convertJsonSchemaToGoogleParameters(jsonSchema);\n\n    return {\n      name,\n      description: tool.description || `Execute ${name}`,\n      parameters,\n    };\n  } catch (error) {\n    console.error(\n      `Error converting tool ${name} to function declaration:`,\n      error,\n    );\n    return null;\n  }\n}\n\n/**\n * Convert JSON schema to Google's parameter format\n */\nfunction convertJsonSchemaToGoogleParameters(schema: {\n  properties?: Record<string, unknown>;\n  required?: string[];\n  type?: string;\n}): {\n  type: Type;\n  properties: Record<string, { type: Type; description?: string }>;\n  required?: string[];\n} {\n  const properties: Record<string, { type: Type; description?: string }> = {};\n\n  if (schema.properties) {\n    for (const [key, value] of Object.entries(schema.properties)) {\n      const propSchema = value as {\n        type?: string;\n        description?: string;\n        items?: { type?: string };\n      };\n      properties[key] = {\n        type: mapJsonTypeToGoogleType(propSchema.type || \"string\"),\n        ...(propSchema.description\n          ? { description: propSchema.description }\n          : {}),\n      };\n    }\n  }\n\n  return {\n    type: Type.OBJECT,\n    properties,\n    ...(schema.required && schema.required.length > 0\n      ? { required: schema.required }\n      : {}),\n  };\n}\n\n/**\n * Map JSON schema types to Google's Type enum\n */\nfunction mapJsonTypeToGoogleType(jsonType: string): Type {\n  switch (jsonType.toLowerCase()) {\n    case \"string\":\n      return Type.STRING;\n    case \"number\":\n    case \"integer\":\n      return Type.NUMBER;\n    case \"boolean\":\n      return Type.BOOLEAN;\n    case \"array\":\n      return Type.ARRAY;\n    case \"object\":\n      return Type.OBJECT;\n    default:\n      return Type.STRING;\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.d.ts

@@ -0,0 +1,22 @@
+import { ModelMessage, LanguageModel } from "ai";
+import { LogLine } from "../../types/public/logs.js";
+import { StagehandZodObject } from "../../zodCompat.js";
+interface DoneResult {
+    reasoning: string;
+    taskComplete: boolean;
+    messages: ModelMessage[];
+    output?: Record<string, unknown>;
+}
+/**
+ * Force a done tool call at the end of an agent run.
+ * This ensures we always get a structured final response,
+ * even if the main loop ended without calling done.
+ */
+export declare function handleDoneToolCall(options: {
+    model: LanguageModel;
+    inputMessages: ModelMessage[];
+    instruction: string;
+    outputSchema?: StagehandZodObject;
+    logger: (message: LogLine) => void;
+}): Promise<DoneResult>;
+export {};

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js

@@ -0,0 +1,105 @@
+import { generateText } from "ai";
+import { z } from "zod";
+import { tool } from "ai";
+import { getZFactory } from "../../../utils.js";
+function buildBaseDoneSchema(factory) {
+    return factory.object({
+        reasoning: factory
+            .string()
+            .describe("Brief summary of what actions were taken and the outcome"),
+        taskComplete: factory
+            .boolean()
+            .describe("true if the task was fully completed, false otherwise"),
+    });
+}
+/**
+ * Force a done tool call at the end of an agent run.
+ * This ensures we always get a structured final response,
+ * even if the main loop ended without calling done.
+ */
+export async function handleDoneToolCall(options) {
+    const { model, inputMessages, instruction, outputSchema, logger } = options;
+    logger({
+        category: "agent",
+        message: "Agent calling tool: done",
+        level: 1,
+    });
+    // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing
+    const factory = outputSchema
+        ? getZFactory(outputSchema)
+        : z;
+    const baseDoneSchema = buildBaseDoneSchema(factory);
+    // Merge base done schema with user-provided output schema if present
+    const doneToolSchema = outputSchema
+        ? baseDoneSchema.extend({
+            output: outputSchema.describe("The specific data the user requested from this task"),
+        })
+        : baseDoneSchema;
+    const outputInstructions = outputSchema
+        ? `\n\nThe user also requested the following information from this task. Provide it in the "output" field:\n${JSON.stringify(Object.fromEntries(Object.entries(outputSchema.shape).map(([key, value]) => [
+            key,
+            value.description || "no description",
+        ])), null, 2)}`
+        : "";
+    const systemPrompt = `You are a web automation assistant that was tasked with completing a task.
+
+The task was:
+"${instruction}"
+
+Review what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}
+
+Call the "done" tool with:
+1. A brief summary of what was done
+2. Whether the task was completed successfully${outputSchema ? "\n3. The requested output data based on what you found" : ""}`;
+    const doneTool = tool({
+        description: outputSchema
+            ? "Complete the task with your assessment and the requested output data."
+            : "Complete the task with your final assessment.",
+        inputSchema: doneToolSchema,
+        execute: async (params) => {
+            return { success: true, ...params };
+        },
+    });
+    const userPrompt = {
+        role: "user",
+        content: outputSchema
+            ? "Provide your final assessment and the requested output data."
+            : "Provide your final assessment.",
+    };
+    const result = await generateText({
+        model,
+        system: systemPrompt,
+        messages: [...inputMessages, userPrompt],
+        tools: { done: doneTool },
+        toolChoice: { type: "tool", toolName: "done" },
+        providerOptions: {
+            google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
+            openai: { store: false },
+        },
+    });
+    const doneToolCall = result.toolCalls.find((tc) => tc.toolName === "done");
+    const outputMessages = [
+        userPrompt,
+        ...(result.response?.messages || []),
+    ];
+    if (!doneToolCall) {
+        return {
+            reasoning: result.text || "Task execution completed",
+            taskComplete: false,
+            messages: outputMessages,
+        };
+    }
+    const input = doneToolCall.input;
+    logger({
+        category: "agent",
+        message: `Task completed`,
+        level: 1,
+    });
+    return {
+        reasoning: input.reasoning,
+        taskComplete: input.taskComplete,
+        messages: outputMessages,
+        output: input.output,
+    };
+}
+//# sourceMappingURL=handleDoneToolCall.js.map

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAwC,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAG1B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAUhD,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,WAAW,CAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAI,CAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,aAAa,EAAE,UAAU,CAAC;QACxC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QAC9C,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;SACzB;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC3E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\ninterface DoneResult {\n  reasoning: string;\n  taskComplete: boolean;\n  messages: ModelMessage[];\n  output?: Record<string, unknown>;\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n  return factory.object({\n    reasoning: factory\n      .string()\n      .describe(\"Brief summary of what actions were taken and the outcome\"),\n    taskComplete: factory\n      .boolean()\n      .describe(\"true if the task was fully completed, false otherwise\"),\n  });\n}\n\n/**\n * Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n  model: LanguageModel;\n  inputMessages: ModelMessage[];\n  instruction: string;\n  outputSchema?: StagehandZodObject;\n  logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n  const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n  logger({\n    category: \"agent\",\n    message: \"Agent calling tool: done\",\n    level: 1,\n  });\n  // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n  const factory = outputSchema\n    ? getZFactory(outputSchema as StagehandZodSchema)\n    : z;\n  const baseDoneSchema = buildBaseDoneSchema(factory);\n\n  // Merge base done schema with user-provided output schema if present\n  const doneToolSchema = outputSchema\n    ? baseDoneSchema.extend({\n        output: outputSchema.describe(\n          \"The specific data the user requested from this task\",\n        ),\n      })\n    : baseDoneSchema;\n\n  const outputInstructions = outputSchema\n    ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n        Object.fromEntries(\n          Object.entries(outputSchema.shape).map(\n            ([key, value]: [string, StagehandZodSchema]) => [\n              key,\n              value.description || \"no description\",\n            ],\n          ),\n        ),\n        null,\n        2,\n      )}`\n    : \"\";\n\n  const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n  const doneTool = tool({\n    description: outputSchema\n      ? \"Complete the task with your assessment and the requested output data.\"\n      : \"Complete the task with your final assessment.\",\n    inputSchema: doneToolSchema,\n    execute: async (params) => {\n      return { success: true, ...params };\n    },\n  });\n\n  const userPrompt: ModelMessage = {\n    role: \"user\",\n    content: outputSchema\n      ? \"Provide your final assessment and the requested output data.\"\n      : \"Provide your final assessment.\",\n  };\n\n  const result = await generateText({\n    model,\n    system: systemPrompt,\n    messages: [...inputMessages, userPrompt],\n    tools: { done: doneTool } as ToolSet,\n    toolChoice: { type: \"tool\", toolName: \"done\" },\n    providerOptions: {\n      google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n      openai: { store: false },\n    },\n  });\n\n  const doneToolCall = result.toolCalls.find((tc) => tc.toolName === \"done\");\n  const outputMessages: ModelMessage[] = [\n    userPrompt,\n    ...(result.response?.messages || []),\n  ];\n\n  if (!doneToolCall) {\n    return {\n      reasoning: result.text || \"Task execution completed\",\n      taskComplete: false,\n      messages: outputMessages,\n    };\n  }\n\n  const input = doneToolCall.input as {\n    reasoning: string;\n    taskComplete: boolean;\n    output?: Record<string, unknown>;\n  };\n  logger({\n    category: \"agent\",\n    message: `Task completed`,\n    level: 1,\n  });\n\n  return {\n    reasoning: input.reasoning,\n    taskComplete: input.taskComplete,\n    messages: outputMessages,\n    output: input.output,\n  };\n}\n"]}

package/dist/esm/lib/v3/agent/utils/imageCompression.d.ts

@@ -0,0 +1,53 @@
+import { AnthropicMessage, AnthropicToolResult, ResponseInputItem as OpenAIResponseInputItem } from "../../types/public/agent.js";
+import type { Content as GoogleContent } from "@google/genai";
+export type ResponseInputItem = AnthropicMessage | AnthropicToolResult;
+export type AnthropicResponseInputItem = AnthropicMessage | AnthropicToolResult;
+export type SupportedInputItem = AnthropicResponseInputItem | OpenAIResponseInputItem | GoogleContent;
+/**
+ * Finds all items in the conversation history that contain images
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findItemsWithImages(items: ResponseInputItem[]): number[];
+/**
+ * Compresses conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressConversationImages(items: ResponseInputItem[], keepMostRecentCount?: number): {
+    items: ResponseInputItem[];
+};
+/**
+ * Finds all items in the conversation history that contain images (Google format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findGoogleItemsWithImages(items: GoogleContent[]): number[];
+/**
+ * Finds all items in the conversation history that contain images (OpenAI format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findOpenAIItemsWithImages(items: OpenAIResponseInputItem[]): number[];
+/**
+ * Compresses OpenAI conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressOpenAIConversationImages(items: OpenAIResponseInputItem[], keepMostRecentCount?: number): {
+    items: OpenAIResponseInputItem[];
+};
+/**
+ * Compresses Google conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressGoogleConversationImages(items: GoogleContent[], keepMostRecentCount?: number): {
+    items: GoogleContent[];
+};