@browserbasehq/orca 3.0.0-preview.4 → 3.0.0-preview.6

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Browserbase Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.js

@@ -1642,13 +1642,10 @@ function decorateRoles(nodes, opts) {
 }
 function buildHierarchicalTree(nodes, opts) {
   return __async(this, null, function* () {
-    var _a2, _b, _c;
+    var _a2;
     const nodeMap = /* @__PURE__ */ new Map();
     for (const n of nodes) {
-      const hasName = Boolean((_a2 = n.name) == null ? void 0 : _a2.trim());
-      const hasDescription = Boolean((_b = n.description) == null ? void 0 : _b.trim());
-      const hasChildren = !!(n.childIds && n.childIds.length);
-      const keep = hasName || hasDescription || hasChildren || !isStructural(n.role);
+      const keep = !!(n.name && n.name.trim()) || !!(n.childIds && n.childIds.length) || !isStructural(n.role);
       if (!keep) continue;
       nodeMap.set(n.nodeId, __spreadValues({}, n));
     }
@@ -1656,7 +1653,7 @@ function buildHierarchicalTree(nodes, opts) {
       if (!n.parentId) continue;
       const parent = nodeMap.get(n.parentId);
       const cur = nodeMap.get(n.nodeId);
-      if (parent && cur) ((_c = parent.children) != null ? _c : parent.children = []).push(cur);
+      if (parent && cur) ((_a2 = parent.children) != null ? _a2 : parent.children = []).push(cur);
     }
     const roots = nodes.filter((n) => !n.parentId && nodeMap.has(n.nodeId)).map((n) => nodeMap.get(n.nodeId));
     const cleaned = (yield Promise.all(roots.map(pruneStructuralSafe))).filter(
@@ -1665,17 +1662,15 @@ function buildHierarchicalTree(nodes, opts) {
     return { tree: cleaned };
     function pruneStructuralSafe(node) {
       return __async(this, null, function* () {
-        var _a3, _b2;
+        var _a3;
         if (+node.nodeId < 0) return null;
-        const hasDescription = Boolean((_a3 = node.description) == null ? void 0 : _a3.trim());
-        const structuralRole = isStructural(node.role) && !hasDescription;
-        const children = (_b2 = node.children) != null ? _b2 : [];
+        const children = (_a3 = node.children) != null ? _a3 : [];
         if (!children.length) {
-          return structuralRole ? null : node;
+          return isStructural(node.role) ? null : node;
         }
         const cleanedKids = (yield Promise.all(children.map(pruneStructuralSafe))).filter(Boolean);
         const prunedStatic = removeRedundantStaticTextChildren(node, cleanedKids);
-        if (structuralRole) {
+        if (isStructural(node.role)) {
           if (prunedStatic.length === 1) return prunedStatic[0];
           if (prunedStatic.length === 0) return null;
         }
@@ -6501,7 +6496,7 @@ var import_path5 = __toESM(require("path"));
 var import_process2 = __toESM(require("process"));
 
 // lib/version.ts
-var STAGEHAND_VERSION = "3.0.0-preview.4";
+var STAGEHAND_VERSION = "3.0.0-preview.6";
 
 // lib/v3/types/public/sdkErrors.ts
 var StagehandError = class extends Error {
@@ -7896,7 +7891,7 @@ Return the element that matches the instruction if it exists. Otherwise, return
 }
 function buildActPrompt(action, supportedActions, variables) {
   let instruction = `Find the most relevant element to perform an action on given the following action: ${action}.  
-  If the action implies choosing/selecting/clicking an option from a dropdown, ignore the 'General Instructions' section, and follow the 'Dropdown Specific Instructions' section carefully.
+  IF AND ONLY IF the action EXPLICITLY includes the word 'dropdown' and implies choosing/selecting an option from a dropdown, ignore the 'General Instructions' section, and follow the 'Dropdown Specific Instructions' section carefully.
   
   General Instructions: 
     Provide an action for this element such as ${supportedActions.join(", ")}. Remember that to users, buttons and links look the same in most cases.
@@ -8305,9 +8300,7 @@ function act(_0) {
           "the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in."
         )
       ),
-      twoStep: import_v32.z.boolean().describe(
-        "true if we will need to take another action after this. false otherwise"
-      )
+      twoStep: import_v32.z.boolean()
     });
     const messages = [
       buildActSystemPrompt(userProvidedInstructions),
@@ -8437,7 +8430,6 @@ function normalizeRootXPath(input) {
 function performUnderstudyMethod(page, frame, method, rawXPath, args, domSettleTimeoutMs) {
   return __async(this, null, function* () {
     var _a2;
-    yield waitForDomNetworkQuiet(frame, domSettleTimeoutMs);
     const selectorRaw = normalizeRootXPath(rawXPath);
     const locator = yield resolveLocatorWithHops(
       page,
@@ -9056,6 +9048,7 @@ var ActHandler = class {
       const llmClient = this.resolveLlmClient(model);
       const doObserveAndAct = () => __async(this, null, function* () {
         var _a2, _b, _c, _d, _e, _f, _g, _h, _i, _j;
+        yield waitForDomNetworkQuiet(page.mainFrame(), this.defaultDomSettleTimeoutMs);
         const snapshot = yield captureHybridSnapshot(page, {
           experimental: true
         });
@@ -9689,7 +9682,8 @@ var createActTool = (v3, executionModel) => (0, import_ai2.tool)({
       });
       return {
         success: (_b = result.success) != null ? _b : true,
-        action: (_c = result == null ? void 0 : result.actionDescription) != null ? _c : action
+        action: (_c = result == null ? void 0 : result.actionDescription) != null ? _c : action,
+        playwrightArguments: actions.length > 0 ? actions[0] : void 0
       };
     } catch (error) {
       return { success: false, error: (_d = error == null ? void 0 : error.message) != null ? _d : String(error) };
@@ -9867,7 +9861,11 @@ For any form with 2+ inputs/textareas. Faster than individual typing.`,
       observeResults,
       actions: replayableActions
     });
-    return { success: true, actions: completed };
+    return {
+      success: true,
+      actions: completed,
+      playwrightArguments: replayableActions
+    };
   })
 });
 
@@ -10088,6 +10086,69 @@ function calculateCompressionStats(originalSize, compressedSize, screenshotCount
   };
 }
 
+// lib/v3/agent/utils/actionMapping.ts
+function mapToolResultToActions({
+  toolCallName,
+  toolResult,
+  args,
+  reasoning
+}) {
+  switch (toolCallName) {
+    case "act":
+      return mapActToolResult(toolResult, args, reasoning);
+    case "fillForm":
+      return mapFillFormToolResult(toolResult, args, reasoning);
+    default:
+      return [createStandardAction(toolCallName, args, reasoning)];
+  }
+}
+function mapActToolResult(toolResult, args, reasoning) {
+  if (!toolResult || typeof toolResult !== "object") {
+    return [createStandardAction("act", args, reasoning)];
+  }
+  const result = toolResult;
+  const output = result.output || result;
+  const action = __spreadValues({
+    type: "act",
+    reasoning,
+    taskCompleted: false
+  }, args);
+  if (output.playwrightArguments) {
+    action.playwrightArguments = output.playwrightArguments;
+  }
+  return [action];
+}
+function mapFillFormToolResult(toolResult, args, reasoning) {
+  if (!toolResult || typeof toolResult !== "object") {
+    return [createStandardAction("fillForm", args, reasoning)];
+  }
+  const result = toolResult;
+  const output = result.output || result;
+  const observeResults = Array.isArray(output == null ? void 0 : output.playwrightArguments) ? output.playwrightArguments : [];
+  const actions = [];
+  actions.push(__spreadValues({
+    type: "fillForm",
+    reasoning,
+    taskCompleted: false
+  }, args));
+  for (const observeResult of observeResults) {
+    actions.push({
+      type: "act",
+      reasoning: "acting from fillform tool",
+      taskCompleted: false,
+      playwrightArguments: observeResult
+    });
+  }
+  return actions;
+}
+function createStandardAction(toolCallName, args, reasoning) {
+  return __spreadValues({
+    type: toolCallName,
+    reasoning,
+    taskCompleted: toolCallName === "close" ? args == null ? void 0 : args.taskComplete : false
+  }, args);
+}
+
 // lib/v3/handlers/v3AgentHandler.ts
 var V3AgentHandler = class {
   constructor(v3, logger, llmClient, executionModel, systemInstructions, mcpTools) {
@@ -10108,6 +10169,7 @@ var V3AgentHandler = class {
       let finalMessage = "";
       let completed = false;
       const collectedReasoning = [];
+      let currentPageUrl = (yield this.v3.context.awaitActivePage()).url();
       try {
         const systemPrompt = this.buildSystemPrompt(
           options.instruction,
@@ -10142,14 +10204,17 @@ var V3AgentHandler = class {
           temperature: 1,
           toolChoice: "auto",
           onStepFinish: (event) => __async(this, null, function* () {
+            var _a3;
             this.logger({
               category: "agent",
               message: `Step finished: ${event.finishReason}`,
               level: 2
             });
             if (event.toolCalls && event.toolCalls.length > 0) {
-              for (const toolCall of event.toolCalls) {
+              for (let i = 0; i < event.toolCalls.length; i++) {
+                const toolCall = event.toolCalls[i];
                 const args = toolCall.input;
+                const toolResult = (_a3 = event.toolResults) == null ? void 0 : _a3[i];
                 if (event.text.length > 0) {
                   collectedReasoning.push(event.text);
                   this.logger({
@@ -10166,13 +10231,19 @@ var V3AgentHandler = class {
                     finalMessage = closeReasoning ? `${allReasoning} ${closeReasoning}`.trim() : allReasoning || "Task completed successfully";
                   }
                 }
-                const action = __spreadValues({
-                  type: toolCall.toolName,
-                  reasoning: event.text || void 0,
-                  taskCompleted: toolCall.toolName === "close" ? args == null ? void 0 : args.taskComplete : false
-                }, args);
-                actions.push(action);
+                const mappedActions = mapToolResultToActions({
+                  toolCallName: toolCall.toolName,
+                  toolResult,
+                  args,
+                  reasoning: event.text || void 0
+                });
+                for (const action of mappedActions) {
+                  action.pageUrl = currentPageUrl;
+                  action.timestamp = Date.now();
+                  actions.push(action);
+                }
               }
+              currentPageUrl = (yield this.v3.context.awaitActivePage()).url();
             }
           })
         });
@@ -11856,6 +11927,7 @@ var GoogleCUAClient = class extends AgentClient {
               level: 2
             });
             if (action.type === "function" && action.name === "open_web_browser") {
+              action.pageUrl = this.currentUrl;
               logger({
                 category: "agent",
                 message: "Skipping open_web_browser action",
@@ -12063,7 +12135,8 @@ var GoogleCUAClient = class extends AgentClient {
         return {
           type: "function",
           name: "open_web_browser",
-          arguments: null
+          arguments: null,
+          timestamp: Date.now()
         };
       case "click_at": {
         const { x, y } = this.normalizeCoordinates(
@@ -12211,6 +12284,9 @@ var GoogleCUAClient = class extends AgentClient {
   }
   captureScreenshot(options) {
     return __async(this, null, function* () {
+      if (options == null ? void 0 : options.currentUrl) {
+        this.currentUrl = options.currentUrl;
+      }
       if (options == null ? void 0 : options.base64Image) {
         return `data:image/png;base64,${options.base64Image}`;
       }
@@ -12332,6 +12408,7 @@ var V3CuaAgentHandler = class {
     }));
     this.agentClient.setActionHandler((action) => __async(this, null, function* () {
       var _a2, _b, _c;
+      action.pageUrl = (yield this.v3.context.awaitActivePage()).url();
       const defaultDelay = 1e3;
       const waitBetween = ((_a2 = this.options.clientOptions) == null ? void 0 : _a2.waitBetweenActions) || defaultDelay;
       try {
@@ -12343,6 +12420,7 @@ var V3CuaAgentHandler = class {
         }
         yield new Promise((r) => setTimeout(r, 300));
         yield this.executeAction(action);
+        action.timestamp = Date.now();
         yield new Promise((r) => setTimeout(r, waitBetween));
         try {
           yield this.captureAndSendScreenshot();
@@ -38677,13 +38755,13 @@ var _V3 = class _V3 {
       level: 1
     });
     if (options == null ? void 0 : options.cua) {
-      if (!(options == null ? void 0 : options.model)) {
-        throw new Error("A CUA agent requires a model to be specified.");
-      }
-      const { modelName, isCua, clientOptions } = resolveModel(options.model);
+      const modelToUse = (options == null ? void 0 : options.model) || __spreadValues({
+        modelName: this.modelName
+      }, this.modelClientOptions);
+      const { modelName, isCua, clientOptions } = resolveModel(modelToUse);
       if (!isCua) {
         throw new Error(
-          "Model is not a CUA model. Try one of the following: " + AVAILABLE_CUA_MODELS.join(", ")
+          "To use the computer use agent, please provide a CUA model in the agent constructor or stagehand config. Try one of our supported CUA models: " + AVAILABLE_CUA_MODELS.join(", ")
         );
       }
       const agentConfigSignature2 = this.agentCache.buildConfigSignature(options);
@@ -38770,10 +38848,11 @@ Do not ask follow up questions, the user will trust your judgement.`
             );
           }
           const tools = (options == null ? void 0 : options.integrations) ? yield resolveTools(options.integrations, options.tools) : (_a2 = options == null ? void 0 : options.tools) != null ? _a2 : {};
+          const agentLlmClient = (options == null ? void 0 : options.model) ? this.resolveLlmClient(options.model) : this.llmClient;
           const handler = new V3AgentHandler(
             this,
             this.logger,
-            this.llmClient,
+            agentLlmClient,
             typeof (options == null ? void 0 : options.executionModel) === "string" ? options.executionModel : (_b = options == null ? void 0 : options.executionModel) == null ? void 0 : _b.modelName,
             options == null ? void 0 : options.systemPrompt,
             tools

package/package.json

@@ -1,21 +1,10 @@
 {
   "name": "@browserbasehq/orca",
-  "version": "3.0.0-preview.4",
+  "version": "3.0.0-preview.6",
   "description": "An AI web browsing framework focused on simplicity and extensibility.",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
-  "scripts": {
-    "gen-version": "tsx scripts/gen-version.ts",
-    "build-dom-scripts": "tsx lib/v3/dom/genDomScripts.ts && tsx lib/v3/dom/genLocatorScripts.ts",
-    "build-js": "tsup --entry.index lib/v3/index.ts --dts",
-    "typecheck": "tsc --noEmit",
-    "prepare": "pnpm run build",
-    "build": "pnpm run gen-version && pnpm run build-dom-scripts && pnpm run build-js && pnpm run typecheck",
-    "example": "node --import tsx -e \"const args=process.argv.slice(1).filter(a=>a!=='--'); const [p]=args; const n=(p||'example').replace(/^\\.\\//,'').replace(/\\.ts$/i,''); import(new URL(require('node:path').resolve('examples', n + '.ts'), 'file:'));\" --",
-    "lint": "cd ../.. && prettier --check packages/core && cd packages/core && eslint .",
-    "format": "prettier --write ."
-  },
   "files": [
     "dist/index.js",
     "dist/index.d.ts",
@@ -89,5 +78,15 @@
   "bugs": {
     "url": "https://github.com/browserbase/stagehand/issues"
   },
-  "homepage": "https://stagehand.dev"
-}
+  "homepage": "https://stagehand.dev",
+  "scripts": {
+    "gen-version": "tsx scripts/gen-version.ts",
+    "build-dom-scripts": "tsx lib/v3/dom/genDomScripts.ts && tsx lib/v3/dom/genLocatorScripts.ts",
+    "build-js": "tsup --entry.index lib/v3/index.ts --dts",
+    "typecheck": "tsc --noEmit",
+    "build": "pnpm run gen-version && pnpm run build-dom-scripts && pnpm run build-js && pnpm run typecheck",
+    "example": "node --import tsx -e \"const args=process.argv.slice(1).filter(a=>a!=='--'); const [p]=args; const n=(p||'example').replace(/^\\.\\//,'').replace(/\\.ts$/i,''); import(new URL(require('node:path').resolve('examples', n + '.ts'), 'file:'));\" --",
+    "lint": "cd ../.. && prettier --check packages/core && cd packages/core && eslint .",
+    "format": "prettier --write ."
+  }
+}