@browserbasehq/stagehand 1.0.3 → 1.1.0-alpha.1

package/dist/lib/prompt.js

@@ -0,0 +1,288 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.actTools = void 0;
+exports.buildVerifyActCompletionSystemPrompt = buildVerifyActCompletionSystemPrompt;
+exports.buildVerifyActCompletionUserPrompt = buildVerifyActCompletionUserPrompt;
+exports.buildActSystemPrompt = buildActSystemPrompt;
+exports.buildActUserPrompt = buildActUserPrompt;
+exports.buildExtractSystemPrompt = buildExtractSystemPrompt;
+exports.buildExtractUserPrompt = buildExtractUserPrompt;
+exports.buildRefineSystemPrompt = buildRefineSystemPrompt;
+exports.buildRefineUserPrompt = buildRefineUserPrompt;
+exports.buildMetadataSystemPrompt = buildMetadataSystemPrompt;
+exports.buildMetadataPrompt = buildMetadataPrompt;
+exports.buildObserveSystemPrompt = buildObserveSystemPrompt;
+exports.buildObserveUserMessage = buildObserveUserMessage;
+exports.buildAskSystemPrompt = buildAskSystemPrompt;
+exports.buildAskUserPrompt = buildAskUserPrompt;
+// act
+const actSystemPrompt = `
+# Instructions
+You are a browser automation assistant. Your job is to accomplish the user's goal across multiple model calls.
+
+You are given:
+1. the user's overall goal
+2. the steps that you've taken so far
+3. a list of active DOM elements in this chunk to consider to get closer to the goal. 
+
+You have 2 tools that you can call: doAction, and skipSection. Do action only performs Playwright actions. Do not perform any other actions.
+
+Note: If there is a popup on the page for cookies or advertising that has nothing to do with the goal, try to close it first before proceeding. As this can block the goal from being completed.
+
+Also, verify if the goal has been accomplished already. Do this by checking if the goal has been accomplished based on the previous steps completed, the current page DOM elements and the current page URL / starting page URL. If it has, set completed to true and finish the task.
+
+Do exactly what the user's goal is. Do not exceed the scope of the goal.
+`;
+const verifyActCompletionSystemPrompt = `
+You are a browser automation assistant. The job has given you a goal and a list of steps that have been taken so far. Your job is to determine if the user's goal has been completed based on the provided information.
+
+# Input
+You will receive:
+1. The user's goal: A clear description of what the user wants to achieve.
+2. Steps taken so far: A list of actions that have been performed up to this point.
+3. An image of the current page
+
+# Your Task
+Analyze the provided information to determine if the user's goal has been fully completed.
+
+# Output
+Return a boolean value:
+- true: If the goal has been definitively completed based on the steps taken and the current page.
+- false: If the goal has not been completed or if there's any uncertainty about its completion.
+
+# Important Considerations
+- False positives are okay. False negatives are not okay.
+- Look for evidence of errors on the page or something having gone wrong in completing the goal. If one does not exist, return true.
+`;
+// ## Examples for completion check
+// ### Example 1
+// 1. User's goal: "input data scientist into role"
+// 2. Steps you've taken so far: "The role input field was filled with 'data scientist'."
+// 3. Active DOM elements: ["<input id="c9" class="VfPpkd-fmcmS-wGMbrd " aria-expanded="false" data-axe="mdc-autocomplete">data scientist</input>", "<button class="VfPpkd-LgbsSe VfPpkd-LgbsSe-OWXEXe-INsAgc lJ9FBc nDgy9d" type="submit">Search</button>"]
+// Output: Will need to have completed set to true. Nothing else matters.
+// Reasoning: The goal the user set has already been accomplished. We should not take any extra actions outside of the scope of the goal (for example, clicking on the search button is an invalid action - ie: not acceptable).
+// ### Example 2
+// 1. User's goal: "Sign up for the newsletter"
+// 2. Steps you've taken so far: ["The email input field was filled with 'test@test.com'."]
+// 3. Active DOM elements: ["<input type='email' id='newsletter-email' placeholder='Enter your email'></input>", "<button id='subscribe-button'>Subscribe</button>"]
+// Output: Will need to have click on the subscribe button as action. And completed set to false.
+// Reasoning: There might be an error when trying to submit the form and you need to make sure the goal is accomplished properly. So you set completed to false.
+function buildVerifyActCompletionSystemPrompt() {
+    return {
+        role: "system",
+        content: verifyActCompletionSystemPrompt,
+    };
+}
+function buildVerifyActCompletionUserPrompt(goal, steps = "None", domElements) {
+    let actUserPrompt = `
+# My Goal
+${goal}
+
+# Steps You've Taken So Far
+${steps}
+`;
+    if (domElements) {
+        actUserPrompt += `
+# Active DOM Elements on the current page
+${domElements}
+`;
+    }
+    return {
+        role: "user",
+        content: actUserPrompt,
+    };
+}
+function buildActSystemPrompt() {
+    return {
+        role: "system",
+        content: actSystemPrompt,
+    };
+}
+function buildActUserPrompt(action, steps = "None", domElements) {
+    const actUserPrompt = `
+# My Goal
+${action}
+
+# Steps You've Taken So Far
+${steps}
+
+# Current Active Dom Elements
+${domElements}
+`;
+    return {
+        role: "user",
+        content: actUserPrompt,
+    };
+}
+exports.actTools = [
+    {
+        type: "function",
+        function: {
+            name: "doAction",
+            description: "execute the next playwright step that directly accomplishes the goal",
+            parameters: {
+                type: "object",
+                required: ["method", "element", "args", "step", "completed"],
+                properties: {
+                    method: {
+                        type: "string",
+                        description: "The playwright function to call.",
+                    },
+                    element: {
+                        type: "number",
+                        description: "The element number to act on",
+                    },
+                    args: {
+                        type: "array",
+                        description: "The required arguments",
+                        items: {
+                            type: "string",
+                            description: "The argument to pass to the function",
+                        },
+                    },
+                    step: {
+                        type: "string",
+                        description: "human readable description of the step that is taken in the past tense. Please be very detailed.",
+                    },
+                    why: {
+                        type: "string",
+                        description: "why is this step taken? how does it advance the goal?",
+                    },
+                    completed: {
+                        type: "boolean",
+                        description: "true if the goal should be accomplished after this step",
+                    },
+                },
+            },
+        },
+    },
+    {
+        type: "function",
+        function: {
+            name: "skipSection",
+            description: "skips this area of the webpage because the current goal cannot be accomplished here",
+            parameters: {
+                type: "object",
+                properties: {
+                    reason: {
+                        type: "string",
+                        description: "reason that no action is taken",
+                    },
+                },
+            },
+        },
+    },
+];
+// extract
+const extractSystemPrompt = `You are extracting content on behalf of a user. You will be given:
+1. An instruction
+2. A list of DOM elements to extract from
+
+Print the exact text from the DOM elements with all symbols, characters, and endlines as is.
+Print null or an empty string if no new information is found.
+
+ONLY print the content using the print_extracted_data tool provided.
+ONLY print the content using the print_extracted_data tool provided.
+`;
+function buildExtractSystemPrompt() {
+    const content = extractSystemPrompt.replace(/\s+/g, " ");
+    return {
+        role: "system",
+        content,
+    };
+}
+function buildExtractUserPrompt(instruction, domElements) {
+    return {
+        role: "user",
+        content: `Instruction: ${instruction}
+DOM: ${domElements}
+
+ONLY print the content using the print_extracted_data tool provided.
+ONLY print the content using the print_extracted_data tool provided.`,
+    };
+}
+const refineSystemPrompt = `You are tasked with refining and filtering information for the final output based on newly extracted and previously extracted content. Your responsibilities are:
+1. Remove exact duplicates for elements in arrays and objects.
+2. For text fields, append or update relevant text if the new content is an extension, replacement, or continuation.
+3. For non-text fields (e.g., numbers, booleans), update with new values if they differ.
+4. Add any completely new fields or objects.
+
+Return the updated content that includes both the previous content and the new, non-duplicate, or extended information.`;
+function buildRefineSystemPrompt() {
+    return {
+        role: "system",
+        content: refineSystemPrompt,
+    };
+}
+function buildRefineUserPrompt(instruction, previouslyExtractedContent, newlyExtractedContent) {
+    return {
+        role: "user",
+        content: `Instruction: ${instruction}
+Previously extracted content: ${JSON.stringify(previouslyExtractedContent, null, 2)}
+Newly extracted content: ${JSON.stringify(newlyExtractedContent, null, 2)}
+Refined content:`,
+    };
+}
+const metadataSystemPrompt = `You are an AI assistant tasked with evaluating the progress and completion status of an extraction task.
+Analyze the extraction response and determine if the task is completed or if more information is needed.
+
+Strictly abide by the following criteria:
+1. Once the instruction has been satisfied by the current extraction response, ALWAYS set completion status to true and stop processing, regardless of remaining chunks.
+2. Only set completion status to false if BOTH of these conditions are true:
+   - The instruction has not been satisfied yet
+   - There are still chunks left to process (chunksTotal > chunksSeen)`;
+function buildMetadataSystemPrompt() {
+    return {
+        role: "system",
+        content: metadataSystemPrompt,
+    };
+}
+function buildMetadataPrompt(instruction, extractionResponse, chunksSeen, chunksTotal) {
+    return {
+        role: "user",
+        content: `Instruction: ${instruction}
+Extracted content: ${JSON.stringify(extractionResponse, null, 2)}
+chunksSeen: ${chunksSeen}
+chunksTotal: ${chunksTotal}`,
+    };
+}
+// observe
+const observeSystemPrompt = `
+You are helping the user automate the browser by finding elements based on what the user wants to observe in the page.
+You will be given:
+1. a instruction of elements to observe
+2. a numbered list of possible elements or an annotated image of the page
+
+Return an array of elements that match the instruction.
+`;
+function buildObserveSystemPrompt() {
+    const content = observeSystemPrompt.replace(/\s+/g, " ");
+    return {
+        role: "system",
+        content,
+    };
+}
+function buildObserveUserMessage(instruction, domElements) {
+    return {
+        role: "user",
+        content: `instruction: ${instruction}
+DOM: ${domElements}`,
+    };
+}
+// ask
+const askSystemPrompt = `
+you are a simple question answering assistent given the user's question. respond with only the answer.
+`;
+function buildAskSystemPrompt() {
+    return {
+        role: "system",
+        content: askSystemPrompt,
+    };
+}
+function buildAskUserPrompt(question) {
+    return {
+        role: "user",
+        content: `question: ${question}`,
+    };
+}
+//# sourceMappingURL=prompt.js.map

package/dist/lib/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../../lib/prompt.ts"],"names":[],"mappings":";;;AA4DA,oFAKC;AAED,gFAwBC;AAED,oDAKC;AAED,gDAoBC;AA+ED,4DAMC;AAED,wDAYC;AAUD,0DAKC;AAED,sDAYC;AAWD,8DAKC;AAED,kDAaC;AAWD,4DAOC;AAED,0DASC;AAMD,oDAKC;AAED,gDAOC;AAtUD,MAAM;AACN,MAAM,eAAe,GAAG;;;;;;;;;;;;;;;;CAgBvB,CAAC;AAEF,MAAM,+BAA+B,GAAG;;;;;;;;;;;;;;;;;;;;CAoBvC,CAAC;AAEF,mCAAmC;AACnC,gBAAgB;AAChB,mDAAmD;AACnD,yFAAyF;AACzF,4PAA4P;AAE5P,yEAAyE;AACzE,gOAAgO;AAEhO,gBAAgB;AAChB,+CAA+C;AAC/C,2FAA2F;AAC3F,oKAAoK;AAEpK,iGAAiG;AACjG,gKAAgK;AAEhK,SAAgB,oCAAoC;IAClD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,+BAA+B;KACzC,CAAC;AACJ,CAAC;AAED,SAAgB,kCAAkC,CAChD,IAAY,EACZ,KAAK,GAAG,MAAM,EACd,WAA+B;IAE/B,IAAI,aAAa,GAAG;;EAEpB,IAAI;;;EAGJ,KAAK;CACN,CAAC;IAEA,IAAI,WAAW,EAAE,CAAC;QAChB,aAAa,IAAI;;EAEnB,WAAW;CACZ,CAAC;IACA,CAAC;IAED,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,aAAa;KACvB,CAAC;AACJ,CAAC;AAED,SAAgB,oBAAoB;IAClC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,eAAe;KACzB,CAAC;AACJ,CAAC;AAED,SAAgB,kBAAkB,CAChC,MAAc,EACd,KAAK,GAAG,MAAM,EACd,WAAmB;IAEnB,MAAM,aAAa,GAAG;;EAEtB,MAAM;;;EAGN,KAAK;;;EAGL,WAAW;CACZ,CAAC;IAEA,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,aAAa;KACvB,CAAC;AACJ,CAAC;AAEY,QAAA,QAAQ,GAAqC;IACxD;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,sEAAsE;YACxE,UAAU,EAAE;gBACV,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC;gBAC5D,UAAU,EAAE;oBACV,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kCAAkC;qBAChD;oBACD,OAAO,EAAE;wBACP,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,8BAA8B;qBAC5C;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,OAAO;wBACb,WAAW,EAAE,wBAAwB;wBACrC,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,sCAAsC;yBACpD;qBACF;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,kGAAkG;qBACrG;oBACD,GAAG,EAAE;wBACH,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,uDAAuD;qBAC1D;oBACD,SAAS,EAAE;wBACT,IAAI,EAAE,SAAS;wBACf,WAAW,EACT,yDAAyD;qBAC5D;iBACF;aACF;SACF;KACF;IACD;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,IAAI,EAAE,aAAa;YACnB,WAAW,EACT,qFAAqF;YACvF,UAAU,EAAE;gBACV,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gCAAgC;qBAC9C;iBACF;aACF;SACF;KACF;CACF,CAAC;AAEF,UAAU;AACV,MAAM,mBAAmB,GAAG;;;;;;;;;CAS3B,CAAC;AAEF,SAAgB,wBAAwB;IACtC,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACzD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAgB,sBAAsB,CACpC,WAAmB,EACnB,WAAmB;IAEnB,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,gBAAgB,WAAW;OACjC,WAAW;;;qEAGmD;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,kBAAkB,GAAG;;;;;;wHAM6F,CAAC;AAEzH,SAAgB,uBAAuB;IACrC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,kBAAkB;KAC5B,CAAC;AACJ,CAAC;AAED,SAAgB,qBAAqB,CACnC,WAAmB,EACnB,0BAAkC,EAClC,qBAA6B;IAE7B,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,gBAAgB,WAAW;gCACR,IAAI,CAAC,SAAS,CAAC,0BAA0B,EAAE,IAAI,EAAE,CAAC,CAAC;2BACxD,IAAI,CAAC,SAAS,CAAC,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACxD;KACd,CAAC;AACJ,CAAC;AAED,MAAM,oBAAoB,GAAG;;;;;;;uEAO0C,CAAC;AAExE,SAAgB,yBAAyB;IACvC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,oBAAoB;KAC9B,CAAC;AACJ,CAAC;AAED,SAAgB,mBAAmB,CACjC,WAAmB,EACnB,kBAA0B,EAC1B,UAAkB,EAClB,WAAmB;IAEnB,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,gBAAgB,WAAW;qBACnB,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;cAClD,UAAU;eACT,WAAW,EAAE;KACzB,CAAC;AACJ,CAAC;AAED,UAAU;AACV,MAAM,mBAAmB,GAAG;;;;;;;CAO3B,CAAC;AACF,SAAgB,wBAAwB;IACtC,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEzD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAgB,uBAAuB,CACrC,WAAmB,EACnB,WAAmB;IAEnB,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,gBAAgB,WAAW;OACjC,WAAW,EAAE;KACjB,CAAC;AACJ,CAAC;AAED,MAAM;AACN,MAAM,eAAe,GAAG;;CAEvB,CAAC;AACF,SAAgB,oBAAoB;IAClC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,eAAe;KACzB,CAAC;AACJ,CAAC;AAED,SAAgB,kBAAkB,CAChC,QAAgB;IAEhB,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,aAAa,QAAQ,EAAE;KACjC,CAAC;AACJ,CAAC"}

package/dist/lib/vision.js

@@ -0,0 +1,194 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScreenshotService = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const sharp_1 = __importDefault(require("sharp"));
+const child_process_1 = require("child_process");
+class ScreenshotService {
+    constructor(page, selectorMap, verbose, isDebugEnabled = false) {
+        this.annotationBoxes = [];
+        this.numberPositions = [];
+        this.page = page;
+        this.selectorMap = selectorMap;
+        this.isDebugEnabled = isDebugEnabled;
+        this.verbose = verbose;
+    }
+    log({ category, message, level = 1, }) {
+        if (this.verbose >= level) {
+            const categoryString = category ? `:${category}` : "";
+            console.log(`[stagehand${categoryString}] ${message}`);
+        }
+    }
+    getScreenshot() {
+        return __awaiter(this, arguments, void 0, function* (fullpage = true, quality) {
+            if (quality && (quality < 0 || quality > 100)) {
+                throw new Error("quality must be between 0 and 100");
+            }
+            return yield this.page.screenshot({
+                fullPage: fullpage,
+                quality,
+                type: "jpeg",
+            });
+        });
+    }
+    getScreenshotPixelCount(screenshot) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const image = (0, sharp_1.default)(screenshot);
+            const metadata = yield image.metadata();
+            if (!metadata.width || !metadata.height) {
+                this.log({
+                    category: "Error",
+                    message: "Unable to determine image dimensions.",
+                    level: 0,
+                });
+                throw new Error("Unable to determine image dimensions.");
+            }
+            const pixelCount = metadata.width * metadata.height;
+            this.log({
+                category: "Info",
+                message: `Screenshot pixel count: ${pixelCount}`,
+                level: 1,
+            });
+            return pixelCount;
+        });
+    }
+    getAnnotatedScreenshot(fullpage) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.annotationBoxes = [];
+            this.numberPositions = [];
+            const screenshot = yield this.getScreenshot(fullpage);
+            const image = (0, sharp_1.default)(screenshot);
+            const { width, height } = yield image.metadata();
+            // this.log({
+            //   category: "Debug",
+            //   message: `Annotating screenshot ${JSON.stringify(this.selectorMap)}`,
+            //   level: 2,
+            // });
+            const svgAnnotations = yield Promise.all(Object.entries(this.selectorMap).map((_a) => __awaiter(this, [_a], void 0, function* ([id, selector]) { return this.createElementAnnotation(id, selector); })));
+            const scrollPosition = yield this.page.evaluate(() => {
+                return {
+                    scrollX: window.scrollX,
+                    scrollY: window.scrollY,
+                };
+            });
+            const svg = `
+      <svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg" style="position:absolute;left:${-scrollPosition.scrollX}px;top:${-scrollPosition.scrollY}px;">
+        ${svgAnnotations.join("")}
+      </svg>
+    `;
+            const annotatedScreenshot = yield image
+                .composite([{ input: Buffer.from(svg), top: 0, left: 0 }])
+                .toBuffer();
+            if (this.isDebugEnabled) {
+                yield ScreenshotService.saveAndOpenScreenshot(annotatedScreenshot);
+            }
+            return annotatedScreenshot;
+        });
+    }
+    createElementAnnotation(id, selector) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const element = yield this.page.locator(`xpath=${selector}`).first();
+                const box = yield element.boundingBox();
+                if (!box) {
+                    this.log({
+                        category: "Debug",
+                        message: `No bounding box for element ${id}`,
+                        level: 2,
+                    });
+                    return "";
+                }
+                const scrollPosition = yield this.page.evaluate(() => ({
+                    scrollX: window.scrollX,
+                    scrollY: window.scrollY,
+                }));
+                const adjustedBox = {
+                    x: box.x + scrollPosition.scrollX,
+                    y: box.y + scrollPosition.scrollY,
+                    width: box.width,
+                    height: box.height,
+                    id: id,
+                };
+                this.annotationBoxes.push(adjustedBox);
+                const numberPosition = this.findNonOverlappingNumberPosition(adjustedBox);
+                const circleRadius = 12;
+                return `
+        <rect x="${adjustedBox.x}" y="${adjustedBox.y}" width="${adjustedBox.width}" height="${adjustedBox.height}" 
+              fill="none" stroke="red" stroke-width="2" />
+        <circle cx="${numberPosition.x}" cy="${numberPosition.y}" r="${circleRadius}" fill="white" stroke="red" stroke-width="2" />
+        <text x="${numberPosition.x}" y="${numberPosition.y}" fill="red" font-size="16" font-weight="bold" 
+              text-anchor="middle" dominant-baseline="central">
+          ${id}
+        </text>
+      `;
+            }
+            catch (error) {
+                this.log({
+                    category: "Error",
+                    message: `Failed to create annotation for element ${id}: ${error}`,
+                    level: 0,
+                });
+                return "";
+            }
+        });
+    }
+    findNonOverlappingNumberPosition(box) {
+        const circleRadius = 12;
+        let position = {
+            x: box.x - circleRadius,
+            y: box.y - circleRadius,
+        };
+        let attempts = 0;
+        const maxAttempts = 10;
+        const offset = 5;
+        while (this.isNumberOverlapping(position) && attempts < maxAttempts) {
+            position.y += offset;
+            attempts++;
+        }
+        this.numberPositions.push(position);
+        return position;
+    }
+    isNumberOverlapping(position) {
+        const circleRadius = 12;
+        return this.numberPositions.some((existingPosition) => Math.sqrt(Math.pow(position.x - existingPosition.x, 2) +
+            Math.pow(position.y - existingPosition.y, 2)) <
+            circleRadius * 2);
+    }
+    static saveAndOpenScreenshot(screenshot) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const screenshotDir = path_1.default.join(process.cwd(), "screenshots");
+            if (!fs_1.default.existsSync(screenshotDir)) {
+                fs_1.default.mkdirSync(screenshotDir);
+            }
+            const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+            const filename = path_1.default.join(screenshotDir, `screenshot-${timestamp}.png`);
+            fs_1.default.writeFileSync(filename, screenshot);
+            console.log(`Screenshot saved to: ${filename}`);
+            // Open the screenshot with the default image viewer
+            if (process.platform === "win32") {
+                (0, child_process_1.exec)(`start ${filename}`);
+            }
+            else if (process.platform === "darwin") {
+                (0, child_process_1.exec)(`open ${filename}`);
+            }
+            else {
+                (0, child_process_1.exec)(`xdg-open ${filename}`);
+            }
+        });
+    }
+}
+exports.ScreenshotService = ScreenshotService;
+//# sourceMappingURL=vision.js.map

package/dist/lib/vision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../lib/vision.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,4CAAoB;AACpB,gDAAwB;AACxB,kDAA0B;AAC1B,iDAAqC;AAerC,MAAa,iBAAiB;IAQ5B,YACE,IAAU,EACV,WAAmC,EACnC,OAAkB,EAClB,iBAA0B,KAAK;QATzB,oBAAe,GAAoB,EAAE,CAAC;QACtC,oBAAe,GAAqB,EAAE,CAAC;QAU7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,GAAG,CAAC,EACF,QAAQ,EACR,OAAO,EACP,KAAK,GAAG,CAAC,GAKV;QACC,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,aAAa,cAAc,KAAK,OAAO,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAEK,aAAa;6DACjB,WAAoB,IAAI,EACxB,OAAgB;YAEhB,IAAI,OAAO,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,GAAG,CAAC,EAAE,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACvD,CAAC;YAED,OAAO,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;gBAChC,QAAQ,EAAE,QAAQ;gBAClB,OAAO;gBACP,IAAI,EAAE,MAAM;aACb,CAAC,CAAC;QACL,CAAC;KAAA;IAEK,uBAAuB,CAAC,UAAkB;;YAC9C,MAAM,KAAK,GAAG,IAAA,eAAK,EAAC,UAAU,CAAC,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;YAExC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACxC,IAAI,CAAC,GAAG,CAAC;oBACP,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,uCAAuC;oBAChD,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YAED,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC;YACpD,IAAI,CAAC,GAAG,CAAC;gBACP,QAAQ,EAAE,MAAM;gBAChB,OAAO,EAAE,2BAA2B,UAAU,EAAE;gBAChD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,OAAO,UAAU,CAAC;QACpB,CAAC;KAAA;IAEK,sBAAsB,CAAC,QAAiB;;YAC5C,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;YAE1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,IAAA,eAAK,EAAC,UAAU,CAAC,CAAC;YAEhC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjD,aAAa;YACb,uBAAuB;YACvB,0EAA0E;YAC1E,cAAc;YACd,MAAM;YAEN,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,KAAuB,EAAE,0CAAlB,CAAC,EAAE,EAAE,QAAQ,CAAC,IACxD,OAAA,IAAI,CAAC,uBAAuB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA,GAAA,CAC3C,CACF,CAAC;YAEF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACnD,OAAO;oBACL,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,GAAG,GAAG;oBACI,KAAK,aAAa,MAAM,sEAAsE,CAAC,cAAc,CAAC,OAAO,UAAU,CAAC,cAAc,CAAC,OAAO;UAChK,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;;KAE5B,CAAC;YAEF,MAAM,mBAAmB,GAAG,MAAM,KAAK;iBACpC,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;iBACzD,QAAQ,EAAE,CAAC;YAEd,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,MAAM,iBAAiB,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,CAAC;YACrE,CAAC;YAED,OAAO,mBAAmB,CAAC;QAC7B,CAAC;KAAA;IAEa,uBAAuB,CACnC,EAAU,EACV,QAAgB;;YAEhB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC;gBAExC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,IAAI,CAAC,GAAG,CAAC;wBACP,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,+BAA+B,EAAE,EAAE;wBAC5C,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;oBACrD,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC,CAAC;gBAEJ,MAAM,WAAW,GAAG;oBAClB,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC,OAAO;oBACjC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC,OAAO;oBACjC,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,EAAE,EAAE,EAAE;iBACP,CAAC;gBAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAEvC,MAAM,cAAc,GAAG,IAAI,CAAC,gCAAgC,CAAC,WAAW,CAAC,CAAC;gBAE1E,MAAM,YAAY,GAAG,EAAE,CAAC;gBAExB,OAAO;mBACM,WAAW,CAAC,CAAC,QAAQ,WAAW,CAAC,CAAC,YAAY,WAAW,CAAC,KAAK,aAAa,WAAW,CAAC,MAAM;;sBAE3F,cAAc,CAAC,CAAC,SAAS,cAAc,CAAC,CAAC,QAAQ,YAAY;mBAChE,cAAc,CAAC,CAAC,QAAQ,cAAc,CAAC,CAAC;;YAE/C,EAAE;;OAEP,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,GAAG,CAAC;oBACP,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,2CAA2C,EAAE,KAAK,KAAK,EAAE;oBAClE,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;KAAA;IAEO,gCAAgC,CAAC,GAAkB;QACzD,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,IAAI,QAAQ,GAAmB;YAC7B,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,YAAY;YACvB,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,YAAY;SACxB,CAAC;QAEF,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,WAAW,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,CAAC,CAAC;QAEjB,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,WAAW,EAAE,CAAC;YACpE,QAAQ,CAAC,CAAC,IAAI,MAAM,CAAC;YACrB,QAAQ,EAAE,CAAC;QACb,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,mBAAmB,CAAC,QAAwB;QAClD,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAC9B,CAAC,gBAAgB,EAAE,EAAE,CACnB,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAC/C;YACD,YAAY,GAAG,CAAC,CACnB,CAAC;IACJ,CAAC;IAED,MAAM,CAAO,qBAAqB,CAAC,UAAkB;;YACnD,MAAM,aAAa,GAAG,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,CAAC,CAAC;YAC9D,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBAClC,YAAE,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9B,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,SAAS,MAAM,CAAC,CAAC;YAEzE,YAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;YAEhD,oDAAoD;YACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACjC,IAAA,oBAAI,EAAC,SAAS,QAAQ,EAAE,CAAC,CAAC;YAC5B,CAAC;iBAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACzC,IAAA,oBAAI,EAAC,QAAQ,QAAQ,EAAE,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,IAAA,oBAAI,EAAC,YAAY,QAAQ,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;KAAA;CACF;AAhOD,8CAgOC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserbasehq/stagehand",
-  "version": "1.0.3",
+  "version": "1.1.0-alpha.1",
   "description": "An AI web browsing framework focused on simplicity and extensibility.",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
@@ -50,6 +50,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.27.3",
+    "@browserbasehq/sdk": "^2.0.0",
     "anthropic": "^0.0.0",
     "anthropic-ai": "^0.0.10",
     "sharp": "^0.33.5",