@browserbasehq/stagehand 1.0.3 → 1.1.0-alpha.1

package/dist/lib/index.js

@@ -0,0 +1,978 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Stagehand = void 0;
+const test_1 = require("@playwright/test");
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = __importDefault(require("fs"));
+const inference_1 = require("./inference");
+const LLMProvider_1 = require("./llm/LLMProvider");
+const path_1 = __importDefault(require("path"));
+const browserbase_1 = __importDefault(require("./browserbase"));
+const vision_1 = require("./vision");
+const LLMClient_1 = require("./llm/LLMClient");
+require("dotenv").config({ path: ".env" });
+function getBrowser() {
+    return __awaiter(this, arguments, void 0, function* (env = "LOCAL", headless = false, logger) {
+        if (env === "BROWSERBASE" && !process.env.BROWSERBASE_API_KEY) {
+            logger({
+                category: "Init",
+                message: "BROWSERBASE_API_KEY is required to use BROWSERBASE env. Defaulting to LOCAL.",
+                level: 0,
+            });
+            env = "LOCAL";
+        }
+        if (env === "BROWSERBASE" && !process.env.BROWSERBASE_PROJECT_ID) {
+            logger({
+                category: "Init",
+                message: "BROWSERBASE_PROJECT_ID is required to use BROWSERBASE env. Defaulting to LOCAL.",
+                level: 0,
+            });
+            env = "LOCAL";
+        }
+        if (env === "BROWSERBASE") {
+            let debugUrl = undefined;
+            let sessionUrl = undefined;
+            logger({
+                category: "Init",
+                message: "Connecting you to Browserbase...",
+                level: 0,
+            });
+            const browserbase = new browserbase_1.default();
+            const { sessionId, connectUrl } = yield browserbase.createSession();
+            const browser = yield test_1.chromium.connectOverCDP(connectUrl);
+            debugUrl = yield browserbase.retrieveDebugConnectionURL(sessionId);
+            sessionUrl = `https://www.browserbase.com/sessions/${sessionId}`;
+            logger({
+                category: "Init",
+                message: `Browserbase session started.\n\nSession Url: ${sessionUrl}\n\nLive debug accessible here: ${debugUrl}.`,
+                level: 0,
+            });
+            const context = browser.contexts()[0];
+            return { browser, context, debugUrl, sessionUrl };
+        }
+        else {
+            logger({
+                category: "Init",
+                message: `Launching local browser in ${headless ? "headless" : "headed"} mode`,
+                level: 0,
+            });
+            const tmpDir = fs_1.default.mkdtempSync(`/tmp/pwtest`);
+            fs_1.default.mkdirSync(`${tmpDir}/userdir/Default`, { recursive: true });
+            const defaultPreferences = {
+                plugins: {
+                    always_open_pdf_externally: true,
+                },
+            };
+            fs_1.default.writeFileSync(`${tmpDir}/userdir/Default/Preferences`, JSON.stringify(defaultPreferences));
+            const downloadsPath = `${process.cwd()}/downloads`;
+            fs_1.default.mkdirSync(downloadsPath, { recursive: true });
+            const context = yield test_1.chromium.launchPersistentContext(`${tmpDir}/userdir`, {
+                acceptDownloads: true,
+                headless: headless,
+                viewport: {
+                    width: 1250,
+                    height: 800,
+                },
+                locale: "en-US",
+                timezoneId: "America/New_York",
+                deviceScaleFactor: 1,
+                args: [
+                    "--enable-webgl",
+                    "--use-gl=swiftshader",
+                    "--enable-accelerated-2d-canvas",
+                    "--disable-blink-features=AutomationControlled",
+                    "--disable-web-security",
+                ],
+                bypassCSP: true,
+            });
+            logger({
+                category: "Init",
+                message: "Local browser started successfully.",
+            });
+            yield applyStealthScripts(context);
+            return { context };
+        }
+    });
+}
+function applyStealthScripts(context) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield context.addInitScript(() => {
+            // Override the navigator.webdriver property
+            Object.defineProperty(navigator, "webdriver", {
+                get: () => undefined,
+            });
+            // Mock languages and plugins to mimic a real browser
+            Object.defineProperty(navigator, "languages", {
+                get: () => ["en-US", "en"],
+            });
+            Object.defineProperty(navigator, "plugins", {
+                get: () => [1, 2, 3, 4, 5],
+            });
+            // Remove Playwright-specific properties
+            delete window.__playwright;
+            delete window.__pw_manual;
+            delete window.__PW_inspect;
+            // Redefine the headless property
+            Object.defineProperty(navigator, "headless", {
+                get: () => false,
+            });
+            // Override the permissions API
+            const originalQuery = window.navigator.permissions.query;
+            window.navigator.permissions.query = (parameters) => parameters.name === "notifications"
+                ? Promise.resolve({
+                    state: Notification.permission,
+                })
+                : originalQuery(parameters);
+        });
+    });
+}
+class Stagehand {
+    constructor({ env, verbose = 0, debugDom = false, llmProvider, headless = false, logger, domSettleTimeoutMs = 60000, } = {
+        env: "BROWSERBASE",
+    }) {
+        // Logging
+        this.pending_logs_to_send_to_browserbase = [];
+        this.is_processing_browserbase_logs = false;
+        this.externalLogger = logger;
+        this.logger = this.log.bind(this);
+        this.llmProvider = llmProvider || new LLMProvider_1.LLMProvider(this.logger);
+        this.env = env;
+        this.observations = {};
+        this.actions = {};
+        this.verbose = verbose;
+        this.debugDom = debugDom;
+        this.defaultModelName = "gpt-4o";
+        this.headless = headless;
+        this.domSettleTimeoutMs = domSettleTimeoutMs;
+    }
+    init() {
+        return __awaiter(this, arguments, void 0, function* ({ modelName = "gpt-4o", } = {}) {
+            const { context, debugUrl, sessionUrl } = yield getBrowser(this.env, this.headless, this.logger).catch((e) => {
+                console.error("Error in init:", e);
+                return { context: undefined, debugUrl: undefined, sessionUrl: undefined };
+            });
+            this.context = context;
+            this.page = context.pages()[0];
+            this.defaultModelName = modelName;
+            // Overload the page.goto method
+            const originalGoto = this.page.goto.bind(this.page);
+            this.page.goto = (url, options) => __awaiter(this, void 0, void 0, function* () {
+                const result = yield originalGoto(url, options);
+                yield this.page.waitForLoadState("domcontentloaded");
+                yield this._waitForSettledDom();
+                return result;
+            });
+            // Set the browser to headless mode if specified
+            if (this.headless) {
+                yield this.page.setViewportSize({ width: 1280, height: 720 });
+            }
+            // This can be greatly improved, but the tldr is we put our built web scripts in dist, which should always
+            // be one level above our running directly across evals, example, and as a package
+            yield this.page.addInitScript({
+                path: path_1.default.join(__dirname, "..", "dist", "dom", "build", "process.js"),
+            });
+            yield this.page.addInitScript({
+                path: path_1.default.join(__dirname, "..", "dist", "dom", "build", "utils.js"),
+            });
+            yield this.page.addInitScript({
+                path: path_1.default.join(__dirname, "..", "dist", "dom", "build", "debug.js"),
+            });
+            return { debugUrl, sessionUrl };
+        });
+    }
+    initFromPage(page, modelName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.page = page;
+            this.context = page.context();
+            this.defaultModelName = modelName || this.defaultModelName;
+            const originalGoto = this.page.goto.bind(this.page);
+            this.page.goto = (url, options) => __awaiter(this, void 0, void 0, function* () {
+                const result = yield originalGoto(url, options);
+                yield this.page.waitForLoadState("domcontentloaded");
+                yield this._waitForSettledDom();
+                return result;
+            });
+            // Set the browser to headless mode if specified
+            if (this.headless) {
+                yield this.page.setViewportSize({ width: 1280, height: 720 });
+            }
+            // Add initialization scripts
+            yield this.page.addInitScript({
+                path: path_1.default.join(__dirname, "..", "dist", "dom", "build", "process.js"),
+            });
+            yield this.page.addInitScript({
+                path: path_1.default.join(__dirname, "..", "dist", "dom", "build", "utils.js"),
+            });
+            yield this.page.addInitScript({
+                path: path_1.default.join(__dirname, "..", "dist", "dom", "build", "debug.js"),
+            });
+            return { context: this.context };
+        });
+    }
+    log({ message, category, level, }) {
+        const logObj = { category, message, level };
+        logObj.level = logObj.level || 1;
+        // Normal Logging
+        if (this.externalLogger) {
+            this.externalLogger(logObj);
+        }
+        else {
+            const categoryString = logObj.category ? `:${logObj.category}` : "";
+            const logMessage = `[stagehand${categoryString}] ${logObj.message}`;
+            console.log(logMessage);
+        }
+        // Add the logs to the browserbase session
+        this.pending_logs_to_send_to_browserbase.push(Object.assign(Object.assign({}, logObj), { id: Math.random().toString(36).substring(2, 15) }));
+        this._run_browserbase_log_processing_cycle();
+    }
+    _run_browserbase_log_processing_cycle() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.is_processing_browserbase_logs) {
+                return;
+            }
+            this.is_processing_browserbase_logs = true;
+            const pending_logs = [...this.pending_logs_to_send_to_browserbase];
+            for (const logObj of pending_logs) {
+                yield this._log_to_browserbase(logObj);
+            }
+            this.is_processing_browserbase_logs = false;
+        });
+    }
+    _log_to_browserbase(logObj) {
+        return __awaiter(this, void 0, void 0, function* () {
+            logObj.level = logObj.level || 1;
+            if (!this.page) {
+                return;
+            }
+            if (this.verbose >= logObj.level) {
+                yield this.page
+                    .evaluate((logObj) => {
+                    const logMessage = `[stagehand${logObj.category ? `:${logObj.category}` : ""}] ${logObj.message}`;
+                    if (logObj.message.toLowerCase().includes("trace") ||
+                        logObj.message.toLowerCase().includes("error:")) {
+                        console.error(logMessage);
+                    }
+                    else {
+                        console.log(logMessage);
+                    }
+                }, logObj)
+                    .then(() => {
+                    this.pending_logs_to_send_to_browserbase =
+                        this.pending_logs_to_send_to_browserbase.filter((log) => log.id !== logObj.id);
+                })
+                    .catch((e) => {
+                    // NAVIDTODO: Rerun the log call on the new page
+                    // This is expected to happen when the user is changing pages
+                    // console.error("Logging Error:", e);
+                });
+            }
+        });
+    }
+    _waitForSettledDom(timeoutMs) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const timeout = timeoutMs !== null && timeoutMs !== void 0 ? timeoutMs : this.domSettleTimeoutMs;
+                let timeoutHandle;
+                const timeoutPromise = new Promise((resolve, reject) => {
+                    timeoutHandle = setTimeout(() => {
+                        this.log({
+                            category: "dom",
+                            message: `DOM settle timeout of ${timeout}ms exceeded, continuing anyway`,
+                            level: 1,
+                        });
+                        resolve();
+                    }, timeout);
+                });
+                try {
+                    yield Promise.race([
+                        this.page.evaluate(() => {
+                            return new Promise((resolve) => {
+                                if (typeof window.waitForDomSettle === "function") {
+                                    window.waitForDomSettle().then(resolve);
+                                }
+                                else {
+                                    console.warn("waitForDomSettle is not defined, considering DOM as settled");
+                                    resolve();
+                                }
+                            });
+                        }),
+                        this.page.waitForLoadState("domcontentloaded"),
+                        this.page.waitForSelector("body"),
+                        timeoutPromise,
+                    ]);
+                }
+                finally {
+                    clearTimeout(timeoutHandle);
+                }
+            }
+            catch (e) {
+                this.log({
+                    category: "dom",
+                    message: `Error in waitForSettledDom: ${e.message}\nTrace: ${e.stack}`,
+                    level: 1,
+                });
+            }
+        });
+    }
+    startDomDebug() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.page
+                    .evaluate(() => {
+                    if (typeof window.debugDom === "function") {
+                        window.debugDom();
+                    }
+                    else {
+                        this.log({
+                            category: "dom",
+                            message: "debugDom is not defined",
+                            level: 1,
+                        });
+                    }
+                })
+                    .catch(() => { });
+            }
+            catch (e) {
+                this.log({
+                    category: "dom",
+                    message: `Error in startDomDebug: ${e.message}\nTrace: ${e.stack}`,
+                    level: 1,
+                });
+            }
+        });
+    }
+    cleanupDomDebug() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.debugDom) {
+                yield this.page.evaluate(() => window.cleanupDebug()).catch(() => { });
+            }
+        });
+    }
+    // Recording
+    _generateId(operation) {
+        return crypto_1.default.createHash("sha256").update(operation).digest("hex");
+    }
+    _recordObservation(instruction, result) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const id = this._generateId(instruction);
+            this.observations[id] = { result, instruction };
+            return id;
+        });
+    }
+    _recordAction(action, result) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const id = this._generateId(action);
+            this.actions[id] = { result, action };
+            return id;
+        });
+    }
+    // Main methods
+    _extract(_a) {
+        return __awaiter(this, arguments, void 0, function* ({ instruction, schema, progress = "", content = {}, chunksSeen = [], modelName, }) {
+            this.log({
+                category: "extraction",
+                message: `starting extraction '${instruction}'`,
+                level: 1,
+            });
+            yield this._waitForSettledDom();
+            yield this.startDomDebug();
+            const { outputString, chunk, chunks } = yield this.page.evaluate((chunksSeen) => window.processDom(chunksSeen !== null && chunksSeen !== void 0 ? chunksSeen : []), chunksSeen);
+            this.log({
+                category: "extraction",
+                message: `received output from processDom. Current chunk index: ${chunk}, Number of chunks left: ${chunks.length - chunksSeen.length}`,
+                level: 1,
+            });
+            const extractionResponse = yield (0, inference_1.extract)({
+                instruction,
+                progress,
+                previouslyExtractedContent: content,
+                domElements: outputString,
+                llmProvider: this.llmProvider,
+                schema,
+                modelName: modelName || this.defaultModelName,
+                chunksSeen: chunksSeen.length,
+                chunksTotal: chunks.length,
+            });
+            const { metadata: { progress: newProgress, completed } } = extractionResponse, output = __rest(extractionResponse, ["metadata"]);
+            yield this.cleanupDomDebug();
+            this.log({
+                category: "extraction",
+                message: `received extraction response: ${JSON.stringify(extractionResponse)}`,
+                level: 1,
+            });
+            chunksSeen.push(chunk);
+            if (completed || chunksSeen.length === chunks.length) {
+                this.log({
+                    category: "extraction",
+                    message: `response: ${JSON.stringify(extractionResponse)}`,
+                    level: 1,
+                });
+                return output;
+            }
+            else {
+                this.log({
+                    category: "extraction",
+                    message: `continuing extraction, progress: '${newProgress}'`,
+                    level: 1,
+                });
+                yield this._waitForSettledDom();
+                return this._extract({
+                    instruction,
+                    schema,
+                    progress: newProgress,
+                    content: output,
+                    chunksSeen,
+                    modelName,
+                });
+            }
+        });
+    }
+    _observe(_a) {
+        return __awaiter(this, arguments, void 0, function* ({ instruction, useVision, fullPage, modelName, }) {
+            if (!instruction) {
+                instruction = `Find elements that can be used for any future actions in the page. These may be navigation links, related pages, section/subsection links, buttons, or other interactive elements. Be comprehensive: if there are multiple elements that may be relevant for future actions, return all of them.`;
+            }
+            const model = modelName !== null && modelName !== void 0 ? modelName : this.defaultModelName;
+            this.log({
+                category: "observation",
+                message: `starting observation: ${instruction}`,
+                level: 1,
+            });
+            yield this._waitForSettledDom();
+            yield this.startDomDebug();
+            let { outputString, selectorMap } = yield this.page.evaluate((fullPage) => fullPage ? window.processAllOfDom() : window.processDom([]), fullPage);
+            let annotatedScreenshot;
+            if (useVision === true) {
+                if (!LLMClient_1.modelsWithVision.includes(model)) {
+                    this.log({
+                        category: "observation",
+                        message: `${model} does not support vision. Skipping vision processing.`,
+                        level: 1,
+                    });
+                }
+                else {
+                    const screenshotService = new vision_1.ScreenshotService(this.page, selectorMap, this.verbose);
+                    annotatedScreenshot =
+                        yield screenshotService.getAnnotatedScreenshot(fullPage);
+                    outputString = "n/a. use the image to find the elements.";
+                }
+            }
+            const observationResponse = yield (0, inference_1.observe)({
+                instruction,
+                domElements: outputString,
+                llmProvider: this.llmProvider,
+                modelName: modelName || this.defaultModelName,
+                image: annotatedScreenshot,
+            });
+            const elementsWithSelectors = observationResponse.elements.map((element) => {
+                const { elementId } = element, rest = __rest(element, ["elementId"]);
+                return Object.assign(Object.assign({}, rest), { selector: `xpath=${selectorMap[elementId]}` });
+            });
+            yield this.cleanupDomDebug();
+            this._recordObservation(instruction, elementsWithSelectors);
+            this.log({
+                category: "observation",
+                message: `found element ${JSON.stringify(elementsWithSelectors)}`,
+                level: 1,
+            });
+            yield this._recordObservation(instruction, elementsWithSelectors);
+            return elementsWithSelectors;
+        });
+    }
+    _act(_a) {
+        return __awaiter(this, arguments, void 0, function* ({ action, steps = "", chunksSeen, modelName, useVision, verifierUseVision, retries = 0, }) {
+            var _b;
+            const model = modelName !== null && modelName !== void 0 ? modelName : this.defaultModelName;
+            if (!LLMClient_1.modelsWithVision.includes(model) &&
+                (useVision !== false || verifierUseVision)) {
+                this.log({
+                    category: "action",
+                    message: `${model} does not support vision, but useVision was set to ${useVision}. Defaulting to false.`,
+                    level: 1,
+                });
+                useVision = false;
+                verifierUseVision = false;
+            }
+            this.log({
+                category: "action",
+                message: `Running / Continuing action: ${action} on page: ${this.page.url()}`,
+                level: 2,
+            });
+            yield this._waitForSettledDom();
+            yield this.startDomDebug();
+            this.log({
+                category: "action",
+                message: `Processing DOM...`,
+                level: 2,
+            });
+            const { outputString, selectorMap, chunk, chunks } = yield this.page.evaluate(({ chunksSeen }) => {
+                // @ts-ignore
+                return window.processDom(chunksSeen);
+            }, { chunksSeen });
+            this.log({
+                category: "action",
+                message: `Looking at chunk ${chunk}. Chunks left: ${chunks.length - chunksSeen.length}`,
+                level: 1,
+            });
+            // Prepare annotated screenshot if vision is enabled
+            let annotatedScreenshot;
+            if (useVision === true) {
+                if (!LLMClient_1.modelsWithVision.includes(model)) {
+                    this.log({
+                        category: "action",
+                        message: `${model} does not support vision. Skipping vision processing.`,
+                        level: 1,
+                    });
+                }
+                else {
+                    const screenshotService = new vision_1.ScreenshotService(this.page, selectorMap, this.verbose);
+                    annotatedScreenshot =
+                        yield screenshotService.getAnnotatedScreenshot(false);
+                }
+            }
+            const response = yield (0, inference_1.act)({
+                action,
+                domElements: outputString,
+                steps,
+                llmProvider: this.llmProvider,
+                modelName: model,
+                screenshot: annotatedScreenshot,
+                logger: this.logger,
+            });
+            this.log({
+                category: "action",
+                message: `Received response from LLM: ${JSON.stringify(response)}`,
+                level: 1,
+            });
+            yield this.cleanupDomDebug();
+            if (!response) {
+                if (chunksSeen.length + 1 < chunks.length) {
+                    chunksSeen.push(chunk);
+                    this.log({
+                        category: "action",
+                        message: `No action found in current chunk. Chunks seen: ${chunksSeen.length}.`,
+                        level: 1,
+                    });
+                    return this._act({
+                        action,
+                        steps: steps +
+                            (!steps.endsWith("\n") ? "\n" : "") +
+                            "## Step: Scrolled to another section\n",
+                        chunksSeen,
+                        modelName,
+                        useVision,
+                        verifierUseVision,
+                    });
+                }
+                else if (useVision === "fallback") {
+                    this.log({
+                        category: "action",
+                        message: `Switching to vision-based processing`,
+                        level: 1,
+                    });
+                    yield this.page.evaluate(() => window.scrollToHeight(0));
+                    return yield this._act({
+                        action,
+                        steps,
+                        chunksSeen,
+                        modelName,
+                        useVision: true,
+                        verifierUseVision,
+                    });
+                }
+                else {
+                    return {
+                        success: false,
+                        message: `Action was not able to be completed.`,
+                        action: action,
+                    };
+                }
+            }
+            // Action found, proceed to execute
+            const elementId = response["element"];
+            const xpath = selectorMap[elementId];
+            const method = response["method"];
+            const args = response["args"];
+            // Get the element text from the outputString
+            const elementLines = outputString.split("\n");
+            const elementText = ((_b = elementLines
+                .find((line) => line.startsWith(`${elementId}:`))) === null || _b === void 0 ? void 0 : _b.split(":")[1]) || "Element not found";
+            this.log({
+                category: "action",
+                message: `Executing method: ${method} on element: ${elementId} (xpath: ${xpath}) with args: ${JSON.stringify(args)}`,
+                level: 1,
+            });
+            let urlChangeString = "";
+            const locator = this.page.locator(`xpath=${xpath}`).first();
+            try {
+                const initialUrl = this.page.url();
+                if (method === "scrollIntoView") {
+                    this.log({
+                        category: "action",
+                        message: `Scrolling element into view`,
+                        level: 2,
+                    });
+                    try {
+                        yield locator
+                            .evaluate((element) => {
+                            element.scrollIntoView({ behavior: "smooth", block: "center" });
+                        })
+                            .catch((e) => {
+                            this.log({
+                                category: "action",
+                                message: `Error scrolling element into view: ${e.message}\nTrace: ${e.stack}`,
+                                level: 1,
+                            });
+                        });
+                    }
+                    catch (e) {
+                        this.log({
+                            category: "action",
+                            message: `Error scrolling element into view (Retries ${retries}): ${e.message}\nTrace: ${e.stack}`,
+                            level: 1,
+                        });
+                        if (retries < 2) {
+                            return this._act({
+                                action,
+                                steps,
+                                modelName,
+                                useVision,
+                                verifierUseVision,
+                                retries: retries + 1,
+                                chunksSeen,
+                            });
+                        }
+                    }
+                }
+                else if (method === "fill" || method === "type") {
+                    try {
+                        yield locator.fill("");
+                        yield locator.click();
+                        const text = args[0];
+                        for (const char of text) {
+                            yield this.page.keyboard.type(char, {
+                                delay: Math.random() * 50 + 25,
+                            });
+                        }
+                    }
+                    catch (e) {
+                        this.log({
+                            category: "action",
+                            message: `Error filling element (Retries ${retries}): ${e.message}\nTrace: ${e.stack}`,
+                            level: 1,
+                        });
+                        if (retries < 2) {
+                            return this._act({
+                                action,
+                                steps,
+                                modelName,
+                                useVision,
+                                verifierUseVision,
+                                retries: retries + 1,
+                                chunksSeen,
+                            });
+                        }
+                    }
+                }
+                else if (method === "press") {
+                    try {
+                        const key = args[0];
+                        yield this.page.keyboard.press(key);
+                    }
+                    catch (e) {
+                        this.log({
+                            category: "action",
+                            message: `Error pressing key (Retries ${retries}): ${e.message}\nTrace: ${e.stack}`,
+                            level: 1,
+                        });
+                        if (retries < 2) {
+                            return this._act({
+                                action,
+                                steps,
+                                modelName,
+                                useVision,
+                                verifierUseVision,
+                                retries: retries + 1,
+                                chunksSeen,
+                            });
+                        }
+                    }
+                }
+                else if (typeof locator[method] === "function") {
+                    // Log current URL before action
+                    this.log({
+                        category: "action",
+                        message: `Page URL before action: ${this.page.url()}`,
+                        level: 2,
+                    });
+                    // Perform the action
+                    try {
+                        // @ts-ignore
+                        yield locator[method](...args);
+                    }
+                    catch (e) {
+                        this.log({
+                            category: "action",
+                            message: `Error performing method ${method} with args ${JSON.stringify(args)} (Retries: ${retries}): ${e.message}\nTrace: ${e.stack}`,
+                            level: 1,
+                        });
+                        if (retries < 2) {
+                            return this._act({
+                                action,
+                                steps,
+                                modelName,
+                                useVision,
+                                verifierUseVision,
+                                retries: retries + 1,
+                                chunksSeen,
+                            });
+                        }
+                    }
+                    // Handle navigation if a new page is opened
+                    if (method === "click") {
+                        this.log({
+                            category: "action",
+                            message: `Clicking element, checking for page navigation`,
+                            level: 1,
+                        });
+                        // NAVIDNOTE: Should this happen before we wait for locator[method]?
+                        const newOpenedTab = yield Promise.race([
+                            new Promise((resolve) => {
+                                this.context.once("page", (page) => resolve(page));
+                                setTimeout(() => resolve(null), 1500);
+                            }),
+                        ]);
+                        this.log({
+                            category: "action",
+                            message: `Clicked element, ${newOpenedTab ? "opened a new tab" : "no new tabs opened"}`,
+                            level: 1,
+                        });
+                        if (newOpenedTab) {
+                            this.log({
+                                category: "action",
+                                message: `New page detected (new tab) with URL: ${newOpenedTab.url()}`,
+                                level: 1,
+                            });
+                            yield newOpenedTab.close();
+                            yield this.page.goto(newOpenedTab.url());
+                            yield this.page.waitForLoadState("domcontentloaded");
+                            yield this._waitForSettledDom();
+                        }
+                        // Wait for the network to be idle with timeout of 5s (will only wait if loading a new page)
+                        // await this.waitForSettledDom();
+                        yield Promise.race([
+                            this.page.waitForLoadState("networkidle"),
+                            new Promise((resolve) => setTimeout(resolve, 5000)),
+                        ]).catch((e) => {
+                            this.log({
+                                category: "action",
+                                message: `Network idle timeout hit`,
+                                level: 1,
+                            });
+                        });
+                        this.log({
+                            category: "action",
+                            message: `Finished waiting for (possible) page navigation`,
+                            level: 1,
+                        });
+                        if (this.page.url() !== initialUrl) {
+                            this.log({
+                                category: "action",
+                                message: `New page detected with URL: ${this.page.url()}`,
+                                level: 1,
+                            });
+                        }
+                    }
+                }
+                else {
+                    this.log({
+                        category: "action",
+                        message: `Chosen method ${method} is invalid`,
+                        level: 1,
+                    });
+                    if (retries < 2) {
+                        return this._act({
+                            action,
+                            steps,
+                            modelName: model,
+                            useVision,
+                            verifierUseVision,
+                            retries: retries + 1,
+                            chunksSeen,
+                        });
+                    }
+                    else {
+                        return {
+                            success: false,
+                            message: `Internal error: Chosen method ${method} is invalid`,
+                            action: action,
+                        };
+                    }
+                }
+                let newSteps = steps +
+                    (!steps.endsWith("\n") ? "\n" : "") +
+                    `## Step: ${response.step}\n` +
+                    `  Element: ${elementText}\n` +
+                    `  Action: ${response.method}\n` +
+                    `  Reasoning: ${response.why}\n`;
+                if (urlChangeString) {
+                    newSteps += `  Result (Important): ${urlChangeString}\n\n`;
+                }
+                let actionComplete = false;
+                if (response.completed) {
+                    // Run action completion verifier
+                    this.log({
+                        category: "action",
+                        message: `Action marked as completed, Verifying if this is true...`,
+                        level: 1,
+                    });
+                    let domElements = undefined;
+                    let fullpageScreenshot = undefined;
+                    if (verifierUseVision) {
+                        try {
+                            const screenshotService = new vision_1.ScreenshotService(this.page, selectorMap, this.verbose);
+                            fullpageScreenshot = yield screenshotService.getScreenshot(true, 15);
+                        }
+                        catch (e) {
+                            this.log({
+                                category: "action",
+                                message: `Error getting full page screenshot: ${e.message}\n. Trying again...`,
+                                level: 1,
+                            });
+                            const screenshotService = new vision_1.ScreenshotService(this.page, selectorMap, this.verbose);
+                            fullpageScreenshot = yield screenshotService.getScreenshot(true, 15);
+                        }
+                    }
+                    else {
+                        ({ outputString: domElements } = yield this.page.evaluate(() => {
+                            return window.processAllOfDom();
+                        }));
+                    }
+                    actionComplete = yield (0, inference_1.verifyActCompletion)({
+                        goal: action,
+                        steps: newSteps,
+                        llmProvider: this.llmProvider,
+                        modelName: model,
+                        screenshot: fullpageScreenshot,
+                        domElements: domElements,
+                        logger: this.logger,
+                    });
+                    this.log({
+                        category: "action",
+                        message: `Action completion verification result: ${actionComplete}`,
+                        level: 1,
+                    });
+                }
+                if (!actionComplete) {
+                    this.log({
+                        category: "action",
+                        message: `Continuing to next action step`,
+                        level: 1,
+                    });
+                    return this._act({
+                        action,
+                        steps: newSteps,
+                        modelName,
+                        chunksSeen,
+                        useVision,
+                        verifierUseVision,
+                    });
+                }
+                else {
+                    this.log({
+                        category: "action",
+                        message: `Action completed successfully`,
+                        level: 1,
+                    });
+                    yield this._recordAction(action, response.step);
+                    return {
+                        success: true,
+                        message: `Action completed successfully: ${steps}${response.step}`,
+                        action: action,
+                    };
+                }
+            }
+            catch (error) {
+                this.log({
+                    category: "action",
+                    message: `Error performing action (Retries: ${retries}): ${error.message}\nTrace: ${error.stack}`,
+                    level: 1,
+                });
+                if (retries < 2) {
+                    return this._act({
+                        action,
+                        steps,
+                        modelName,
+                        useVision,
+                        verifierUseVision,
+                        retries: retries + 1,
+                        chunksSeen,
+                    });
+                }
+                yield this._recordAction(action, "");
+                return {
+                    success: false,
+                    message: `Error performing action: ${error.message}`,
+                    action: action,
+                };
+            }
+        });
+    }
+    act(_a) {
+        return __awaiter(this, arguments, void 0, function* ({ action, modelName, useVision = "fallback", }) {
+            useVision = useVision !== null && useVision !== void 0 ? useVision : "fallback";
+            return this._act({
+                action,
+                modelName,
+                chunksSeen: [],
+                useVision,
+                verifierUseVision: useVision !== false,
+            });
+        });
+    }
+    extract(_a) {
+        return __awaiter(this, arguments, void 0, function* ({ instruction, schema, modelName, }) {
+            return this._extract({
+                instruction,
+                schema,
+                modelName,
+            });
+        });
+    }
+    observe(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            return this._observe({
+                instruction: (_a = options === null || options === void 0 ? void 0 : options.instruction) !== null && _a !== void 0 ? _a : "Find actions that can be performed on this page.",
+                modelName: options === null || options === void 0 ? void 0 : options.modelName,
+                useVision: (_b = options === null || options === void 0 ? void 0 : options.useVision) !== null && _b !== void 0 ? _b : false,
+                fullPage: false,
+            });
+        });
+    }
+}
+exports.Stagehand = Stagehand;
+//# sourceMappingURL=index.js.map