@browserbasehq/stagehand 1.0.3 → 1.1.0-alpha.1

package/dist/evals/index.eval.js

@@ -0,0 +1,1075 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+const braintrust_1 = require("braintrust");
+const lib_1 = require("../lib");
+const zod_1 = require("zod");
+const process_1 = __importDefault(require("process"));
+const utils_1 = require("./utils");
+const env = ((_a = process_1.default.env.EVAL_ENV) === null || _a === void 0 ? void 0 : _a.toLowerCase()) === "browserbase"
+    ? "BROWSERBASE"
+    : "LOCAL";
+const expedia = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        headless: false,
+        verbose: 2,
+        debugDom: true,
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto("https://www.expedia.com/flights");
+        yield stagehand.act({
+            action: "find round-trip flights from San Francisco (SFO) to Toronto (YYZ) for Jan 1, 2025 (up to one to two weeks)",
+        });
+        yield stagehand.act({ action: "Go to the first non-stop flight" });
+        yield stagehand.act({ action: "select the cheapest flight" });
+        yield stagehand.act({ action: "click on the first non-stop flight" });
+        yield stagehand.act({
+            action: "Take me to the checkout page",
+        });
+        const url = yield stagehand.page.url();
+        return {
+            _success: url.startsWith("https://www.expedia.com/Checkout/"),
+            logs: logger.getLogs(),
+            debugUrl,
+            sessionUrl,
+        };
+    }
+    catch (error) {
+        logger.error(`Error in expedia function: ${JSON.stringify(error, null, 2)}. Trace: ${error.stack}`);
+        return {
+            _success: false,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close().catch(() => { });
+    }
+});
+const vanta = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message);
+        },
+        verbose: 2,
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    yield stagehand.page.goto("https://www.vanta.com/");
+    const observations = yield stagehand.observe();
+    if (observations.length === 0) {
+        yield stagehand.context.close();
+        return {
+            _success: false,
+            observations,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    const expectedLocator = `body > div.page-wrapper > div.nav_component > div.nav_element.w-nav > div.padding-global > div > div > nav > div.nav_cta-wrapper.is-new > a.nav_cta-button-desktop.is-smaller.w-button`;
+    const expectedResult = yield stagehand.page
+        .locator(expectedLocator)
+        .first()
+        .innerHTML();
+    let foundMatch = false;
+    for (const observation of observations) {
+        try {
+            const observationResult = yield stagehand.page
+                .locator(observation.selector)
+                .first()
+                .innerHTML();
+            if (observationResult === expectedResult) {
+                foundMatch = true;
+                break;
+            }
+        }
+        catch (error) {
+            console.warn(`Failed to check observation with selector ${observation.selector}:`, error.message);
+            continue;
+        }
+    }
+    yield stagehand.context.close();
+    return {
+        _success: foundMatch,
+        expected: expectedResult,
+        observations,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+    };
+});
+const vanta_h = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+        verbose: 2,
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    yield stagehand.page.goto("https://www.vanta.com/");
+    const observations = yield stagehand.observe({
+        instruction: "find the buy now button",
+    });
+    yield stagehand.context.close();
+    // we should have no saved observation since the element shouldn't exist
+    return {
+        _success: observations.length === 0,
+        observations,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+    };
+});
+const simple_google_search = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+        verbose: 2,
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    yield stagehand.page.goto("https://www.google.com");
+    yield stagehand.act({
+        action: 'Search for "OpenAI"',
+    });
+    const expectedUrl = "https://www.google.com/search?q=OpenAI";
+    const currentUrl = yield stagehand.page.url();
+    yield stagehand.context.close();
+    return {
+        _success: currentUrl.startsWith(expectedUrl),
+        currentUrl,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+    };
+});
+const peeler_simple = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env: "LOCAL",
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+        verbose: 2,
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    yield stagehand.page.goto(`file://${process_1.default.cwd()}/evals/assets/peeler.html`);
+    yield stagehand.act({ action: "add the peeler to cart" });
+    const successMessageLocator = stagehand.page.locator('text="Congratulations, you have 1 A in your cart"');
+    const isVisible = yield successMessageLocator.isVisible();
+    yield stagehand.context.close();
+    return {
+        _success: isVisible,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+    };
+});
+const peeler_complex = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto(`https://chefstoys.com/`, { timeout: 60000 });
+        yield stagehand.act({
+            action: "search for peelers",
+        });
+        yield stagehand.act({
+            action: 'click on the first "OXO" brand peeler',
+        });
+        const { price } = yield stagehand.extract({
+            instruction: "get the price of the peeler",
+            schema: zod_1.z.object({ price: zod_1.z.number().nullable() }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        return {
+            _success: price === 11.99,
+            price,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        const errorMessage = JSON.parse(JSON.stringify(error, null, 2));
+        const errorStack = errorMessage.stack;
+        const fullError = `Error in peeler_complex function: ${errorMessage.message} Trace: ${errorStack}`;
+        logger.error(fullError);
+        return {
+            _success: false,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close();
+    }
+});
+const homedepot = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto("https://www.homedepot.com/");
+        yield stagehand.act({ action: "search for gas grills" });
+        yield stagehand.act({ action: "click on the best selling gas grill" });
+        yield stagehand.act({ action: "click on the Product Details" });
+        yield stagehand.act({ action: "find the Primary Burner BTU" });
+        const productSpecs = yield stagehand.extract({
+            instruction: "Extract the Primary exact Burner BTU of the product",
+            schema: zod_1.z.object({
+                productSpecs: zod_1.z
+                    .array(zod_1.z.object({
+                    burnerBTU: zod_1.z.string().describe("Primary Burner BTU exact value"),
+                }))
+                    .describe("Gas grill Primary Burner BTU exact value"),
+            }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        logger.log(`The gas grill primary burner BTU is: ${productSpecs}`);
+        if (!productSpecs ||
+            !productSpecs.productSpecs ||
+            productSpecs.productSpecs.length !== 1) {
+            return {
+                _success: false,
+                productSpecs,
+                debugUrl,
+                sessionUrl,
+                logs: logger.getLogs(),
+            };
+        }
+        if ((productSpecs.productSpecs[0].burnerBTU.match(/0/g) || []).length == 4 &&
+            (productSpecs.productSpecs[0].burnerBTU.match(/4/g) || []).length === 1) {
+            return {
+                _success: true,
+                productSpecs,
+                debugUrl,
+                sessionUrl,
+                logs: logger.getLogs(),
+            };
+        }
+        else {
+            return {
+                _success: false,
+                productSpecs,
+                debugUrl,
+                sessionUrl,
+                logs: logger.getLogs(),
+            };
+        }
+    }
+    catch (error) {
+        logger.error(`Error in homedepot function: ${JSON.stringify(error, null, 2)}, Trace: ${error.stack}`);
+        return {
+            _success: false,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close().catch(() => { });
+    }
+});
+const extract_github_stars = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto("https://github.com/facebook/react");
+        const { stars } = yield stagehand.extract({
+            instruction: "Extract the number of stars for the project",
+            schema: zod_1.z.object({
+                stars: zod_1.z.number().describe("the number of stars for the project"),
+            }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        const expectedStarsString = yield stagehand.page
+            .locator("#repo-stars-counter-star")
+            .first()
+            .innerHTML();
+        const expectedStars = expectedStarsString.toLowerCase().endsWith('k')
+            ? parseFloat(expectedStarsString.slice(0, -1)) * 1000
+            : parseFloat(expectedStarsString);
+        yield stagehand.context.close().catch(() => { });
+        return {
+            _success: stars === expectedStars,
+            stars,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        console.error("Error or timeout occurred:", error);
+        yield stagehand.context.close().catch(() => { });
+        return {
+            _success: false,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+});
+const extract_collaborators_from_github_repository = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto("https://github.com/facebook/react");
+        yield stagehand.act({
+            action: "find the contributors section",
+        });
+        const { contributors } = yield stagehand.extract({
+            instruction: "Extract top 20 contributors of this repository",
+            schema: zod_1.z.object({
+                contributors: zod_1.z.array(zod_1.z.object({
+                    github_username: zod_1.z
+                        .string()
+                        .describe("the github username of the contributor"),
+                    information: zod_1.z.string().describe("number of commits contributed"),
+                })),
+            }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        console.log("Extracted collaborators:", contributors);
+        yield stagehand.context.close().catch(() => { });
+        return {
+            _success: contributors.length === 20,
+            contributors,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        console.error("Error or timeout occurred:", error);
+        yield stagehand.context.close().catch(() => { });
+        return {
+            _success: false,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+});
+const extract_last_twenty_github_commits = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto("https://github.com/facebook/react");
+        yield stagehand.act({
+            action: "find commit history, generally described by the number of commits",
+        });
+        const { commits } = yield stagehand.extract({
+            instruction: "Extract last 20 commits",
+            schema: zod_1.z.object({
+                commits: zod_1.z.array(zod_1.z.object({
+                    commit_message: zod_1.z.string(),
+                    commit_url: zod_1.z.string(),
+                    commit_hash: zod_1.z.string(),
+                })),
+            }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        logger.log(`Extracted commits: ${commits}`);
+        yield stagehand.context.close().catch(() => { });
+        return {
+            _success: commits.length === 20,
+            commits,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        console.error("Error or timeout occurred:", error);
+        yield stagehand.context.close().catch(() => { });
+        return {
+            _success: false,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+});
+const wikipedia = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    yield stagehand.page.goto(`https://en.wikipedia.org/wiki/Baseball`);
+    yield stagehand.act({
+        action: 'click the "hit and run" link in this article',
+    });
+    const url = "https://en.wikipedia.org/wiki/Hit_and_run_(baseball)";
+    const currentUrl = yield stagehand.page.url();
+    yield stagehand.context.close().catch(() => { });
+    return {
+        _success: currentUrl === url,
+        expected: url,
+        actual: currentUrl,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+    };
+});
+// Validate that the action is not found on the page
+const nonsense_action = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env: "LOCAL",
+        verbose: 2,
+        debugDom: true,
+        headless: true,
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto("https://www.homedepot.com/");
+        const result = yield stagehand.act({
+            action: "click on the first banana",
+        });
+        console.log("result", result);
+        // Assert the output
+        const expectedResult = {
+            success: false,
+            message: "Action not found on the current page after checking all chunks.",
+            action: "click on the first banana",
+        };
+        const isResultCorrect = JSON.stringify(result) === JSON.stringify(expectedResult);
+        return {
+            _success: isResultCorrect,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        console.error(`Error in nonsense_action function: ${error.message}`);
+        return {
+            _success: false,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close();
+    }
+});
+const costar = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        debugDom: true,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    // TODO: fix this eval - does not work in headless mode
+    try {
+        yield Promise.race([
+            stagehand.page.goto("https://www.costar.com/"),
+            new Promise((_, reject) => setTimeout(() => reject(new Error("Navigation timeout")), 30000)),
+        ]);
+        yield stagehand.act({ action: "click on the first article" });
+        yield stagehand.act({
+            action: "click on the learn more button for the first job",
+        });
+        const articleTitle = yield stagehand.extract({
+            instruction: "extract the title of the article",
+            schema: zod_1.z.object({
+                title: zod_1.z.string().describe("the title of the article").nullable(),
+            }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        logger.log(`articleTitle: ${articleTitle}`);
+        // Check if the title is more than 5 characters
+        const isTitleValid = articleTitle.title !== null && articleTitle.title.length > 5;
+        yield stagehand.context.close();
+        return {
+            title: articleTitle.title,
+            _success: isTitleValid,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        logger.error(`Error in costar function: ${error.message}`);
+        return {
+            title: null,
+            _success: false,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close();
+    }
+});
+const google_jobs = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        debugDom: true,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init();
+    try {
+        yield stagehand.page.goto("https://www.google.com/");
+        yield stagehand.act({ action: "click on the about page" });
+        yield stagehand.act({ action: "click on the careers page" });
+        yield stagehand.act({ action: "input data scientist into role" });
+        yield stagehand.act({ action: "input new york city into location" });
+        yield stagehand.act({ action: "click on the search button" });
+        // NOTE: "click on the first Learn More button" is not working - the span for learn more is not clickable and the a href is after it
+        yield stagehand.act({ action: "click on the first job link" });
+        const jobDetails = yield stagehand.extract({
+            instruction: "Extract the following details from the job posting: application deadline, minimum qualifications (degree and years of experience), and preferred qualifications (degree and years of experience)",
+            schema: zod_1.z.object({
+                applicationDeadline: zod_1.z
+                    .string()
+                    .describe("The date until which the application window will be open")
+                    .nullable(),
+                minimumQualifications: zod_1.z.object({
+                    degree: zod_1.z.string().describe("The minimum required degree").nullable(),
+                    yearsOfExperience: zod_1.z
+                        .number()
+                        .describe("The minimum required years of experience")
+                        .nullable(),
+                }),
+                preferredQualifications: zod_1.z.object({
+                    degree: zod_1.z.string().describe("The preferred degree").nullable(),
+                    yearsOfExperience: zod_1.z
+                        .number()
+                        .describe("The preferred years of experience")
+                        .nullable(),
+                }),
+            }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        logger.log(`Job Details: ${jobDetails}`);
+        const isJobDetailsValid = jobDetails &&
+            Object.values(jobDetails).every((value) => value !== null &&
+                value !== undefined &&
+                (typeof value !== "object" ||
+                    Object.values(value).every((v) => v !== null &&
+                        v !== undefined &&
+                        (typeof v === "number" || typeof v === "string"))));
+        logger.log(`Job Details valid: ${isJobDetailsValid}`);
+        return {
+            _success: isJobDetailsValid,
+            jobDetails,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        logger.error(`Error in google_jobs function: ${error.message}. Trace: ${error.stack}`);
+        return {
+            _success: false,
+            debugUrl,
+            sessionUrl,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close();
+    }
+});
+const extract_partners = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        debugDom: true,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init({
+        modelName: "gpt-4o",
+    });
+    try {
+        yield stagehand.page.goto("https://ramp.com");
+        yield stagehand.act({
+            action: "Close the popup.",
+        });
+        yield stagehand.act({
+            action: "Scroll down to the bottom of the page.",
+        });
+        yield stagehand.act({
+            action: "Click on the link or button that leads to the partners page. If it's in a dropdown or hidden section, first interact with the element to reveal it, then click the link.",
+        });
+        const partners = yield stagehand.extract({
+            instruction: `
+      Extract the names of all partner companies mentioned on this page.
+      These could be inside text, links, or images representing partner companies.
+      If no specific partner names are found, look for any sections or categories of partners mentioned.
+      Also, check for any text that explains why partner names might not be listed, if applicable.
+    `,
+            schema: zod_1.z.object({
+                partners: zod_1.z.array(zod_1.z.object({
+                    name: zod_1.z
+                        .string()
+                        .describe("The name of the partner company or category of partners"),
+                })),
+                explanation: zod_1.z
+                    .string()
+                    .optional()
+                    .describe("Any explanation about partner listing or absence thereof"),
+            }),
+        });
+        const expectedPartners = [
+            "Accounting Partners",
+            "Private Equity & Venture Capital Partners",
+            "Services Partners",
+            "Affiliates",
+        ];
+        if (partners.explanation) {
+            logger.log(`Explanation: ${partners.explanation}`);
+        }
+        const foundPartners = partners.partners.map((partner) => partner.name.toLowerCase());
+        const allExpectedPartnersFound = expectedPartners.every((partner) => foundPartners.includes(partner.toLowerCase()));
+        logger.log(`All expected partners found: ${allExpectedPartnersFound}`);
+        logger.log(`Expected: ${expectedPartners}`);
+        logger.log(`Found: ${foundPartners}`);
+        return {
+            _success: allExpectedPartnersFound,
+            partners,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    catch (error) {
+        logger.error(`Error in extractPartners function: ${error.message}. Trace: ${error.stack}`);
+        return {
+            _success: false,
+            debugUrl,
+            sessionUrl,
+            error: JSON.parse(JSON.stringify(error, null, 2)),
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close().catch(() => { });
+    }
+});
+const laroche_form = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        debugDom: true,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init({
+        modelName: "gpt-4o",
+    });
+    try {
+        yield stagehand.page.goto("https://www.laroche-posay.us/offers/anthelios-melt-in-milk-sunscreen-sample.html");
+        yield stagehand.act({ action: "close the privacy policy popup" });
+        // Wait for possible navigation
+        yield stagehand.page
+            .waitForNavigation({ waitUntil: "domcontentloaded", timeout: 10000 })
+            .catch(() => { });
+        yield stagehand.act({ action: "fill the last name field" });
+        yield stagehand.act({ action: "fill address 1 field" });
+        yield stagehand.act({ action: "select a state" });
+        yield stagehand.act({ action: "select a skin type" });
+        // TODO - finish this eval once we have a way to extract form data from children iframes
+        // const formData = await stagehand.extract({
+        //   instruction: "Extract the filled form data",
+        //   schema: z.object({
+        //     firstName: z.string(),
+        //     lastName: z.string(),
+        //     email: z.string(),
+        //     phone: z.string(),
+        //     zipCode: z.string(),
+        //     interestedIn: z.string(),
+        //     startTerm: z.string(),
+        //     programOfInterest: z.string(),
+        //   }),
+        //   modelName: "gpt-4o",
+        // });
+        // console.log("Extracted form data:", formData);
+        // const isFormDataValid =
+        //   formData.firstName === "John" &&
+        //   formData.lastName === "Doe" &&
+        //   formData.email === "john.doe@example.com" &&
+        //   formData.phone === "1234567890" &&
+        //   formData.zipCode === "12345" &&
+        return {
+            _success: true,
+            logs: logger.getLogs(),
+            debugUrl,
+            sessionUrl,
+        };
+    }
+    catch (error) {
+        logger.error(`Error in LarocheForm function: ${error.message}. Trace: ${error.stack}`);
+        return {
+            _success: false,
+            error: error.message,
+            debugUrl,
+            sessionUrl,
+            logs: logger.getLogs(),
+        };
+    }
+    finally {
+        yield stagehand.context.close().catch(() => { });
+    }
+});
+const arxiv = () => __awaiter(void 0, void 0, void 0, function* () {
+    const logger = new utils_1.EvalLogger();
+    const stagehand = new lib_1.Stagehand({
+        env,
+        verbose: 2,
+        debugDom: true,
+        headless: process_1.default.env.HEADLESS !== "false",
+        logger: (message) => {
+            logger.log(message.message);
+        },
+    });
+    logger.init(stagehand);
+    const { debugUrl, sessionUrl } = yield stagehand.init({
+        modelName: "gpt-4o-2024-08-06",
+    });
+    const papers = [];
+    try {
+        yield stagehand.page.goto("https://arxiv.org/search/");
+        yield stagehand.act({
+            action: "search for the recent papers about web agents with multimodal models",
+        });
+        const paper_links = yield stagehand.extract({
+            instruction: "extract the titles and links for two papers",
+            schema: zod_1.z.object({
+                papers: zod_1.z
+                    .array(zod_1.z.object({
+                    title: zod_1.z.string().describe("the title of the paper"),
+                    link: zod_1.z.string().describe("the link to the paper").nullable(),
+                }))
+                    .describe("list of papers"),
+            }),
+            modelName: "gpt-4o-2024-08-06",
+        });
+        if (!paper_links ||
+            !paper_links.papers ||
+            paper_links.papers.length === 0) {
+            return {
+                _success: false,
+                logs: logger.getLogs(),
+                debugUrl,
+                sessionUrl,
+            };
+        }
+        for (const paper of paper_links.papers) {
+            if (paper.link) {
+                yield stagehand.page.goto(paper.link);
+                const abstract = yield stagehand.extract({
+                    instruction: "extract details of the paper from the abstract",
+                    schema: zod_1.z.object({
+                        category: zod_1.z
+                            .string()
+                            .describe("the category of the paper. one of {'Benchmark', 'Dataset', 'Model', 'Framework', 'System', 'Other'}"),
+                        problem: zod_1.z
+                            .string()
+                            .describe("summarize the problem that the paper is trying to solve in one sentence")
+                            .nullable(),
+                        methodology: zod_1.z
+                            .string()
+                            .describe("summarize the methodology of the paper in one sentence")
+                            .nullable(),
+                        results: zod_1.z
+                            .string()
+                            .describe("summarize the results of the paper in one sentence")
+                            .nullable(),
+                        conclusion: zod_1.z
+                            .string()
+                            .describe("summarize the conclusion of the paper in one sentence")
+                            .nullable(),
+                        code: zod_1.z
+                            .string()
+                            .describe("if provided, extract only the link to the code repository, without additional text. this is often optional and not always provided.")
+                            .nullable(),
+                    }),
+                    modelName: "gpt-4o-2024-08-06",
+                });
+                papers.push({
+                    title: paper.title,
+                    link: paper.link,
+                    category: abstract.category,
+                    problem: abstract.problem,
+                    methodology: abstract.methodology,
+                    results: abstract.results,
+                    conclusion: abstract.conclusion,
+                    code: abstract.code,
+                });
+            }
+        }
+        if (!papers || papers.length === 0) {
+            return {
+                _success: false,
+                logs: logger.getLogs(),
+                debugUrl,
+                sessionUrl,
+            };
+        }
+        logger.log(JSON.stringify(papers, null, 2));
+        // Assert that the length of papers is three
+        if (papers.length !== 2) {
+            logger.log(`Expected 2 papers, but got ${papers.length}`);
+            return {
+                _success: false,
+                error: "Incorrect number of papers extracted",
+                logs: logger.getLogs(),
+                debugUrl,
+                sessionUrl,
+            };
+        }
+        // Ensure that every paper has a problem and methodology
+        for (const paper of papers) {
+            if (!paper.problem || !paper.methodology) {
+                logger.log(`Paper "${paper.title}" is missing problem or methodology`);
+                return {
+                    _success: false,
+                    error: "Incomplete paper information",
+                    logs: logger.getLogs(),
+                    debugUrl,
+                    sessionUrl,
+                };
+            }
+        }
+        return {
+            _success: true,
+            papers,
+            logs: logger.getLogs(),
+            debugUrl,
+            sessionUrl,
+        };
+    }
+    catch (error) {
+        logger.error(`Error in arxiv function: ${error.message}. Trace: ${error.stack}`);
+        return {
+            _success: false,
+            logs: logger.getLogs(),
+            debugUrl,
+            sessionUrl,
+        };
+    }
+    finally {
+        yield stagehand.context.close().catch(() => { });
+    }
+});
+const tasks = {
+    vanta,
+    vanta_h,
+    peeler_simple,
+    peeler_complex,
+    wikipedia,
+    simple_google_search,
+    extract_github_stars,
+    extract_collaborators_from_github_repository,
+    extract_last_twenty_github_commits,
+    costar,
+    google_jobs,
+    homedepot,
+    extract_partners,
+    laroche_form,
+    arxiv,
+    expedia,
+};
+const exactMatch = (args) => {
+    var _a, _b;
+    console.log(`Task "${args.input.name}" returned: ${args.output}`);
+    const expected = (_a = args.expected) !== null && _a !== void 0 ? _a : true;
+    if (expected === true) {
+        return {
+            name: "Exact match",
+            score: args.output === true || ((_b = args.output) === null || _b === void 0 ? void 0 : _b._success) == true,
+        };
+    }
+    return {
+        name: "Exact match",
+        score: args.output === expected,
+    };
+};
+const testcases = [
+    {
+        input: {
+            name: "vanta",
+        },
+    },
+    {
+        input: {
+            name: "vanta_h",
+        },
+    },
+    {
+        input: {
+            name: "peeler_simple",
+        },
+    },
+    {
+        input: { name: "wikipedia" },
+    },
+    { input: { name: "peeler_complex" } },
+    { input: { name: "simple_google_search" } },
+    { input: { name: "extract_github_stars" } },
+    {
+        input: {
+            name: "extract_collaborators_from_github_repository",
+        },
+    },
+    { input: { name: "extract_last_twenty_github_commits" } },
+    { input: { name: "google_jobs" } },
+    { input: { name: "homedepot" } },
+    { input: { name: "extract_partners" } },
+    { input: { name: "laroche_form" } },
+    { input: { name: "arxiv" } },
+    // { input: { name: "expedia" } },
+];
+(0, braintrust_1.Eval)("stagehand", {
+    data: () => {
+        return testcases;
+    },
+    task: (input) => __awaiter(void 0, void 0, void 0, function* () {
+        // console.log("input", input);
+        try {
+            // Handle predefined tasks
+            const result = yield tasks[input.name](input);
+            if (result) {
+                console.log(`✅ ${input.name}: Passed`);
+            }
+            else {
+                console.log(`❌ ${input.name}: Failed`);
+            }
+            return result;
+        }
+        catch (error) {
+            console.error(`❌ ${input.name}: Error - ${error}`);
+            return {
+                _success: false,
+                error: JSON.parse(JSON.stringify(error, null, 2)),
+            };
+        }
+    }),
+    scores: [exactMatch],
+    maxConcurrency: 5,
+    // trialCount: 3,
+});
+//# sourceMappingURL=index.eval.js.map