@browserbasehq/orca 3.4.0-preview-5 → 3.5.0-preview.0

package/dist/esm/lib/v3Evaluator.d.ts

@@ -1,19 +1,27 @@
-/**
- * V3Evaluator mirrors Evaluator but operates on a V3 instance instead of Stagehand.
- * It uses the V3 page/screenshot APIs and constructs an LLM client to run
- * structured evaluations (YES/NO with reasoning) on screenshots and/or text.
- */
 import type { AvailableModel, ClientOptions } from "./v3/types/public/model.js";
 import type { EvaluateOptions, BatchAskOptions, EvaluationResult } from "./v3/types/private/evaluator.js";
 import { V3 } from "./v3/v3.js";
+export type V3EvaluatorBackend = "legacy" | "verifier";
+export type V3EvaluatorOptions = {
+    /**
+     * Selects the evaluator implementation.
+     *
+     * "legacy" preserves the existing screenshot/text YES/NO evaluator.
+     * "verifier" is reserved for the rubric verifier backend.
+     *
+     * @default process.env.STAGEHAND_EVALUATOR_BACKEND || "legacy"
+     */
+    backend?: V3EvaluatorBackend;
+};
+export type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {
+    modelName?: AvailableModel;
+    modelClientOptions?: ClientOptions;
+};
 export declare class V3Evaluator {
-    private v3;
-    private modelName;
-    private modelClientOptions;
-    private silentLogger;
-    constructor(v3: V3, modelName?: AvailableModel, modelClientOptions?: ClientOptions);
-    private getClient;
+    private readonly backend;
+    private readonly legacyEvaluator;
+    constructor(v3: V3, modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions, modelClientOptions?: ClientOptions, options?: V3EvaluatorOptions);
     ask(options: EvaluateOptions): Promise<EvaluationResult>;
     batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]>;
-    private _evaluateWithMultipleScreenshots;
+    private getLegacyBackend;
 }

package/dist/esm/lib/v3Evaluator.js

@@ -1,210 +1,52 @@
-/**
- * V3Evaluator mirrors Evaluator but operates on a V3 instance instead of Stagehand.
- * It uses the V3 page/screenshot APIs and constructs an LLM client to run
- * structured evaluations (YES/NO with reasoning) on screenshots and/or text.
- */
-import { z } from "zod";
-import { LLMProvider } from "./v3/llm/LLMProvider.js";
 import { StagehandInvalidArgumentError } from "./v3/types/public/sdkErrors.js";
-const EvaluationSchema = z.object({
-    evaluation: z.enum(["YES", "NO"]),
-    reasoning: z.string(),
-});
-const BatchEvaluationSchema = z.array(EvaluationSchema);
+import { LegacyV3Evaluator } from "./v3LegacyEvaluator.js";
+const EVALUATOR_BACKEND_ENV = "STAGEHAND_EVALUATOR_BACKEND";
+const DEFAULT_EVALUATOR_BACKEND = "legacy";
 export class V3Evaluator {
-    v3;
-    modelName;
-    modelClientOptions;
-    silentLogger = () => { };
-    constructor(v3, modelName, modelClientOptions) {
-        this.v3 = v3;
-        this.modelName = modelName || "google/gemini-2.5-flash";
-        this.modelClientOptions = modelClientOptions || {
-            apiKey: process.env.GEMINI_API_KEY ||
-                process.env.GOOGLE_GENERATIVE_AI_API_KEY ||
-                "",
-        };
-    }
-    getClient() {
-        // Prefer a dedicated provider so we can override model per-evaluation
-        const provider = new LLMProvider(this.v3.logger);
-        return provider.getClient(this.modelName, this.modelClientOptions);
+    backend;
+    legacyEvaluator;
+    constructor(v3, modelNameOrOptions, modelClientOptions, options) {
+        const normalizedOptions = normalizeConstructorOptions(modelNameOrOptions, modelClientOptions, options);
+        this.backend = resolveEvaluatorBackend(normalizedOptions.backend);
+        this.legacyEvaluator = new LegacyV3Evaluator(v3, normalizedOptions.modelName, normalizedOptions.modelClientOptions);
     }
     async ask(options) {
-        const { question, answer, screenshot = true, systemPrompt, screenshotDelayMs = 250, agentReasoning, } = options;
-        if (!question)
-            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
-        if (!answer && !screenshot)
-            throw new StagehandInvalidArgumentError("Either answer (text) or screenshot must be provided");
-        if (Array.isArray(screenshot)) {
-            return this._evaluateWithMultipleScreenshots({
-                question,
-                screenshots: screenshot,
-                systemPrompt,
-                agentReasoning,
-            });
-        }
-        const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? "a screenshot" : "the agents reasoning and actions throughout the task"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\n          Today's date is ${new Date().toLocaleDateString()}`;
-        await new Promise((r) => setTimeout(r, screenshotDelayMs));
-        let imageBuffer;
-        if (screenshot) {
-            const page = await this.v3.context.awaitActivePage();
-            imageBuffer = await page.screenshot({ fullPage: false });
-        }
-        const llmClient = this.getClient();
-        const response = await llmClient.createChatCompletion({
-            logger: this.silentLogger,
-            options: {
-                messages: [
-                    { role: "system", content: systemPrompt || defaultSystemPrompt },
-                    {
-                        role: "user",
-                        content: [
-                            {
-                                type: "text",
-                                text: agentReasoning
-                                    ? `Question: ${question}\n\nAgent's reasoning and actions taken:\n${agentReasoning}`
-                                    : question,
-                            },
-                            ...(screenshot && imageBuffer
-                                ? [
-                                    {
-                                        type: "image_url",
-                                        image_url: {
-                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
-                                        },
-                                    },
-                                ]
-                                : []),
-                            ...(answer
-                                ? [{ type: "text", text: `the answer is ${answer}` }]
-                                : []),
-                        ],
-                    },
-                ],
-                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
-            },
-        });
-        try {
-            const result = response.data;
-            return { evaluation: result.evaluation, reasoning: result.reasoning };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                evaluation: "INVALID",
-                reasoning: `Failed to get structured response: ${errorMessage}`,
-            };
-        }
+        return this.getLegacyBackend("ask").ask(options);
     }
     async batchAsk(options) {
-        const { questions, screenshot = true, systemPrompt = "You are an expert evaluator that returns YES or NO with a concise reasoning.", screenshotDelayMs = 250, } = options;
-        if (!questions?.length)
-            throw new StagehandInvalidArgumentError("Questions array cannot be empty");
-        await new Promise((r) => setTimeout(r, screenshotDelayMs));
-        let imageBuffer;
-        if (screenshot) {
-            const page = await this.v3.context.awaitActivePage();
-            imageBuffer = await page.screenshot({ fullPage: false });
-        }
-        const llmClient = this.getClient();
-        const formatted = questions
-            .map((item, i) => `${i + 1}. ${item.question}${item.answer ? `\n   Answer: ${item.answer}` : ""}`)
-            .join("\n\n");
-        const response = await llmClient.createChatCompletion({
-            logger: this.silentLogger,
-            options: {
-                messages: [
-                    {
-                        role: "system",
-                        content: `${systemPrompt}\n\nYou will be given multiple questions${screenshot ? " with a screenshot" : ""}. ${questions.some((q) => q.answer) ? "Some questions include answers to evaluate." : ""} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,
-                    },
-                    {
-                        role: "user",
-                        content: [
-                            { type: "text", text: formatted },
-                            ...(screenshot && imageBuffer
-                                ? [
-                                    {
-                                        type: "image_url",
-                                        image_url: {
-                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
-                                        },
-                                    },
-                                ]
-                                : []),
-                        ],
-                    },
-                ],
-                response_model: {
-                    name: "BatchEvaluationResult",
-                    schema: BatchEvaluationSchema,
-                },
-            },
-        });
-        try {
-            const results = response.data;
-            return results.map((r) => ({
-                evaluation: r.evaluation,
-                reasoning: r.reasoning,
-            }));
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return questions.map(() => ({
-                evaluation: "INVALID",
-                reasoning: `Failed to get structured response: ${errorMessage}`,
-            }));
-        }
+        return this.getLegacyBackend("batchAsk").batchAsk(options);
     }
-    async _evaluateWithMultipleScreenshots(options) {
-        const { question, screenshots, agentReasoning, systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.
-        ${agentReasoning ? "You also have access to the agent's detailed reasoning and thought process throughout the task." : ""}
-        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.
-        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).
-        ${agentReasoning ? "The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation." : ""}
-        Today's date is ${new Date().toLocaleDateString()}`, } = options;
-        if (!question)
-            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
-        if (!screenshots || screenshots.length === 0)
-            throw new StagehandInvalidArgumentError("At least one screenshot must be provided");
-        const llmClient = this.getClient();
-        const imageContents = screenshots.map((s) => ({
-            type: "image_url",
-            image_url: { url: `data:image/jpeg;base64,${s.toString("base64")}` },
-        }));
-        const response = await llmClient.createChatCompletion({
-            logger: this.silentLogger,
-            options: {
-                messages: [
-                    { role: "system", content: systemPrompt },
-                    {
-                        role: "user",
-                        content: [
-                            {
-                                type: "text",
-                                text: agentReasoning
-                                    ? `Question: ${question}\n\nAgent's reasoning and actions throughout the task:\n${agentReasoning}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`
-                                    : `${question}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,
-                            },
-                            ...imageContents,
-                        ],
-                    },
-                ],
-                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
-            },
-        });
-        try {
-            const result = response.data;
-            return { evaluation: result.evaluation, reasoning: result.reasoning };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                evaluation: "INVALID",
-                reasoning: `Failed to get structured response: ${errorMessage}`,
-            };
+    getLegacyBackend(methodName) {
+        if (this.backend === "legacy") {
+            return this.legacyEvaluator;
         }
+        throw new StagehandInvalidArgumentError(`V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use "legacy" or install the verifier backend PR.`);
+    }
+}
+function normalizeConstructorOptions(modelNameOrOptions, modelClientOptions, options) {
+    if (modelNameOrOptions &&
+        typeof modelNameOrOptions === "object" &&
+        !Array.isArray(modelNameOrOptions)) {
+        return {
+            modelName: modelNameOrOptions.modelName,
+            modelClientOptions: modelNameOrOptions.modelClientOptions,
+            backend: modelNameOrOptions.backend ?? options?.backend,
+        };
+    }
+    return {
+        modelName: modelNameOrOptions,
+        modelClientOptions,
+        backend: options?.backend,
+    };
+}
+function resolveEvaluatorBackend(explicitBackend) {
+    const configuredBackend = explicitBackend ??
+        process.env[EVALUATOR_BACKEND_ENV] ??
+        DEFAULT_EVALUATOR_BACKEND;
+    const normalizedBackend = configuredBackend.trim().toLowerCase();
+    if (normalizedBackend === "legacy" || normalizedBackend === "verifier") {
+        return normalizedBackend;
     }
+    throw new StagehandInvalidArgumentError(`Invalid ${EVALUATOR_BACKEND_ENV}="${configuredBackend}". Expected "legacy" or "verifier".`);
 }
 //# sourceMappingURL=v3Evaluator.js.map

package/dist/esm/lib/v3Evaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAExD,MAAM,OAAO,WAAW;IACd,EAAE,CAAK;IACP,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,YAAY,GAA+B,GAAG,EAAE,GAAE,CAAC,CAAC;IAE5D,YACE,EAAM,EACN,SAA0B,EAC1B,kBAAkC;QAElC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,IAAK,yBAA4C,CAAC;QAC5E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI;YAC9C,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,sEAAsE;QACtE,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,UAAU,GAAG,IAAI,EACjB,YAAY,EACZ,iBAAiB,GAAG,GAAG,EACvB,cAAc,GACf,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;YACxB,MAAM,IAAI,6BAA6B,CACrC,qDAAqD,CACtD,CAAC;QAEJ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,gCAAgC,CAAC;gBAC3C,QAAQ;gBACR,WAAW,EAAE,UAAU;gBACvB,YAAY;gBACZ,cAAc;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,mBAAmB,GAAG,kIAAkI,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,sDAAsD,8HAA8H,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC;QAElZ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE;oBAChE;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,6CAA6C,cAAc,EAAE;oCACpF,CAAC,CAAC,QAAQ;6BACb;4BACD,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,EACJ,SAAS,EACT,UAAU,GAAG,IAAI,EACjB,YAAY,GAAG,8EAA8E,EAC7F,iBAAiB,GAAG,GAAG,GACxB,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,SAAS,EAAE,MAAM;YACpB,MAAM,IAAI,6BAA6B,CACrC,iCAAiC,CAClC,CAAC;QAEJ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAG,SAAS;aACxB,GAAG,CACF,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF;aACA,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,GAAG,YAAY,2CAA2C,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE,6KAA6K;qBAChX;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;4BACjC,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,uBAAuB;oBAC7B,MAAM,EAAE,qBAAqB;iBAC9B;aACF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAExB,CAAC;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1B,UAAU,EAAE,SAAkB;gBAC9B,SAAS,EAAE,sCAAsC,YAAY,EAAE;aAChE,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAK9C;QACC,MAAM,EACJ,QAAQ,EACR,WAAW,EACX,cAAc,EACd,YAAY,GAAG;UACX,cAAc,CAAC,CAAC,CAAC,iGAAiG,CAAC,CAAC,CAAC,EAAE;;;UAGvH,cAAc,CAAC,CAAC,CAAC,kNAAkN,CAAC,CAAC,CAAC,EAAE;0BACxN,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,GACtD,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAC1C,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,CAC3C,CAAC;QAEJ,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,WAAoB;YAC1B,SAAS,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;SACrE,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;oBACzC;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,2DAA2D,cAAc,qBAAqB,WAAW,CAAC,MAAM,sKAAsK;oCAC7S,CAAC,CAAC,GAAG,QAAQ,qBAAqB,WAAW,CAAC,MAAM,mIAAmI;6BAC1L;4BACD,GAAG,aAAa;yBACjB;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;CACF","sourcesContent":["/**\n * V3Evaluator mirrors Evaluator but operates on a V3 instance instead of Stagehand.\n * It uses the V3 page/screenshot APIs and constructs an LLM client to run\n * structured evaluations (YES/NO with reasoning) on screenshots and/or text.\n */\n\nimport { z } from \"zod\";\nimport type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { LLMParsedResponse } from \"./inference.js\";\nimport { LLMResponse, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\nconst EvaluationSchema = z.object({\n  evaluation: z.enum([\"YES\", \"NO\"]),\n  reasoning: z.string(),\n});\n\nconst BatchEvaluationSchema = z.array(EvaluationSchema);\n\nexport class V3Evaluator {\n  private v3: V3;\n  private modelName: AvailableModel;\n  private modelClientOptions: ClientOptions | { apiKey: string };\n  private silentLogger: (message: LogLine) => void = () => {};\n\n  constructor(\n    v3: V3,\n    modelName?: AvailableModel,\n    modelClientOptions?: ClientOptions,\n  ) {\n    this.v3 = v3;\n    this.modelName = modelName || (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n  }\n\n  private getClient(): LLMClient {\n    // Prefer a dedicated provider so we can override model per-evaluation\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshot = true,\n      systemPrompt,\n      screenshotDelayMs = 250,\n      agentReasoning,\n    } = options;\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!answer && !screenshot)\n      throw new StagehandInvalidArgumentError(\n        \"Either answer (text) or screenshot must be provided\",\n      );\n\n    if (Array.isArray(screenshot)) {\n      return this._evaluateWithMultipleScreenshots({\n        question,\n        screenshots: screenshot,\n        systemPrompt,\n        agentReasoning,\n      });\n    }\n\n    const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? \"a screenshot\" : \"the agents reasoning and actions throughout the task\"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\\n          Today's date is ${new Date().toLocaleDateString()}`;\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt || defaultSystemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions taken:\\n${agentReasoning}`\n                  : question,\n              },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    const {\n      questions,\n      screenshot = true,\n      systemPrompt = \"You are an expert evaluator that returns YES or NO with a concise reasoning.\",\n      screenshotDelayMs = 250,\n    } = options;\n    if (!questions?.length)\n      throw new StagehandInvalidArgumentError(\n        \"Questions array cannot be empty\",\n      );\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const formatted = questions\n      .map(\n        (item, i) =>\n          `${i + 1}. ${item.question}${item.answer ? `\\n   Answer: ${item.answer}` : \"\"}`,\n      )\n      .join(\"\\n\\n\");\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          {\n            role: \"system\",\n            content: `${systemPrompt}\\n\\nYou will be given multiple questions${screenshot ? \" with a screenshot\" : \"\"}. ${questions.some((q) => q.answer) ? \"Some questions include answers to evaluate.\" : \"\"} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,\n          },\n          {\n            role: \"user\",\n            content: [\n              { type: \"text\", text: formatted },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n            ],\n          },\n        ],\n        response_model: {\n          name: \"BatchEvaluationResult\",\n          schema: BatchEvaluationSchema,\n        },\n      },\n    });\n\n    try {\n      const results = response.data as unknown as z.infer<\n        typeof BatchEvaluationSchema\n      >;\n      return results.map((r) => ({\n        evaluation: r.evaluation,\n        reasoning: r.reasoning,\n      }));\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return questions.map(() => ({\n        evaluation: \"INVALID\" as const,\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      }));\n    }\n  }\n\n  private async _evaluateWithMultipleScreenshots(options: {\n    question: string;\n    screenshots: Buffer[];\n    systemPrompt?: string;\n    agentReasoning?: string;\n  }): Promise<EvaluationResult> {\n    const {\n      question,\n      screenshots,\n      agentReasoning,\n      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.\n        ${agentReasoning ? \"You also have access to the agent's detailed reasoning and thought process throughout the task.\" : \"\"}\n        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.\n        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).\n        ${agentReasoning ? \"The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation.\" : \"\"}\n        Today's date is ${new Date().toLocaleDateString()}`,\n    } = options;\n\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!screenshots || screenshots.length === 0)\n      throw new StagehandInvalidArgumentError(\n        \"At least one screenshot must be provided\",\n      );\n\n    const llmClient = this.getClient();\n\n    const imageContents = screenshots.map((s) => ({\n      type: \"image_url\" as const,\n      image_url: { url: `data:image/jpeg;base64,${s.toString(\"base64\")}` },\n    }));\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions throughout the task:\\n${agentReasoning}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`\n                  : `${question}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,\n              },\n              ...imageContents,\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n}\n"]}
+{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,qBAAqB,GAAG,6BAA6B,CAAC;AAC5D,MAAM,yBAAyB,GAAuB,QAAQ,CAAC;AA2B/D,MAAM,OAAO,WAAW;IACL,OAAO,CAAqB;IAC5B,eAAe,CAAoB;IAEpD,YACE,EAAM,EACN,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;QAE5B,MAAM,iBAAiB,GAAG,2BAA2B,CACnD,kBAAkB,EAClB,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAC1C,EAAE,EACF,iBAAiB,CAAC,SAAS,EAC3B,iBAAiB,CAAC,kBAAkB,CACrC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,MAAM,IAAI,6BAA6B,CACrC,eAAe,UAAU,0BAA0B,qBAAqB,sHAAsH,CAC/L,CAAC;IACJ,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;IAE5B,IACE,kBAAkB;QAClB,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,CAAC;QACD,OAAO;YACL,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,kBAAkB,EAAE,kBAAkB,CAAC,kBAAkB;YACzD,OAAO,EAAE,kBAAkB,CAAC,OAAO,IAAI,OAAO,EAAE,OAAO;SACxD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAgD;QAC3D,kBAAkB;QAClB,OAAO,EAAE,OAAO,EAAE,OAAO;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,eAAoC;IAEpC,MAAM,iBAAiB,GACrB,eAAe;QACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;QAClC,yBAAyB,CAAC;IAC5B,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEjE,IAAI,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,KAAK,UAAU,EAAE,CAAC;QACvE,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,6BAA6B,CACrC,WAAW,qBAAqB,KAAK,iBAAiB,qCAAqC,CAC5F,CAAC;AACJ,CAAC","sourcesContent":["import type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\nimport { LegacyV3Evaluator } from \"./v3LegacyEvaluator.js\";\n\nconst EVALUATOR_BACKEND_ENV = \"STAGEHAND_EVALUATOR_BACKEND\";\nconst DEFAULT_EVALUATOR_BACKEND: V3EvaluatorBackend = \"legacy\";\n\nexport type V3EvaluatorBackend = \"legacy\" | \"verifier\";\n\nexport type V3EvaluatorOptions = {\n  /**\n   * Selects the evaluator implementation.\n   *\n   * \"legacy\" preserves the existing screenshot/text YES/NO evaluator.\n   * \"verifier\" is reserved for the rubric verifier backend.\n   *\n   * @default process.env.STAGEHAND_EVALUATOR_BACKEND || \"legacy\"\n   */\n  backend?: V3EvaluatorBackend;\n};\n\nexport type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n};\n\ntype NormalizedConstructorOptions = {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n  backend?: V3EvaluatorBackend;\n};\n\nexport class V3Evaluator {\n  private readonly backend: V3EvaluatorBackend;\n  private readonly legacyEvaluator: LegacyV3Evaluator;\n\n  constructor(\n    v3: V3,\n    modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n    modelClientOptions?: ClientOptions,\n    options?: V3EvaluatorOptions,\n  ) {\n    const normalizedOptions = normalizeConstructorOptions(\n      modelNameOrOptions,\n      modelClientOptions,\n      options,\n    );\n\n    this.backend = resolveEvaluatorBackend(normalizedOptions.backend);\n    this.legacyEvaluator = new LegacyV3Evaluator(\n      v3,\n      normalizedOptions.modelName,\n      normalizedOptions.modelClientOptions,\n    );\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    return this.getLegacyBackend(\"ask\").ask(options);\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    return this.getLegacyBackend(\"batchAsk\").batchAsk(options);\n  }\n\n  private getLegacyBackend(methodName: string): LegacyV3Evaluator {\n    if (this.backend === \"legacy\") {\n      return this.legacyEvaluator;\n    }\n\n    throw new StagehandInvalidArgumentError(\n      `V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use \"legacy\" or install the verifier backend PR.`,\n    );\n  }\n}\n\nfunction normalizeConstructorOptions(\n  modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n  modelClientOptions?: ClientOptions,\n  options?: V3EvaluatorOptions,\n): NormalizedConstructorOptions {\n  if (\n    modelNameOrOptions &&\n    typeof modelNameOrOptions === \"object\" &&\n    !Array.isArray(modelNameOrOptions)\n  ) {\n    return {\n      modelName: modelNameOrOptions.modelName,\n      modelClientOptions: modelNameOrOptions.modelClientOptions,\n      backend: modelNameOrOptions.backend ?? options?.backend,\n    };\n  }\n\n  return {\n    modelName: modelNameOrOptions as AvailableModel | undefined,\n    modelClientOptions,\n    backend: options?.backend,\n  };\n}\n\nfunction resolveEvaluatorBackend(\n  explicitBackend?: V3EvaluatorBackend,\n): V3EvaluatorBackend {\n  const configuredBackend =\n    explicitBackend ??\n    process.env[EVALUATOR_BACKEND_ENV] ??\n    DEFAULT_EVALUATOR_BACKEND;\n  const normalizedBackend = configuredBackend.trim().toLowerCase();\n\n  if (normalizedBackend === \"legacy\" || normalizedBackend === \"verifier\") {\n    return normalizedBackend;\n  }\n\n  throw new StagehandInvalidArgumentError(\n    `Invalid ${EVALUATOR_BACKEND_ENV}=\"${configuredBackend}\". Expected \"legacy\" or \"verifier\".`,\n  );\n}\n"]}

package/dist/esm/lib/v3LegacyEvaluator.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Legacy V3 evaluator implementation.
+ *
+ * This is the behavior-preserving implementation that backs V3Evaluator when
+ * STAGEHAND_EVALUATOR_BACKEND=legacy.
+ */
+import type { AvailableModel, ClientOptions } from "./v3/types/public/model.js";
+import type { EvaluateOptions, BatchAskOptions, EvaluationResult } from "./v3/types/private/evaluator.js";
+import { V3 } from "./v3/v3.js";
+export declare class LegacyV3Evaluator {
+    private v3;
+    private modelName;
+    private modelClientOptions;
+    private silentLogger;
+    constructor(v3: V3, modelName?: AvailableModel, modelClientOptions?: ClientOptions);
+    private getClient;
+    ask(options: EvaluateOptions): Promise<EvaluationResult>;
+    batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]>;
+    private _evaluateWithMultipleScreenshots;
+}

package/dist/esm/lib/v3LegacyEvaluator.js

@@ -0,0 +1,211 @@
+/**
+ * Legacy V3 evaluator implementation.
+ *
+ * This is the behavior-preserving implementation that backs V3Evaluator when
+ * STAGEHAND_EVALUATOR_BACKEND=legacy.
+ */
+import { z } from "zod";
+import { LLMProvider } from "./v3/llm/LLMProvider.js";
+import { StagehandInvalidArgumentError } from "./v3/types/public/sdkErrors.js";
+const EvaluationSchema = z.object({
+    evaluation: z.enum(["YES", "NO"]),
+    reasoning: z.string(),
+});
+const BatchEvaluationSchema = z.array(EvaluationSchema);
+export class LegacyV3Evaluator {
+    v3;
+    modelName;
+    modelClientOptions;
+    silentLogger = () => { };
+    constructor(v3, modelName, modelClientOptions) {
+        this.v3 = v3;
+        this.modelName = modelName || "google/gemini-2.5-flash";
+        this.modelClientOptions = modelClientOptions || {
+            apiKey: process.env.GEMINI_API_KEY ||
+                process.env.GOOGLE_GENERATIVE_AI_API_KEY ||
+                "",
+        };
+    }
+    getClient() {
+        // Prefer a dedicated provider so we can override model per-evaluation
+        const provider = new LLMProvider(this.v3.logger);
+        return provider.getClient(this.modelName, this.modelClientOptions);
+    }
+    async ask(options) {
+        const { question, answer, screenshot = true, systemPrompt, screenshotDelayMs = 250, agentReasoning, } = options;
+        if (!question)
+            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
+        if (!answer && !screenshot)
+            throw new StagehandInvalidArgumentError("Either answer (text) or screenshot must be provided");
+        if (Array.isArray(screenshot)) {
+            return this._evaluateWithMultipleScreenshots({
+                question,
+                screenshots: screenshot,
+                systemPrompt,
+                agentReasoning,
+            });
+        }
+        const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? "a screenshot" : "the agents reasoning and actions throughout the task"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\n          Today's date is ${new Date().toLocaleDateString()}`;
+        await new Promise((r) => setTimeout(r, screenshotDelayMs));
+        let imageBuffer;
+        if (screenshot) {
+            const page = await this.v3.context.awaitActivePage();
+            imageBuffer = await page.screenshot({ fullPage: false });
+        }
+        const llmClient = this.getClient();
+        const response = await llmClient.createChatCompletion({
+            logger: this.silentLogger,
+            options: {
+                messages: [
+                    { role: "system", content: systemPrompt || defaultSystemPrompt },
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "text",
+                                text: agentReasoning
+                                    ? `Question: ${question}\n\nAgent's reasoning and actions taken:\n${agentReasoning}`
+                                    : question,
+                            },
+                            ...(screenshot && imageBuffer
+                                ? [
+                                    {
+                                        type: "image_url",
+                                        image_url: {
+                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
+                                        },
+                                    },
+                                ]
+                                : []),
+                            ...(answer
+                                ? [{ type: "text", text: `the answer is ${answer}` }]
+                                : []),
+                        ],
+                    },
+                ],
+                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
+            },
+        });
+        try {
+            const result = response.data;
+            return { evaluation: result.evaluation, reasoning: result.reasoning };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                evaluation: "INVALID",
+                reasoning: `Failed to get structured response: ${errorMessage}`,
+            };
+        }
+    }
+    async batchAsk(options) {
+        const { questions, screenshot = true, systemPrompt = "You are an expert evaluator that returns YES or NO with a concise reasoning.", screenshotDelayMs = 250, } = options;
+        if (!questions?.length)
+            throw new StagehandInvalidArgumentError("Questions array cannot be empty");
+        await new Promise((r) => setTimeout(r, screenshotDelayMs));
+        let imageBuffer;
+        if (screenshot) {
+            const page = await this.v3.context.awaitActivePage();
+            imageBuffer = await page.screenshot({ fullPage: false });
+        }
+        const llmClient = this.getClient();
+        const formatted = questions
+            .map((item, i) => `${i + 1}. ${item.question}${item.answer ? `\n   Answer: ${item.answer}` : ""}`)
+            .join("\n\n");
+        const response = await llmClient.createChatCompletion({
+            logger: this.silentLogger,
+            options: {
+                messages: [
+                    {
+                        role: "system",
+                        content: `${systemPrompt}\n\nYou will be given multiple questions${screenshot ? " with a screenshot" : ""}. ${questions.some((q) => q.answer) ? "Some questions include answers to evaluate." : ""} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,
+                    },
+                    {
+                        role: "user",
+                        content: [
+                            { type: "text", text: formatted },
+                            ...(screenshot && imageBuffer
+                                ? [
+                                    {
+                                        type: "image_url",
+                                        image_url: {
+                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
+                                        },
+                                    },
+                                ]
+                                : []),
+                        ],
+                    },
+                ],
+                response_model: {
+                    name: "BatchEvaluationResult",
+                    schema: BatchEvaluationSchema,
+                },
+            },
+        });
+        try {
+            const results = response.data;
+            return results.map((r) => ({
+                evaluation: r.evaluation,
+                reasoning: r.reasoning,
+            }));
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return questions.map(() => ({
+                evaluation: "INVALID",
+                reasoning: `Failed to get structured response: ${errorMessage}`,
+            }));
+        }
+    }
+    async _evaluateWithMultipleScreenshots(options) {
+        const { question, screenshots, agentReasoning, systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.
+        ${agentReasoning ? "You also have access to the agent's detailed reasoning and thought process throughout the task." : ""}
+        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.
+        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).
+        ${agentReasoning ? "The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation." : ""}
+        Today's date is ${new Date().toLocaleDateString()}`, } = options;
+        if (!question)
+            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
+        if (!screenshots || screenshots.length === 0)
+            throw new StagehandInvalidArgumentError("At least one screenshot must be provided");
+        const llmClient = this.getClient();
+        const imageContents = screenshots.map((s) => ({
+            type: "image_url",
+            image_url: { url: `data:image/jpeg;base64,${s.toString("base64")}` },
+        }));
+        const response = await llmClient.createChatCompletion({
+            logger: this.silentLogger,
+            options: {
+                messages: [
+                    { role: "system", content: systemPrompt },
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "text",
+                                text: agentReasoning
+                                    ? `Question: ${question}\n\nAgent's reasoning and actions throughout the task:\n${agentReasoning}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`
+                                    : `${question}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,
+                            },
+                            ...imageContents,
+                        ],
+                    },
+                ],
+                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
+            },
+        });
+        try {
+            const result = response.data;
+            return { evaluation: result.evaluation, reasoning: result.reasoning };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                evaluation: "INVALID",
+                reasoning: `Failed to get structured response: ${errorMessage}`,
+            };
+        }
+    }
+}
+//# sourceMappingURL=v3LegacyEvaluator.js.map

package/dist/esm/lib/v3LegacyEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"v3LegacyEvaluator.js","sourceRoot":"","sources":["../../../lib/v3LegacyEvaluator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAExD,MAAM,OAAO,iBAAiB;IACpB,EAAE,CAAK;IACP,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,YAAY,GAA+B,GAAG,EAAE,GAAE,CAAC,CAAC;IAE5D,YACE,EAAM,EACN,SAA0B,EAC1B,kBAAkC;QAElC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,IAAK,yBAA4C,CAAC;QAC5E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI;YAC9C,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,sEAAsE;QACtE,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,UAAU,GAAG,IAAI,EACjB,YAAY,EACZ,iBAAiB,GAAG,GAAG,EACvB,cAAc,GACf,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;YACxB,MAAM,IAAI,6BAA6B,CACrC,qDAAqD,CACtD,CAAC;QAEJ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,gCAAgC,CAAC;gBAC3C,QAAQ;gBACR,WAAW,EAAE,UAAU;gBACvB,YAAY;gBACZ,cAAc;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,mBAAmB,GAAG,kIAAkI,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,sDAAsD,8HAA8H,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC;QAElZ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE;oBAChE;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,6CAA6C,cAAc,EAAE;oCACpF,CAAC,CAAC,QAAQ;6BACb;4BACD,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,EACJ,SAAS,EACT,UAAU,GAAG,IAAI,EACjB,YAAY,GAAG,8EAA8E,EAC7F,iBAAiB,GAAG,GAAG,GACxB,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,SAAS,EAAE,MAAM;YACpB,MAAM,IAAI,6BAA6B,CACrC,iCAAiC,CAClC,CAAC;QAEJ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAG,SAAS;aACxB,GAAG,CACF,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF;aACA,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,GAAG,YAAY,2CAA2C,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE,6KAA6K;qBAChX;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;4BACjC,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,uBAAuB;oBAC7B,MAAM,EAAE,qBAAqB;iBAC9B;aACF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAExB,CAAC;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1B,UAAU,EAAE,SAAkB;gBAC9B,SAAS,EAAE,sCAAsC,YAAY,EAAE;aAChE,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAK9C;QACC,MAAM,EACJ,QAAQ,EACR,WAAW,EACX,cAAc,EACd,YAAY,GAAG;UACX,cAAc,CAAC,CAAC,CAAC,iGAAiG,CAAC,CAAC,CAAC,EAAE;;;UAGvH,cAAc,CAAC,CAAC,CAAC,kNAAkN,CAAC,CAAC,CAAC,EAAE;0BACxN,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,GACtD,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAC1C,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,CAC3C,CAAC;QAEJ,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,WAAoB;YAC1B,SAAS,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;SACrE,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;oBACzC;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,2DAA2D,cAAc,qBAAqB,WAAW,CAAC,MAAM,sKAAsK;oCAC7S,CAAC,CAAC,GAAG,QAAQ,qBAAqB,WAAW,CAAC,MAAM,mIAAmI;6BAC1L;4BACD,GAAG,aAAa;yBACjB;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;CACF","sourcesContent":["/**\n * Legacy V3 evaluator implementation.\n *\n * This is the behavior-preserving implementation that backs V3Evaluator when\n * STAGEHAND_EVALUATOR_BACKEND=legacy.\n */\n\nimport { z } from \"zod\";\nimport type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { LLMParsedResponse } from \"./inference.js\";\nimport { LLMResponse, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\nconst EvaluationSchema = z.object({\n  evaluation: z.enum([\"YES\", \"NO\"]),\n  reasoning: z.string(),\n});\n\nconst BatchEvaluationSchema = z.array(EvaluationSchema);\n\nexport class LegacyV3Evaluator {\n  private v3: V3;\n  private modelName: AvailableModel;\n  private modelClientOptions: ClientOptions | { apiKey: string };\n  private silentLogger: (message: LogLine) => void = () => {};\n\n  constructor(\n    v3: V3,\n    modelName?: AvailableModel,\n    modelClientOptions?: ClientOptions,\n  ) {\n    this.v3 = v3;\n    this.modelName = modelName || (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n  }\n\n  private getClient(): LLMClient {\n    // Prefer a dedicated provider so we can override model per-evaluation\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshot = true,\n      systemPrompt,\n      screenshotDelayMs = 250,\n      agentReasoning,\n    } = options;\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!answer && !screenshot)\n      throw new StagehandInvalidArgumentError(\n        \"Either answer (text) or screenshot must be provided\",\n      );\n\n    if (Array.isArray(screenshot)) {\n      return this._evaluateWithMultipleScreenshots({\n        question,\n        screenshots: screenshot,\n        systemPrompt,\n        agentReasoning,\n      });\n    }\n\n    const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? \"a screenshot\" : \"the agents reasoning and actions throughout the task\"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\\n          Today's date is ${new Date().toLocaleDateString()}`;\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt || defaultSystemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions taken:\\n${agentReasoning}`\n                  : question,\n              },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    const {\n      questions,\n      screenshot = true,\n      systemPrompt = \"You are an expert evaluator that returns YES or NO with a concise reasoning.\",\n      screenshotDelayMs = 250,\n    } = options;\n    if (!questions?.length)\n      throw new StagehandInvalidArgumentError(\n        \"Questions array cannot be empty\",\n      );\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const formatted = questions\n      .map(\n        (item, i) =>\n          `${i + 1}. ${item.question}${item.answer ? `\\n   Answer: ${item.answer}` : \"\"}`,\n      )\n      .join(\"\\n\\n\");\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          {\n            role: \"system\",\n            content: `${systemPrompt}\\n\\nYou will be given multiple questions${screenshot ? \" with a screenshot\" : \"\"}. ${questions.some((q) => q.answer) ? \"Some questions include answers to evaluate.\" : \"\"} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,\n          },\n          {\n            role: \"user\",\n            content: [\n              { type: \"text\", text: formatted },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n            ],\n          },\n        ],\n        response_model: {\n          name: \"BatchEvaluationResult\",\n          schema: BatchEvaluationSchema,\n        },\n      },\n    });\n\n    try {\n      const results = response.data as unknown as z.infer<\n        typeof BatchEvaluationSchema\n      >;\n      return results.map((r) => ({\n        evaluation: r.evaluation,\n        reasoning: r.reasoning,\n      }));\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return questions.map(() => ({\n        evaluation: \"INVALID\" as const,\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      }));\n    }\n  }\n\n  private async _evaluateWithMultipleScreenshots(options: {\n    question: string;\n    screenshots: Buffer[];\n    systemPrompt?: string;\n    agentReasoning?: string;\n  }): Promise<EvaluationResult> {\n    const {\n      question,\n      screenshots,\n      agentReasoning,\n      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.\n        ${agentReasoning ? \"You also have access to the agent's detailed reasoning and thought process throughout the task.\" : \"\"}\n        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.\n        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).\n        ${agentReasoning ? \"The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation.\" : \"\"}\n        Today's date is ${new Date().toLocaleDateString()}`,\n    } = options;\n\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!screenshots || screenshots.length === 0)\n      throw new StagehandInvalidArgumentError(\n        \"At least one screenshot must be provided\",\n      );\n\n    const llmClient = this.getClient();\n\n    const imageContents = screenshots.map((s) => ({\n      type: \"image_url\" as const,\n      image_url: { url: `data:image/jpeg;base64,${s.toString(\"base64\")}` },\n    }));\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions throughout the task:\\n${agentReasoning}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`\n                  : `${question}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,\n              },\n              ...imageContents,\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserbasehq/orca",
-  "version": "3.4.0-preview-5",
+  "version": "3.5.0-preview.0",
   "description": "An AI web browsing framework focused on simplicity and extensibility.",
   "type": "module",
   "main": "./dist/cjs/index.js",
@@ -35,9 +35,9 @@
     "build-dom-scripts": "tsx scripts/build-dom-scripts.ts",
     "build:cjs": "tsx scripts/build-cjs.ts",
     "build:esm": "tsx scripts/build-esm.ts",
-    "build": "pnpm --filter @browserbasehq/stagehand run --parallel \"/^build:(esm|cjs)$/\"",
+    "build": "pnpm --filter @browserbasehq/orca run --parallel \"/^build:(esm|cjs)$/\"",
     "example": "node --import tsx -e \"const args=process.argv.slice(1).filter(a=>a!=='--'); const [p]=args; const n=(p||'example').replace(/^\\.\\//,'').replace(/\\.ts$/i,''); import('node:path').then(path=>import(new URL(path.resolve('examples', n + '.ts'), 'file:')));\" --",
-    "test": "pnpm -w --dir ../.. exec turbo run test:core test:e2e --filter=@browserbasehq/stagehand --",
+    "test": "pnpm -w --dir ../.. exec turbo run test:core test:e2e --filter=@browserbasehq/orca --",
     "test:core": "tsx scripts/test-core.ts",
     "test:e2e": "tsx scripts/test-e2e.ts",
     "format": "prettier --write .",