@browserbasehq/orca 3.4.0-preview.0 → 3.5.0-preview.1

package/dist/esm/lib/v3Evaluator.js

@@ -1,210 +1,200 @@
-/**
- * V3Evaluator mirrors Evaluator but operates on a V3 instance instead of Stagehand.
- * It uses the V3 page/screenshot APIs and constructs an LLM client to run
- * structured evaluations (YES/NO with reasoning) on screenshots and/or text.
- */
-import { z } from "zod";
-import { LLMProvider } from "./v3/llm/LLMProvider.js";
 import { StagehandInvalidArgumentError } from "./v3/types/public/sdkErrors.js";
-const EvaluationSchema = z.object({
-    evaluation: z.enum(["YES", "NO"]),
-    reasoning: z.string(),
-});
-const BatchEvaluationSchema = z.array(EvaluationSchema);
+import { LegacyV3Evaluator } from "./v3LegacyEvaluator.js";
+const EVALUATOR_BACKEND_ENV = "STAGEHAND_EVALUATOR_BACKEND";
+const DEFAULT_EVALUATOR_BACKEND = "legacy";
 export class V3Evaluator {
-    v3;
-    modelName;
-    modelClientOptions;
-    silentLogger = () => { };
-    constructor(v3, modelName, modelClientOptions) {
-        this.v3 = v3;
-        this.modelName = modelName || "google/gemini-2.5-flash";
-        this.modelClientOptions = modelClientOptions || {
-            apiKey: process.env.GEMINI_API_KEY ||
-                process.env.GOOGLE_GENERATIVE_AI_API_KEY ||
-                "",
-        };
-    }
-    getClient() {
-        // Prefer a dedicated provider so we can override model per-evaluation
-        const provider = new LLMProvider(this.v3.logger);
-        return provider.getClient(this.modelName, this.modelClientOptions);
+    backend;
+    legacyEvaluator;
+    constructor(v3, modelNameOrOptions, modelClientOptions, options) {
+        const normalizedOptions = normalizeConstructorOptions(modelNameOrOptions, modelClientOptions, options);
+        this.backend = resolveEvaluatorBackend(normalizedOptions.backend);
+        this.legacyEvaluator = new LegacyV3Evaluator(v3, normalizedOptions.modelName, normalizedOptions.modelClientOptions);
     }
     async ask(options) {
-        const { question, answer, screenshot = true, systemPrompt, screenshotDelayMs = 250, agentReasoning, } = options;
-        if (!question)
-            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
-        if (!answer && !screenshot)
-            throw new StagehandInvalidArgumentError("Either answer (text) or screenshot must be provided");
-        if (Array.isArray(screenshot)) {
-            return this._evaluateWithMultipleScreenshots({
-                question,
-                screenshots: screenshot,
-                systemPrompt,
-                agentReasoning,
-            });
+        return this.getLegacyBackend("ask").ask(options);
+    }
+    async batchAsk(options) {
+        return this.getLegacyBackend("batchAsk").batchAsk(options);
+    }
+    async verify(trajectory) {
+        const taskSpec = assertVerifierInput(trajectory);
+        if (this.backend === "legacy") {
+            return this.verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec);
         }
-        const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? "a screenshot" : "the agents reasoning and actions throughout the task"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\n          Today's date is ${new Date().toLocaleDateString()}`;
-        await new Promise((r) => setTimeout(r, screenshotDelayMs));
-        let imageBuffer;
-        if (screenshot) {
-            const page = await this.v3.context.awaitActivePage();
-            imageBuffer = await page.screenshot({ fullPage: false });
+        return this.unavailableVerifierBackend("verify");
+    }
+    async generateRubric(taskSpec) {
+        if (!taskSpec?.id) {
+            throw new StagehandInvalidArgumentError("TaskSpec.id is required for rubric generation");
         }
-        const llmClient = this.getClient();
-        const response = await llmClient.createChatCompletion({
-            logger: this.silentLogger,
-            options: {
-                messages: [
-                    { role: "system", content: systemPrompt || defaultSystemPrompt },
-                    {
-                        role: "user",
-                        content: [
-                            {
-                                type: "text",
-                                text: agentReasoning
-                                    ? `Question: ${question}\n\nAgent's reasoning and actions taken:\n${agentReasoning}`
-                                    : question,
-                            },
-                            ...(screenshot && imageBuffer
-                                ? [
-                                    {
-                                        type: "image_url",
-                                        image_url: {
-                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
-                                        },
-                                    },
-                                ]
-                                : []),
-                            ...(answer
-                                ? [{ type: "text", text: `the answer is ${answer}` }]
-                                : []),
-                        ],
-                    },
-                ],
-                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
-            },
-        });
-        try {
-            const result = response.data;
-            return { evaluation: result.evaluation, reasoning: result.reasoning };
+        if (this.backend === "verifier") {
+            return this.unavailableVerifierBackend("generateRubric");
         }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                evaluation: "INVALID",
-                reasoning: `Failed to get structured response: ${errorMessage}`,
-            };
+        return {
+            items: [legacyTaskCompletionCriterion(taskSpec)],
+        };
+    }
+    getLegacyBackend(methodName) {
+        if (this.backend === "legacy") {
+            return this.legacyEvaluator;
         }
+        return this.unavailableVerifierBackend(methodName);
     }
-    async batchAsk(options) {
-        const { questions, screenshot = true, systemPrompt = "You are an expert evaluator that returns YES or NO with a concise reasoning.", screenshotDelayMs = 250, } = options;
-        if (!questions?.length)
-            throw new StagehandInvalidArgumentError("Questions array cannot be empty");
-        await new Promise((r) => setTimeout(r, screenshotDelayMs));
-        let imageBuffer;
-        if (screenshot) {
-            const page = await this.v3.context.awaitActivePage();
-            imageBuffer = await page.screenshot({ fullPage: false });
+    unavailableVerifierBackend(methodName) {
+        throw new StagehandInvalidArgumentError(`V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use "legacy" or install the verifier backend PR.`);
+    }
+    async verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec) {
+        const screenshots = collectLegacyScreenshots(trajectory);
+        const agentReasoning = renderLegacyAgentReasoning(trajectory);
+        const answer = trajectory.finalAnswer;
+        if (!screenshots.length && !answer) {
+            return legacyInsufficientEvidenceResult("Legacy evaluator compatibility mode had no screenshots or final answer to evaluate.");
         }
-        const llmClient = this.getClient();
-        const formatted = questions
-            .map((item, i) => `${i + 1}. ${item.question}${item.answer ? `\n   Answer: ${item.answer}` : ""}`)
-            .join("\n\n");
-        const response = await llmClient.createChatCompletion({
-            logger: this.silentLogger,
-            options: {
-                messages: [
-                    {
-                        role: "system",
-                        content: `${systemPrompt}\n\nYou will be given multiple questions${screenshot ? " with a screenshot" : ""}. ${questions.some((q) => q.answer) ? "Some questions include answers to evaluate." : ""} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,
-                    },
-                    {
-                        role: "user",
-                        content: [
-                            { type: "text", text: formatted },
-                            ...(screenshot && imageBuffer
-                                ? [
-                                    {
-                                        type: "image_url",
-                                        image_url: {
-                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
-                                        },
-                                    },
-                                ]
-                                : []),
-                        ],
-                    },
-                ],
-                response_model: {
-                    name: "BatchEvaluationResult",
-                    schema: BatchEvaluationSchema,
-                },
-            },
+        const result = await this.legacyEvaluator.ask({
+            question: taskSpec.instruction,
+            screenshot: screenshots.length ? screenshots : false,
+            answer,
+            agentReasoning,
         });
-        try {
-            const results = response.data;
-            return results.map((r) => ({
-                evaluation: r.evaluation,
-                reasoning: r.reasoning,
-            }));
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return questions.map(() => ({
-                evaluation: "INVALID",
-                reasoning: `Failed to get structured response: ${errorMessage}`,
-            }));
-        }
+        return legacyEvaluationToResult(result, screenshots.length);
     }
-    async _evaluateWithMultipleScreenshots(options) {
-        const { question, screenshots, agentReasoning, systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.
-        ${agentReasoning ? "You also have access to the agent's detailed reasoning and thought process throughout the task." : ""}
-        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.
-        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).
-        ${agentReasoning ? "The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation." : ""}
-        Today's date is ${new Date().toLocaleDateString()}`, } = options;
-        if (!question)
-            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
-        if (!screenshots || screenshots.length === 0)
-            throw new StagehandInvalidArgumentError("At least one screenshot must be provided");
-        const llmClient = this.getClient();
-        const imageContents = screenshots.map((s) => ({
-            type: "image_url",
-            image_url: { url: `data:image/jpeg;base64,${s.toString("base64")}` },
-        }));
-        const response = await llmClient.createChatCompletion({
-            logger: this.silentLogger,
-            options: {
-                messages: [
-                    { role: "system", content: systemPrompt },
-                    {
-                        role: "user",
-                        content: [
-                            {
-                                type: "text",
-                                text: agentReasoning
-                                    ? `Question: ${question}\n\nAgent's reasoning and actions throughout the task:\n${agentReasoning}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`
-                                    : `${question}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,
-                            },
-                            ...imageContents,
-                        ],
-                    },
-                ],
-                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
-            },
-        });
-        try {
-            const result = response.data;
-            return { evaluation: result.evaluation, reasoning: result.reasoning };
+}
+function normalizeConstructorOptions(modelNameOrOptions, modelClientOptions, options) {
+    if (modelNameOrOptions &&
+        typeof modelNameOrOptions === "object" &&
+        !Array.isArray(modelNameOrOptions)) {
+        return {
+            modelName: modelNameOrOptions.modelName,
+            modelClientOptions: modelNameOrOptions.modelClientOptions,
+            backend: modelNameOrOptions.backend ?? options?.backend,
+        };
+    }
+    return {
+        modelName: modelNameOrOptions,
+        modelClientOptions,
+        backend: options?.backend,
+    };
+}
+function resolveEvaluatorBackend(explicitBackend) {
+    const configuredBackend = explicitBackend ??
+        process.env[EVALUATOR_BACKEND_ENV] ??
+        DEFAULT_EVALUATOR_BACKEND;
+    const normalizedBackend = configuredBackend.trim().toLowerCase();
+    if (normalizedBackend === "legacy" || normalizedBackend === "verifier") {
+        return normalizedBackend;
+    }
+    throw new StagehandInvalidArgumentError(`Invalid ${EVALUATOR_BACKEND_ENV}="${configuredBackend}". Expected "legacy" or "verifier".`);
+}
+function assertVerifierInput(trajectory) {
+    if (!trajectory) {
+        throw new StagehandInvalidArgumentError("Trajectory is required for verification");
+    }
+    if (!trajectory.task?.id) {
+        throw new StagehandInvalidArgumentError("Trajectory.task.id is required for verification");
+    }
+    return trajectory.task;
+}
+function legacyTaskCompletionCriterion(taskSpec) {
+    return {
+        criterion: "legacy-task-completion",
+        description: `Evaluate whether the task was completed successfully: ${taskSpec.instruction}`,
+        maxPoints: 1,
+    };
+}
+function collectLegacyScreenshots(trajectory) {
+    const screenshots = [];
+    for (const step of trajectory.steps ?? []) {
+        if (Buffer.isBuffer(step.probeEvidence?.screenshot)) {
+            screenshots.push(step.probeEvidence.screenshot);
+            continue;
         }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                evaluation: "INVALID",
-                reasoning: `Failed to get structured response: ${errorMessage}`,
-            };
+        const agentImage = step.agentEvidence?.modalities?.find((modality) => modality.type === "image" && Buffer.isBuffer(modality.bytes));
+        if (agentImage) {
+            screenshots.push(agentImage.bytes);
         }
     }
+    if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {
+        screenshots.push(trajectory.finalObservation.screenshot);
+    }
+    return screenshots;
+}
+function renderLegacyAgentReasoning(trajectory) {
+    const stepLines = (trajectory.steps ?? []).map((step, i) => {
+        const status = step.toolOutput?.ok === false ? "Tool status: failed" : "";
+        const output = step.toolOutput?.error
+            ? `Tool error: ${step.toolOutput.error}`
+            : `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;
+        return [
+            `Step ${i}: ${step.actionName}`,
+            step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,
+            status || undefined,
+            output,
+        ]
+            .filter(Boolean)
+            .join("\n");
+    });
+    if (!stepLines.length) {
+        return undefined;
+    }
+    return truncateForPrompt(`Agent trajectory:\n${stepLines.join("\n\n")}`, 16000);
+}
+function stringifyForPrompt(value) {
+    if (typeof value === "string") {
+        return value;
+    }
+    try {
+        const serialized = JSON.stringify(value);
+        return serialized ?? String(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function truncateForPrompt(value, maxLength) {
+    if (value.length <= maxLength) {
+        return value;
+    }
+    return `${value.slice(0, maxLength)}... [truncated]`;
+}
+function legacyEvaluationToResult(result, screenshotCount) {
+    const outcomeSuccess = result.evaluation === "YES";
+    const invalid = result.evaluation === "INVALID";
+    const findings = invalid
+        ? [
+            {
+                category: "verifier_uncertainty",
+                severity: "warning",
+                description: result.reasoning,
+            },
+        ]
+        : [];
+    return {
+        outcomeSuccess,
+        explanation: result.reasoning,
+        ...(findings.length ? { findings } : {}),
+        rawSteps: {
+            backend: "legacy",
+            legacyEvaluation: result.evaluation,
+            screenshotCount,
+        },
+    };
+}
+function legacyInsufficientEvidenceResult(reason) {
+    return {
+        outcomeSuccess: false,
+        explanation: reason,
+        findings: [
+            {
+                category: "trajectory_capture",
+                severity: "blocking",
+                description: reason,
+            },
+        ],
+        rawSteps: {
+            backend: "legacy",
+            legacyEvaluation: "INVALID",
+            screenshotCount: 0,
+        },
+    };
 }
 //# sourceMappingURL=v3Evaluator.js.map

package/dist/esm/lib/v3Evaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAExD,MAAM,OAAO,WAAW;IACd,EAAE,CAAK;IACP,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,YAAY,GAA+B,GAAG,EAAE,GAAE,CAAC,CAAC;IAE5D,YACE,EAAM,EACN,SAA0B,EAC1B,kBAAkC;QAElC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,IAAK,yBAA4C,CAAC;QAC5E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI;YAC9C,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,sEAAsE;QACtE,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,UAAU,GAAG,IAAI,EACjB,YAAY,EACZ,iBAAiB,GAAG,GAAG,EACvB,cAAc,GACf,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;YACxB,MAAM,IAAI,6BAA6B,CACrC,qDAAqD,CACtD,CAAC;QAEJ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,gCAAgC,CAAC;gBAC3C,QAAQ;gBACR,WAAW,EAAE,UAAU;gBACvB,YAAY;gBACZ,cAAc;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,mBAAmB,GAAG,kIAAkI,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,sDAAsD,8HAA8H,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC;QAElZ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE;oBAChE;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,6CAA6C,cAAc,EAAE;oCACpF,CAAC,CAAC,QAAQ;6BACb;4BACD,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,EACJ,SAAS,EACT,UAAU,GAAG,IAAI,EACjB,YAAY,GAAG,8EAA8E,EAC7F,iBAAiB,GAAG,GAAG,GACxB,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,SAAS,EAAE,MAAM;YACpB,MAAM,IAAI,6BAA6B,CACrC,iCAAiC,CAClC,CAAC;QAEJ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAG,SAAS;aACxB,GAAG,CACF,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF;aACA,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,GAAG,YAAY,2CAA2C,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE,6KAA6K;qBAChX;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;4BACjC,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,uBAAuB;oBAC7B,MAAM,EAAE,qBAAqB;iBAC9B;aACF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAExB,CAAC;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1B,UAAU,EAAE,SAAkB;gBAC9B,SAAS,EAAE,sCAAsC,YAAY,EAAE;aAChE,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAK9C;QACC,MAAM,EACJ,QAAQ,EACR,WAAW,EACX,cAAc,EACd,YAAY,GAAG;UACX,cAAc,CAAC,CAAC,CAAC,iGAAiG,CAAC,CAAC,CAAC,EAAE;;;UAGvH,cAAc,CAAC,CAAC,CAAC,kNAAkN,CAAC,CAAC,CAAC,EAAE;0BACxN,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,GACtD,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAC1C,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,CAC3C,CAAC;QAEJ,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,WAAoB;YAC1B,SAAS,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;SACrE,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;oBACzC;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,2DAA2D,cAAc,qBAAqB,WAAW,CAAC,MAAM,sKAAsK;oCAC7S,CAAC,CAAC,GAAG,QAAQ,qBAAqB,WAAW,CAAC,MAAM,mIAAmI;6BAC1L;4BACD,GAAG,aAAa;yBACjB;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;CACF","sourcesContent":["/**\n * V3Evaluator mirrors Evaluator but operates on a V3 instance instead of Stagehand.\n * It uses the V3 page/screenshot APIs and constructs an LLM client to run\n * structured evaluations (YES/NO with reasoning) on screenshots and/or text.\n */\n\nimport { z } from \"zod\";\nimport type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { LLMParsedResponse } from \"./inference.js\";\nimport { LLMResponse, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\nconst EvaluationSchema = z.object({\n  evaluation: z.enum([\"YES\", \"NO\"]),\n  reasoning: z.string(),\n});\n\nconst BatchEvaluationSchema = z.array(EvaluationSchema);\n\nexport class V3Evaluator {\n  private v3: V3;\n  private modelName: AvailableModel;\n  private modelClientOptions: ClientOptions | { apiKey: string };\n  private silentLogger: (message: LogLine) => void = () => {};\n\n  constructor(\n    v3: V3,\n    modelName?: AvailableModel,\n    modelClientOptions?: ClientOptions,\n  ) {\n    this.v3 = v3;\n    this.modelName = modelName || (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n  }\n\n  private getClient(): LLMClient {\n    // Prefer a dedicated provider so we can override model per-evaluation\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshot = true,\n      systemPrompt,\n      screenshotDelayMs = 250,\n      agentReasoning,\n    } = options;\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!answer && !screenshot)\n      throw new StagehandInvalidArgumentError(\n        \"Either answer (text) or screenshot must be provided\",\n      );\n\n    if (Array.isArray(screenshot)) {\n      return this._evaluateWithMultipleScreenshots({\n        question,\n        screenshots: screenshot,\n        systemPrompt,\n        agentReasoning,\n      });\n    }\n\n    const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? \"a screenshot\" : \"the agents reasoning and actions throughout the task\"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\\n          Today's date is ${new Date().toLocaleDateString()}`;\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt || defaultSystemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions taken:\\n${agentReasoning}`\n                  : question,\n              },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    const {\n      questions,\n      screenshot = true,\n      systemPrompt = \"You are an expert evaluator that returns YES or NO with a concise reasoning.\",\n      screenshotDelayMs = 250,\n    } = options;\n    if (!questions?.length)\n      throw new StagehandInvalidArgumentError(\n        \"Questions array cannot be empty\",\n      );\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const formatted = questions\n      .map(\n        (item, i) =>\n          `${i + 1}. ${item.question}${item.answer ? `\\n   Answer: ${item.answer}` : \"\"}`,\n      )\n      .join(\"\\n\\n\");\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          {\n            role: \"system\",\n            content: `${systemPrompt}\\n\\nYou will be given multiple questions${screenshot ? \" with a screenshot\" : \"\"}. ${questions.some((q) => q.answer) ? \"Some questions include answers to evaluate.\" : \"\"} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,\n          },\n          {\n            role: \"user\",\n            content: [\n              { type: \"text\", text: formatted },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n            ],\n          },\n        ],\n        response_model: {\n          name: \"BatchEvaluationResult\",\n          schema: BatchEvaluationSchema,\n        },\n      },\n    });\n\n    try {\n      const results = response.data as unknown as z.infer<\n        typeof BatchEvaluationSchema\n      >;\n      return results.map((r) => ({\n        evaluation: r.evaluation,\n        reasoning: r.reasoning,\n      }));\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return questions.map(() => ({\n        evaluation: \"INVALID\" as const,\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      }));\n    }\n  }\n\n  private async _evaluateWithMultipleScreenshots(options: {\n    question: string;\n    screenshots: Buffer[];\n    systemPrompt?: string;\n    agentReasoning?: string;\n  }): Promise<EvaluationResult> {\n    const {\n      question,\n      screenshots,\n      agentReasoning,\n      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.\n        ${agentReasoning ? \"You also have access to the agent's detailed reasoning and thought process throughout the task.\" : \"\"}\n        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.\n        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).\n        ${agentReasoning ? \"The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation.\" : \"\"}\n        Today's date is ${new Date().toLocaleDateString()}`,\n    } = options;\n\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!screenshots || screenshots.length === 0)\n      throw new StagehandInvalidArgumentError(\n        \"At least one screenshot must be provided\",\n      );\n\n    const llmClient = this.getClient();\n\n    const imageContents = screenshots.map((s) => ({\n      type: \"image_url\" as const,\n      image_url: { url: `data:image/jpeg;base64,${s.toString(\"base64\")}` },\n    }));\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions throughout the task:\\n${agentReasoning}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`\n                  : `${question}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,\n              },\n              ...imageContents,\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n}\n"]}
+{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAW3D,MAAM,qBAAqB,GAAG,6BAA6B,CAAC;AAC5D,MAAM,yBAAyB,GAAuB,QAAQ,CAAC;AA2B/D,MAAM,OAAO,WAAW;IACL,OAAO,CAAqB;IAC5B,eAAe,CAAoB;IAEpD,YACE,EAAM,EACN,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;QAE5B,MAAM,iBAAiB,GAAG,2BAA2B,CACnD,kBAAkB,EAClB,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAC1C,EAAE,EACF,iBAAiB,CAAC,SAAS,EAC3B,iBAAiB,CAAC,kBAAkB,CACrC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAsB;QACjC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,mCAAmC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAED,OAAO,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,6BAA6B,CACrC,+CAA+C,CAChD,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO;YACL,KAAK,EAAE,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC;SACjD,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAEO,0BAA0B,CAAC,UAAkB;QACnD,MAAM,IAAI,6BAA6B,CACrC,eAAe,UAAU,0BAA0B,qBAAqB,sHAAsH,CAC/L,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mCAAmC,CAC/C,UAAsB,EACtB,QAAkB;QAElB,MAAM,WAAW,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC;QAEtC,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,OAAO,gCAAgC,CACrC,qFAAqF,CACtF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;YAC5C,QAAQ,EAAE,QAAQ,CAAC,WAAW;YAC9B,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK;YACpD,MAAM;YACN,cAAc;SACf,CAAC,CAAC;QAEH,OAAO,wBAAwB,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9D,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;IAE5B,IACE,kBAAkB;QAClB,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,CAAC;QACD,OAAO;YACL,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,kBAAkB,EAAE,kBAAkB,CAAC,kBAAkB;YACzD,OAAO,EAAE,kBAAkB,CAAC,OAAO,IAAI,OAAO,EAAE,OAAO;SACxD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAgD;QAC3D,kBAAkB;QAClB,OAAO,EAAE,OAAO,EAAE,OAAO;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,eAAoC;IAEpC,MAAM,iBAAiB,GACrB,eAAe;QACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;QAClC,yBAAyB,CAAC;IAC5B,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEjE,IAAI,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,KAAK,UAAU,EAAE,CAAC;QACvE,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,6BAA6B,CACrC,WAAW,qBAAqB,KAAK,iBAAiB,qCAAqC,CAC5F,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAsB;IACjD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,6BAA6B,CACrC,yCAAyC,CAC1C,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;QACzB,MAAM,IAAI,6BAA6B,CACrC,iDAAiD,CAClD,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC;AACzB,CAAC;AAED,SAAS,6BAA6B,CAAC,QAAkB;IACvD,OAAO;QACL,SAAS,EAAE,wBAAwB;QACnC,WAAW,EAAE,yDAAyD,QAAQ,CAAC,WAAW,EAAE;QAC5F,SAAS,EAAE,CAAC;KACb,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAsB;IACtD,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,CAAC;YACpD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,IAAI,CACrD,CACE,QAAQ,EACuD,EAAE,CACjE,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC/D,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAAE,CAAC;QAC7D,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,0BAA0B,CACjC,UAAsB;IAEtB,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK;YACnC,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE;YACxC,CAAC,CAAC,gBAAgB,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;QAClE,OAAO;YACL,QAAQ,CAAC,KAAK,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;YAC3D,MAAM,IAAI,SAAS;YACnB,MAAM;SACP;aACE,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,iBAAiB,CACtB,sBAAsB,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAC9C,KAAK,CACN,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAE,SAAiB;IACzD,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;AACvD,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAA8B,EAC9B,eAAuB;IAEvB,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC;IAChD,MAAM,QAAQ,GAAsB,OAAO;QACzC,CAAC,CAAC;YACE;gBACE,QAAQ,EAAE,sBAAsB;gBAChC,QAAQ,EAAE,SAAS;gBACnB,WAAW,EAAE,MAAM,CAAC,SAAS;aAC9B;SACF;QACH,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;QACL,cAAc;QACd,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,MAAM,CAAC,UAAU;YACnC,eAAe;SAChB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,gCAAgC,CAAC,MAAc;IACtD,OAAO;QACL,cAAc,EAAE,KAAK;QACrB,WAAW,EAAE,MAAM;QACnB,QAAQ,EAAE;YACR;gBACE,QAAQ,EAAE,oBAAoB;gBAC9B,QAAQ,EAAE,UAAU;gBACpB,WAAW,EAAE,MAAM;aACpB;SACF;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,SAAS;YAC3B,eAAe,EAAE,CAAC;SACnB;KACF,CAAC;AACJ,CAAC","sourcesContent":["import type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult as LegacyEvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\nimport { LegacyV3Evaluator } from \"./v3LegacyEvaluator.js\";\nimport type {\n  Trajectory,\n  TaskSpec,\n  EvaluationResult,\n  Rubric,\n  Verifier,\n  AgentEvidenceModality,\n  VerifierFinding,\n} from \"./v3/verifier/index.js\";\n\nconst EVALUATOR_BACKEND_ENV = \"STAGEHAND_EVALUATOR_BACKEND\";\nconst DEFAULT_EVALUATOR_BACKEND: V3EvaluatorBackend = \"legacy\";\n\nexport type V3EvaluatorBackend = \"legacy\" | \"verifier\";\n\nexport type V3EvaluatorOptions = {\n  /**\n   * Selects the evaluator implementation.\n   *\n   * \"legacy\" preserves the existing screenshot/text YES/NO evaluator.\n   * \"verifier\" is reserved for the rubric verifier backend.\n   *\n   * @default process.env.STAGEHAND_EVALUATOR_BACKEND || \"legacy\"\n   */\n  backend?: V3EvaluatorBackend;\n};\n\nexport type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n};\n\ntype NormalizedConstructorOptions = {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n  backend?: V3EvaluatorBackend;\n};\n\nexport class V3Evaluator implements Verifier {\n  private readonly backend: V3EvaluatorBackend;\n  private readonly legacyEvaluator: LegacyV3Evaluator;\n\n  constructor(\n    v3: V3,\n    modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n    modelClientOptions?: ClientOptions,\n    options?: V3EvaluatorOptions,\n  ) {\n    const normalizedOptions = normalizeConstructorOptions(\n      modelNameOrOptions,\n      modelClientOptions,\n      options,\n    );\n\n    this.backend = resolveEvaluatorBackend(normalizedOptions.backend);\n    this.legacyEvaluator = new LegacyV3Evaluator(\n      v3,\n      normalizedOptions.modelName,\n      normalizedOptions.modelClientOptions,\n    );\n  }\n\n  async ask(options: EvaluateOptions): Promise<LegacyEvaluationResult> {\n    return this.getLegacyBackend(\"ask\").ask(options);\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<LegacyEvaluationResult[]> {\n    return this.getLegacyBackend(\"batchAsk\").batchAsk(options);\n  }\n\n  async verify(trajectory: Trajectory): Promise<EvaluationResult> {\n    const taskSpec = assertVerifierInput(trajectory);\n\n    if (this.backend === \"legacy\") {\n      return this.verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec);\n    }\n\n    return this.unavailableVerifierBackend(\"verify\");\n  }\n\n  async generateRubric(taskSpec: TaskSpec): Promise<Rubric> {\n    if (!taskSpec?.id) {\n      throw new StagehandInvalidArgumentError(\n        \"TaskSpec.id is required for rubric generation\",\n      );\n    }\n\n    if (this.backend === \"verifier\") {\n      return this.unavailableVerifierBackend(\"generateRubric\");\n    }\n\n    return {\n      items: [legacyTaskCompletionCriterion(taskSpec)],\n    };\n  }\n\n  private getLegacyBackend(methodName: string): LegacyV3Evaluator {\n    if (this.backend === \"legacy\") {\n      return this.legacyEvaluator;\n    }\n\n    return this.unavailableVerifierBackend(methodName);\n  }\n\n  private unavailableVerifierBackend(methodName: string): never {\n    throw new StagehandInvalidArgumentError(\n      `V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use \"legacy\" or install the verifier backend PR.`,\n    );\n  }\n\n  private async verifyTrajectoryWithLegacyEvaluator(\n    trajectory: Trajectory,\n    taskSpec: TaskSpec,\n  ): Promise<EvaluationResult> {\n    const screenshots = collectLegacyScreenshots(trajectory);\n    const agentReasoning = renderLegacyAgentReasoning(trajectory);\n    const answer = trajectory.finalAnswer;\n\n    if (!screenshots.length && !answer) {\n      return legacyInsufficientEvidenceResult(\n        \"Legacy evaluator compatibility mode had no screenshots or final answer to evaluate.\",\n      );\n    }\n\n    const result = await this.legacyEvaluator.ask({\n      question: taskSpec.instruction,\n      screenshot: screenshots.length ? screenshots : false,\n      answer,\n      agentReasoning,\n    });\n\n    return legacyEvaluationToResult(result, screenshots.length);\n  }\n}\n\nfunction normalizeConstructorOptions(\n  modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n  modelClientOptions?: ClientOptions,\n  options?: V3EvaluatorOptions,\n): NormalizedConstructorOptions {\n  if (\n    modelNameOrOptions &&\n    typeof modelNameOrOptions === \"object\" &&\n    !Array.isArray(modelNameOrOptions)\n  ) {\n    return {\n      modelName: modelNameOrOptions.modelName,\n      modelClientOptions: modelNameOrOptions.modelClientOptions,\n      backend: modelNameOrOptions.backend ?? options?.backend,\n    };\n  }\n\n  return {\n    modelName: modelNameOrOptions as AvailableModel | undefined,\n    modelClientOptions,\n    backend: options?.backend,\n  };\n}\n\nfunction resolveEvaluatorBackend(\n  explicitBackend?: V3EvaluatorBackend,\n): V3EvaluatorBackend {\n  const configuredBackend =\n    explicitBackend ??\n    process.env[EVALUATOR_BACKEND_ENV] ??\n    DEFAULT_EVALUATOR_BACKEND;\n  const normalizedBackend = configuredBackend.trim().toLowerCase();\n\n  if (normalizedBackend === \"legacy\" || normalizedBackend === \"verifier\") {\n    return normalizedBackend;\n  }\n\n  throw new StagehandInvalidArgumentError(\n    `Invalid ${EVALUATOR_BACKEND_ENV}=\"${configuredBackend}\". Expected \"legacy\" or \"verifier\".`,\n  );\n}\n\nfunction assertVerifierInput(trajectory: Trajectory): TaskSpec {\n  if (!trajectory) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory is required for verification\",\n    );\n  }\n  if (!trajectory.task?.id) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory.task.id is required for verification\",\n    );\n  }\n  return trajectory.task;\n}\n\nfunction legacyTaskCompletionCriterion(taskSpec: TaskSpec) {\n  return {\n    criterion: \"legacy-task-completion\",\n    description: `Evaluate whether the task was completed successfully: ${taskSpec.instruction}`,\n    maxPoints: 1,\n  };\n}\n\nfunction collectLegacyScreenshots(trajectory: Trajectory): Buffer[] {\n  const screenshots: Buffer[] = [];\n\n  for (const step of trajectory.steps ?? []) {\n    if (Buffer.isBuffer(step.probeEvidence?.screenshot)) {\n      screenshots.push(step.probeEvidence.screenshot);\n      continue;\n    }\n\n    const agentImage = step.agentEvidence?.modalities?.find(\n      (\n        modality,\n      ): modality is Extract<AgentEvidenceModality, { type: \"image\" }> =>\n        modality.type === \"image\" && Buffer.isBuffer(modality.bytes),\n    );\n\n    if (agentImage) {\n      screenshots.push(agentImage.bytes);\n    }\n  }\n\n  if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {\n    screenshots.push(trajectory.finalObservation.screenshot);\n  }\n\n  return screenshots;\n}\n\nfunction renderLegacyAgentReasoning(\n  trajectory: Trajectory,\n): string | undefined {\n  const stepLines = (trajectory.steps ?? []).map((step, i) => {\n    const status = step.toolOutput?.ok === false ? \"Tool status: failed\" : \"\";\n    const output = step.toolOutput?.error\n      ? `Tool error: ${step.toolOutput.error}`\n      : `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;\n    return [\n      `Step ${i}: ${step.actionName}`,\n      step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,\n      status || undefined,\n      output,\n    ]\n      .filter(Boolean)\n      .join(\"\\n\");\n  });\n\n  if (!stepLines.length) {\n    return undefined;\n  }\n\n  return truncateForPrompt(\n    `Agent trajectory:\\n${stepLines.join(\"\\n\\n\")}`,\n    16000,\n  );\n}\n\nfunction stringifyForPrompt(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n\n  try {\n    const serialized = JSON.stringify(value);\n    return serialized ?? String(value);\n  } catch {\n    return String(value);\n  }\n}\n\nfunction truncateForPrompt(value: string, maxLength: number): string {\n  if (value.length <= maxLength) {\n    return value;\n  }\n\n  return `${value.slice(0, maxLength)}... [truncated]`;\n}\n\nfunction legacyEvaluationToResult(\n  result: LegacyEvaluationResult,\n  screenshotCount: number,\n): EvaluationResult {\n  const outcomeSuccess = result.evaluation === \"YES\";\n  const invalid = result.evaluation === \"INVALID\";\n  const findings: VerifierFinding[] = invalid\n    ? [\n        {\n          category: \"verifier_uncertainty\",\n          severity: \"warning\",\n          description: result.reasoning,\n        },\n      ]\n    : [];\n\n  return {\n    outcomeSuccess,\n    explanation: result.reasoning,\n    ...(findings.length ? { findings } : {}),\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: result.evaluation,\n      screenshotCount,\n    },\n  };\n}\n\nfunction legacyInsufficientEvidenceResult(reason: string): EvaluationResult {\n  return {\n    outcomeSuccess: false,\n    explanation: reason,\n    findings: [\n      {\n        category: \"trajectory_capture\",\n        severity: \"blocking\",\n        description: reason,\n      },\n    ],\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: \"INVALID\",\n      screenshotCount: 0,\n    },\n  };\n}\n"]}

package/dist/esm/lib/v3LegacyEvaluator.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Legacy V3 evaluator implementation.
+ *
+ * This is the behavior-preserving implementation that backs V3Evaluator when
+ * STAGEHAND_EVALUATOR_BACKEND=legacy.
+ */
+import type { AvailableModel, ClientOptions } from "./v3/types/public/model.js";
+import type { EvaluateOptions, BatchAskOptions, EvaluationResult } from "./v3/types/private/evaluator.js";
+import { V3 } from "./v3/v3.js";
+export declare class LegacyV3Evaluator {
+    private v3;
+    private modelName;
+    private modelClientOptions;
+    private silentLogger;
+    constructor(v3: V3, modelName?: AvailableModel, modelClientOptions?: ClientOptions);
+    private getClient;
+    ask(options: EvaluateOptions): Promise<EvaluationResult>;
+    batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]>;
+    private _evaluateWithMultipleScreenshots;
+}