@browserbasehq/orca 3.5.0-vertex-test → 3.6.0-gemini-cua

package/dist/esm/lib/v3Evaluator.js

@@ -1,13 +1,27 @@
+import { LLMProvider } from "./v3/llm/LLMProvider.js";
 import { StagehandInvalidArgumentError } from "./v3/types/public/sdkErrors.js";
 import { LegacyV3Evaluator } from "./v3LegacyEvaluator.js";
+import { RubricVerifier } from "./v3/verifier/rubricVerifier.js";
 const EVALUATOR_BACKEND_ENV = "STAGEHAND_EVALUATOR_BACKEND";
 const DEFAULT_EVALUATOR_BACKEND = "legacy";
 export class V3Evaluator {
+    v3;
     backend;
+    modelName;
+    modelClientOptions;
     legacyEvaluator;
     constructor(v3, modelNameOrOptions, modelClientOptions, options) {
         const normalizedOptions = normalizeConstructorOptions(modelNameOrOptions, modelClientOptions, options);
         this.backend = resolveEvaluatorBackend(normalizedOptions.backend);
+        this.v3 = v3;
+        this.modelName =
+            normalizedOptions.modelName ||
+                "google/gemini-2.5-flash";
+        this.modelClientOptions = normalizedOptions.modelClientOptions || {
+            apiKey: process.env.GEMINI_API_KEY ||
+                process.env.GOOGLE_GENERATIVE_AI_API_KEY ||
+                "",
+        };
         this.legacyEvaluator = new LegacyV3Evaluator(v3, normalizedOptions.modelName, normalizedOptions.modelClientOptions);
     }
     async ask(options) {
@@ -16,11 +30,73 @@ export class V3Evaluator {
     async batchAsk(options) {
         return this.getLegacyBackend("batchAsk").batchAsk(options);
     }
+    async verify(trajectory) {
+        const taskSpec = assertVerifierInput(trajectory);
+        if (this.backend === "legacy") {
+            return this.verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec);
+        }
+        const verifier = new RubricVerifier({
+            getClient: () => this.getClient(),
+            getRubricGenClient: () => this.getRubricGenClient(),
+        });
+        return verifier.verify(trajectory);
+    }
+    async generateRubric(taskSpec) {
+        if (!taskSpec?.id) {
+            throw new StagehandInvalidArgumentError("TaskSpec.id is required for rubric generation");
+        }
+        if (this.backend === "legacy") {
+            return {
+                items: [legacyTaskCompletionCriterion(taskSpec)],
+            };
+        }
+        const verifier = new RubricVerifier({
+            getClient: () => this.getClient(),
+            getRubricGenClient: () => this.getRubricGenClient(),
+        });
+        return verifier.generateRubric(taskSpec);
+    }
     getLegacyBackend(methodName) {
         if (this.backend === "legacy") {
             return this.legacyEvaluator;
         }
-        throw new StagehandInvalidArgumentError(`V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use "legacy" or install the verifier backend PR.`);
+        return this.unavailableVerifierBackend(methodName);
+    }
+    unavailableVerifierBackend(methodName) {
+        throw new StagehandInvalidArgumentError(`V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend only supports verify() and generateRubric(). Use "legacy" for ask()/batchAsk().`);
+    }
+    getClient() {
+        const provider = new LLMProvider(this.v3.logger);
+        return provider.getClient(this.modelName, this.modelClientOptions);
+    }
+    getRubricGenClient() {
+        const override = process.env.VERIFIER_RUBRIC_MODEL;
+        if (!override)
+            return this.getClient();
+        const provider = new LLMProvider(this.v3.logger);
+        const overrideProvider = override.includes("/")
+            ? override.slice(0, override.indexOf("/"))
+            : undefined;
+        const defaultProvider = this.modelName.includes("/")
+            ? this.modelName.slice(0, this.modelName.indexOf("/"))
+            : undefined;
+        const sameProvider = overrideProvider !== undefined && overrideProvider === defaultProvider;
+        return provider.getClient(override, sameProvider ? this.modelClientOptions : undefined);
+    }
+    async verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec) {
+        const screenshots = collectLegacyScreenshots(trajectory);
+        const agentReasoning = renderLegacyAgentReasoning(trajectory);
+        const answer = trajectory.finalAnswer;
+        if (!screenshots.length && !answer) {
+            return legacyInsufficientEvidenceResult("Legacy evaluator compatibility mode had no screenshots or final answer to evaluate.");
+        }
+        const result = await this.legacyEvaluator.ask({
+            question: taskSpec.instruction,
+            screenshot: screenshots.length ? screenshots : false,
+            answer,
+            agentReasoning,
+        });
+        return legacyEvaluationToResult(result, screenshots.length);
     }
 }
 function normalizeConstructorOptions(modelNameOrOptions, modelClientOptions, options) {
@@ -49,4 +125,116 @@ function resolveEvaluatorBackend(explicitBackend) {
     }
     throw new StagehandInvalidArgumentError(`Invalid ${EVALUATOR_BACKEND_ENV}="${configuredBackend}". Expected "legacy" or "verifier".`);
 }
+function assertVerifierInput(trajectory) {
+    if (!trajectory) {
+        throw new StagehandInvalidArgumentError("Trajectory is required for verification");
+    }
+    if (!trajectory.task?.id) {
+        throw new StagehandInvalidArgumentError("Trajectory.task.id is required for verification");
+    }
+    return trajectory.task;
+}
+function legacyTaskCompletionCriterion(taskSpec) {
+    return {
+        criterion: "legacy-task-completion",
+        description: `Evaluate whether the task was completed successfully: ${taskSpec.instruction}`,
+        maxPoints: 1,
+    };
+}
+function collectLegacyScreenshots(trajectory) {
+    const screenshots = [];
+    for (const step of trajectory.steps ?? []) {
+        if (Buffer.isBuffer(step.probeEvidence?.screenshot)) {
+            screenshots.push(step.probeEvidence.screenshot);
+            continue;
+        }
+        const agentImage = step.agentEvidence?.modalities?.find((modality) => modality.type === "image" && Buffer.isBuffer(modality.bytes));
+        if (agentImage) {
+            screenshots.push(agentImage.bytes);
+        }
+    }
+    if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {
+        screenshots.push(trajectory.finalObservation.screenshot);
+    }
+    return screenshots;
+}
+function renderLegacyAgentReasoning(trajectory) {
+    const stepLines = (trajectory.steps ?? []).map((step, i) => {
+        const status = step.toolOutput?.ok === false ? "Tool status: failed" : "";
+        const output = step.toolOutput?.error
+            ? `Tool error: ${step.toolOutput.error}`
+            : `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;
+        return [
+            `Step ${i}: ${step.actionName}`,
+            step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,
+            status || undefined,
+            output,
+        ]
+            .filter(Boolean)
+            .join("\n");
+    });
+    if (!stepLines.length) {
+        return undefined;
+    }
+    return truncateForPrompt(`Agent trajectory:\n${stepLines.join("\n\n")}`, 16000);
+}
+function stringifyForPrompt(value) {
+    if (typeof value === "string") {
+        return value;
+    }
+    try {
+        const serialized = JSON.stringify(value);
+        return serialized ?? String(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function truncateForPrompt(value, maxLength) {
+    if (value.length <= maxLength) {
+        return value;
+    }
+    return `${value.slice(0, maxLength)}... [truncated]`;
+}
+function legacyEvaluationToResult(result, screenshotCount) {
+    const outcomeSuccess = result.evaluation === "YES";
+    const invalid = result.evaluation === "INVALID";
+    const findings = invalid
+        ? [
+            {
+                category: "verifier_uncertainty",
+                severity: "warning",
+                description: result.reasoning,
+            },
+        ]
+        : [];
+    return {
+        outcomeSuccess,
+        explanation: result.reasoning,
+        ...(findings.length ? { findings } : {}),
+        rawSteps: {
+            backend: "legacy",
+            legacyEvaluation: result.evaluation,
+            screenshotCount,
+        },
+    };
+}
+function legacyInsufficientEvidenceResult(reason) {
+    return {
+        outcomeSuccess: false,
+        explanation: reason,
+        findings: [
+            {
+                category: "trajectory_capture",
+                severity: "blocking",
+                description: reason,
+            },
+        ],
+        rawSteps: {
+            backend: "legacy",
+            legacyEvaluation: "INVALID",
+            screenshotCount: 0,
+        },
+    };
+}
 //# sourceMappingURL=v3Evaluator.js.map

package/dist/esm/lib/v3Evaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,qBAAqB,GAAG,6BAA6B,CAAC;AAC5D,MAAM,yBAAyB,GAAuB,QAAQ,CAAC;AA2B/D,MAAM,OAAO,WAAW;IACL,OAAO,CAAqB;IAC5B,eAAe,CAAoB;IAEpD,YACE,EAAM,EACN,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;QAE5B,MAAM,iBAAiB,GAAG,2BAA2B,CACnD,kBAAkB,EAClB,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAC1C,EAAE,EACF,iBAAiB,CAAC,SAAS,EAC3B,iBAAiB,CAAC,kBAAkB,CACrC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,MAAM,IAAI,6BAA6B,CACrC,eAAe,UAAU,0BAA0B,qBAAqB,sHAAsH,CAC/L,CAAC;IACJ,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;IAE5B,IACE,kBAAkB;QAClB,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,CAAC;QACD,OAAO;YACL,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,kBAAkB,EAAE,kBAAkB,CAAC,kBAAkB;YACzD,OAAO,EAAE,kBAAkB,CAAC,OAAO,IAAI,OAAO,EAAE,OAAO;SACxD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAgD;QAC3D,kBAAkB;QAClB,OAAO,EAAE,OAAO,EAAE,OAAO;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,eAAoC;IAEpC,MAAM,iBAAiB,GACrB,eAAe;QACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;QAClC,yBAAyB,CAAC;IAC5B,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEjE,IAAI,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,KAAK,UAAU,EAAE,CAAC;QACvE,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,6BAA6B,CACrC,WAAW,qBAAqB,KAAK,iBAAiB,qCAAqC,CAC5F,CAAC;AACJ,CAAC","sourcesContent":["import type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\nimport { LegacyV3Evaluator } from \"./v3LegacyEvaluator.js\";\n\nconst EVALUATOR_BACKEND_ENV = \"STAGEHAND_EVALUATOR_BACKEND\";\nconst DEFAULT_EVALUATOR_BACKEND: V3EvaluatorBackend = \"legacy\";\n\nexport type V3EvaluatorBackend = \"legacy\" | \"verifier\";\n\nexport type V3EvaluatorOptions = {\n  /**\n   * Selects the evaluator implementation.\n   *\n   * \"legacy\" preserves the existing screenshot/text YES/NO evaluator.\n   * \"verifier\" is reserved for the rubric verifier backend.\n   *\n   * @default process.env.STAGEHAND_EVALUATOR_BACKEND || \"legacy\"\n   */\n  backend?: V3EvaluatorBackend;\n};\n\nexport type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n};\n\ntype NormalizedConstructorOptions = {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n  backend?: V3EvaluatorBackend;\n};\n\nexport class V3Evaluator {\n  private readonly backend: V3EvaluatorBackend;\n  private readonly legacyEvaluator: LegacyV3Evaluator;\n\n  constructor(\n    v3: V3,\n    modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n    modelClientOptions?: ClientOptions,\n    options?: V3EvaluatorOptions,\n  ) {\n    const normalizedOptions = normalizeConstructorOptions(\n      modelNameOrOptions,\n      modelClientOptions,\n      options,\n    );\n\n    this.backend = resolveEvaluatorBackend(normalizedOptions.backend);\n    this.legacyEvaluator = new LegacyV3Evaluator(\n      v3,\n      normalizedOptions.modelName,\n      normalizedOptions.modelClientOptions,\n    );\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    return this.getLegacyBackend(\"ask\").ask(options);\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    return this.getLegacyBackend(\"batchAsk\").batchAsk(options);\n  }\n\n  private getLegacyBackend(methodName: string): LegacyV3Evaluator {\n    if (this.backend === \"legacy\") {\n      return this.legacyEvaluator;\n    }\n\n    throw new StagehandInvalidArgumentError(\n      `V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use \"legacy\" or install the verifier backend PR.`,\n    );\n  }\n}\n\nfunction normalizeConstructorOptions(\n  modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n  modelClientOptions?: ClientOptions,\n  options?: V3EvaluatorOptions,\n): NormalizedConstructorOptions {\n  if (\n    modelNameOrOptions &&\n    typeof modelNameOrOptions === \"object\" &&\n    !Array.isArray(modelNameOrOptions)\n  ) {\n    return {\n      modelName: modelNameOrOptions.modelName,\n      modelClientOptions: modelNameOrOptions.modelClientOptions,\n      backend: modelNameOrOptions.backend ?? options?.backend,\n    };\n  }\n\n  return {\n    modelName: modelNameOrOptions as AvailableModel | undefined,\n    modelClientOptions,\n    backend: options?.backend,\n  };\n}\n\nfunction resolveEvaluatorBackend(\n  explicitBackend?: V3EvaluatorBackend,\n): V3EvaluatorBackend {\n  const configuredBackend =\n    explicitBackend ??\n    process.env[EVALUATOR_BACKEND_ENV] ??\n    DEFAULT_EVALUATOR_BACKEND;\n  const normalizedBackend = configuredBackend.trim().toLowerCase();\n\n  if (normalizedBackend === \"legacy\" || normalizedBackend === \"verifier\") {\n    return normalizedBackend;\n  }\n\n  throw new StagehandInvalidArgumentError(\n    `Invalid ${EVALUATOR_BACKEND_ENV}=\"${configuredBackend}\". Expected \"legacy\" or \"verifier\".`,\n  );\n}\n"]}
+{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAWjE,MAAM,qBAAqB,GAAG,6BAA6B,CAAC;AAC5D,MAAM,yBAAyB,GAAuB,QAAQ,CAAC;AA2B/D,MAAM,OAAO,WAAW;IACL,EAAE,CAAK;IACP,OAAO,CAAqB;IAC5B,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,eAAe,CAAoB;IAEpD,YACE,EAAM,EACN,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;QAE5B,MAAM,iBAAiB,GAAG,2BAA2B,CACnD,kBAAkB,EAClB,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS;YACZ,iBAAiB,CAAC,SAAS;gBAC1B,yBAA4C,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC,kBAAkB,IAAI;YAChE,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;QACF,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAC1C,EAAE,EACF,iBAAiB,CAAC,SAAS,EAC3B,iBAAiB,CAAC,kBAAkB,CACrC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAsB;QACjC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,mCAAmC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE;YACjC,kBAAkB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE;SACpD,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,6BAA6B,CACrC,+CAA+C,CAChD,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO;gBACL,KAAK,EAAE,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE;YACjC,kBAAkB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE;SACpD,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAEO,0BAA0B,CAAC,UAAkB;QACnD,MAAM,IAAI,6BAA6B,CACrC,eAAe,UAAU,0BAA0B,qBAAqB,qHAAqH,CAC9L,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACxB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,qBAEhB,CAAC;QACd,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;YAC7C,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC1C,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACtD,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,YAAY,GAChB,gBAAgB,KAAK,SAAS,IAAI,gBAAgB,KAAK,eAAe,CAAC;QAEzE,OAAO,QAAQ,CAAC,SAAS,CACvB,QAAQ,EACR,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CACnD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mCAAmC,CAC/C,UAAsB,EACtB,QAAkB;QAElB,MAAM,WAAW,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC;QAEtC,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,OAAO,gCAAgC,CACrC,qFAAqF,CACtF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;YAC5C,QAAQ,EAAE,QAAQ,CAAC,WAAW;YAC9B,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK;YACpD,MAAM;YACN,cAAc;SACf,CAAC,CAAC;QAEH,OAAO,wBAAwB,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9D,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;IAE5B,IACE,kBAAkB;QAClB,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,CAAC;QACD,OAAO;YACL,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,kBAAkB,EAAE,kBAAkB,CAAC,kBAAkB;YACzD,OAAO,EAAE,kBAAkB,CAAC,OAAO,IAAI,OAAO,EAAE,OAAO;SACxD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAgD;QAC3D,kBAAkB;QAClB,OAAO,EAAE,OAAO,EAAE,OAAO;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,eAAoC;IAEpC,MAAM,iBAAiB,GACrB,eAAe;QACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;QAClC,yBAAyB,CAAC;IAC5B,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEjE,IAAI,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,KAAK,UAAU,EAAE,CAAC;QACvE,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,6BAA6B,CACrC,WAAW,qBAAqB,KAAK,iBAAiB,qCAAqC,CAC5F,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAsB;IACjD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,6BAA6B,CACrC,yCAAyC,CAC1C,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;QACzB,MAAM,IAAI,6BAA6B,CACrC,iDAAiD,CAClD,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC;AACzB,CAAC;AAED,SAAS,6BAA6B,CAAC,QAAkB;IACvD,OAAO;QACL,SAAS,EAAE,wBAAwB;QACnC,WAAW,EAAE,yDAAyD,QAAQ,CAAC,WAAW,EAAE;QAC5F,SAAS,EAAE,CAAC;KACb,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAsB;IACtD,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,CAAC;YACpD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,IAAI,CACrD,CACE,QAAQ,EACuD,EAAE,CACjE,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC/D,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAAE,CAAC;QAC7D,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,0BAA0B,CACjC,UAAsB;IAEtB,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK;YACnC,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE;YACxC,CAAC,CAAC,gBAAgB,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;QAClE,OAAO;YACL,QAAQ,CAAC,KAAK,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;YAC3D,MAAM,IAAI,SAAS;YACnB,MAAM;SACP;aACE,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,iBAAiB,CACtB,sBAAsB,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAC9C,KAAK,CACN,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAE,SAAiB;IACzD,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;AACvD,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAA8B,EAC9B,eAAuB;IAEvB,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC;IAChD,MAAM,QAAQ,GAAsB,OAAO;QACzC,CAAC,CAAC;YACE;gBACE,QAAQ,EAAE,sBAAsB;gBAChC,QAAQ,EAAE,SAAS;gBACnB,WAAW,EAAE,MAAM,CAAC,SAAS;aAC9B;SACF;QACH,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;QACL,cAAc;QACd,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,MAAM,CAAC,UAAU;YACnC,eAAe;SAChB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,gCAAgC,CAAC,MAAc;IACtD,OAAO;QACL,cAAc,EAAE,KAAK;QACrB,WAAW,EAAE,MAAM;QACnB,QAAQ,EAAE;YACR;gBACE,QAAQ,EAAE,oBAAoB;gBAC9B,QAAQ,EAAE,UAAU;gBACpB,WAAW,EAAE,MAAM;aACpB;SACF;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,SAAS;YAC3B,eAAe,EAAE,CAAC;SACnB;KACF,CAAC;AACJ,CAAC","sourcesContent":["import type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult as LegacyEvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport type { LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\nimport { LegacyV3Evaluator } from \"./v3LegacyEvaluator.js\";\nimport { RubricVerifier } from \"./v3/verifier/rubricVerifier.js\";\nimport type {\n  Trajectory,\n  TaskSpec,\n  EvaluationResult,\n  Rubric,\n  Verifier,\n  AgentEvidenceModality,\n  VerifierFinding,\n} from \"./v3/verifier/index.js\";\n\nconst EVALUATOR_BACKEND_ENV = \"STAGEHAND_EVALUATOR_BACKEND\";\nconst DEFAULT_EVALUATOR_BACKEND: V3EvaluatorBackend = \"legacy\";\n\nexport type V3EvaluatorBackend = \"legacy\" | \"verifier\";\n\nexport type V3EvaluatorOptions = {\n  /**\n   * Selects the evaluator implementation.\n   *\n   * \"legacy\" preserves the existing screenshot/text YES/NO evaluator.\n   * \"verifier\" is reserved for the rubric verifier backend.\n   *\n   * @default process.env.STAGEHAND_EVALUATOR_BACKEND || \"legacy\"\n   */\n  backend?: V3EvaluatorBackend;\n};\n\nexport type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n};\n\ntype NormalizedConstructorOptions = {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n  backend?: V3EvaluatorBackend;\n};\n\nexport class V3Evaluator implements Verifier {\n  private readonly v3: V3;\n  private readonly backend: V3EvaluatorBackend;\n  private readonly modelName: AvailableModel;\n  private readonly modelClientOptions: ClientOptions | { apiKey: string };\n  private readonly legacyEvaluator: LegacyV3Evaluator;\n\n  constructor(\n    v3: V3,\n    modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n    modelClientOptions?: ClientOptions,\n    options?: V3EvaluatorOptions,\n  ) {\n    const normalizedOptions = normalizeConstructorOptions(\n      modelNameOrOptions,\n      modelClientOptions,\n      options,\n    );\n\n    this.backend = resolveEvaluatorBackend(normalizedOptions.backend);\n    this.v3 = v3;\n    this.modelName =\n      normalizedOptions.modelName ||\n      (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = normalizedOptions.modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n    this.legacyEvaluator = new LegacyV3Evaluator(\n      v3,\n      normalizedOptions.modelName,\n      normalizedOptions.modelClientOptions,\n    );\n  }\n\n  async ask(options: EvaluateOptions): Promise<LegacyEvaluationResult> {\n    return this.getLegacyBackend(\"ask\").ask(options);\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<LegacyEvaluationResult[]> {\n    return this.getLegacyBackend(\"batchAsk\").batchAsk(options);\n  }\n\n  async verify(trajectory: Trajectory): Promise<EvaluationResult> {\n    const taskSpec = assertVerifierInput(trajectory);\n\n    if (this.backend === \"legacy\") {\n      return this.verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec);\n    }\n\n    const verifier = new RubricVerifier({\n      getClient: () => this.getClient(),\n      getRubricGenClient: () => this.getRubricGenClient(),\n    });\n    return verifier.verify(trajectory);\n  }\n\n  async generateRubric(taskSpec: TaskSpec): Promise<Rubric> {\n    if (!taskSpec?.id) {\n      throw new StagehandInvalidArgumentError(\n        \"TaskSpec.id is required for rubric generation\",\n      );\n    }\n\n    if (this.backend === \"legacy\") {\n      return {\n        items: [legacyTaskCompletionCriterion(taskSpec)],\n      };\n    }\n\n    const verifier = new RubricVerifier({\n      getClient: () => this.getClient(),\n      getRubricGenClient: () => this.getRubricGenClient(),\n    });\n    return verifier.generateRubric(taskSpec);\n  }\n\n  private getLegacyBackend(methodName: string): LegacyV3Evaluator {\n    if (this.backend === \"legacy\") {\n      return this.legacyEvaluator;\n    }\n\n    return this.unavailableVerifierBackend(methodName);\n  }\n\n  private unavailableVerifierBackend(methodName: string): never {\n    throw new StagehandInvalidArgumentError(\n      `V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend only supports verify() and generateRubric(). Use \"legacy\" for ask()/batchAsk().`,\n    );\n  }\n\n  private getClient(): LLMClient {\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  private getRubricGenClient(): LLMClient {\n    const override = process.env.VERIFIER_RUBRIC_MODEL as\n      | AvailableModel\n      | undefined;\n    if (!override) return this.getClient();\n\n    const provider = new LLMProvider(this.v3.logger);\n    const overrideProvider = override.includes(\"/\")\n      ? override.slice(0, override.indexOf(\"/\"))\n      : undefined;\n    const defaultProvider = this.modelName.includes(\"/\")\n      ? this.modelName.slice(0, this.modelName.indexOf(\"/\"))\n      : undefined;\n    const sameProvider =\n      overrideProvider !== undefined && overrideProvider === defaultProvider;\n\n    return provider.getClient(\n      override,\n      sameProvider ? this.modelClientOptions : undefined,\n    );\n  }\n\n  private async verifyTrajectoryWithLegacyEvaluator(\n    trajectory: Trajectory,\n    taskSpec: TaskSpec,\n  ): Promise<EvaluationResult> {\n    const screenshots = collectLegacyScreenshots(trajectory);\n    const agentReasoning = renderLegacyAgentReasoning(trajectory);\n    const answer = trajectory.finalAnswer;\n\n    if (!screenshots.length && !answer) {\n      return legacyInsufficientEvidenceResult(\n        \"Legacy evaluator compatibility mode had no screenshots or final answer to evaluate.\",\n      );\n    }\n\n    const result = await this.legacyEvaluator.ask({\n      question: taskSpec.instruction,\n      screenshot: screenshots.length ? screenshots : false,\n      answer,\n      agentReasoning,\n    });\n\n    return legacyEvaluationToResult(result, screenshots.length);\n  }\n}\n\nfunction normalizeConstructorOptions(\n  modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n  modelClientOptions?: ClientOptions,\n  options?: V3EvaluatorOptions,\n): NormalizedConstructorOptions {\n  if (\n    modelNameOrOptions &&\n    typeof modelNameOrOptions === \"object\" &&\n    !Array.isArray(modelNameOrOptions)\n  ) {\n    return {\n      modelName: modelNameOrOptions.modelName,\n      modelClientOptions: modelNameOrOptions.modelClientOptions,\n      backend: modelNameOrOptions.backend ?? options?.backend,\n    };\n  }\n\n  return {\n    modelName: modelNameOrOptions as AvailableModel | undefined,\n    modelClientOptions,\n    backend: options?.backend,\n  };\n}\n\nfunction resolveEvaluatorBackend(\n  explicitBackend?: V3EvaluatorBackend,\n): V3EvaluatorBackend {\n  const configuredBackend =\n    explicitBackend ??\n    process.env[EVALUATOR_BACKEND_ENV] ??\n    DEFAULT_EVALUATOR_BACKEND;\n  const normalizedBackend = configuredBackend.trim().toLowerCase();\n\n  if (normalizedBackend === \"legacy\" || normalizedBackend === \"verifier\") {\n    return normalizedBackend;\n  }\n\n  throw new StagehandInvalidArgumentError(\n    `Invalid ${EVALUATOR_BACKEND_ENV}=\"${configuredBackend}\". Expected \"legacy\" or \"verifier\".`,\n  );\n}\n\nfunction assertVerifierInput(trajectory: Trajectory): TaskSpec {\n  if (!trajectory) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory is required for verification\",\n    );\n  }\n  if (!trajectory.task?.id) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory.task.id is required for verification\",\n    );\n  }\n  return trajectory.task;\n}\n\nfunction legacyTaskCompletionCriterion(taskSpec: TaskSpec) {\n  return {\n    criterion: \"legacy-task-completion\",\n    description: `Evaluate whether the task was completed successfully: ${taskSpec.instruction}`,\n    maxPoints: 1,\n  };\n}\n\nfunction collectLegacyScreenshots(trajectory: Trajectory): Buffer[] {\n  const screenshots: Buffer[] = [];\n\n  for (const step of trajectory.steps ?? []) {\n    if (Buffer.isBuffer(step.probeEvidence?.screenshot)) {\n      screenshots.push(step.probeEvidence.screenshot);\n      continue;\n    }\n\n    const agentImage = step.agentEvidence?.modalities?.find(\n      (\n        modality,\n      ): modality is Extract<AgentEvidenceModality, { type: \"image\" }> =>\n        modality.type === \"image\" && Buffer.isBuffer(modality.bytes),\n    );\n\n    if (agentImage) {\n      screenshots.push(agentImage.bytes);\n    }\n  }\n\n  if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {\n    screenshots.push(trajectory.finalObservation.screenshot);\n  }\n\n  return screenshots;\n}\n\nfunction renderLegacyAgentReasoning(\n  trajectory: Trajectory,\n): string | undefined {\n  const stepLines = (trajectory.steps ?? []).map((step, i) => {\n    const status = step.toolOutput?.ok === false ? \"Tool status: failed\" : \"\";\n    const output = step.toolOutput?.error\n      ? `Tool error: ${step.toolOutput.error}`\n      : `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;\n    return [\n      `Step ${i}: ${step.actionName}`,\n      step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,\n      status || undefined,\n      output,\n    ]\n      .filter(Boolean)\n      .join(\"\\n\");\n  });\n\n  if (!stepLines.length) {\n    return undefined;\n  }\n\n  return truncateForPrompt(\n    `Agent trajectory:\\n${stepLines.join(\"\\n\\n\")}`,\n    16000,\n  );\n}\n\nfunction stringifyForPrompt(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n\n  try {\n    const serialized = JSON.stringify(value);\n    return serialized ?? String(value);\n  } catch {\n    return String(value);\n  }\n}\n\nfunction truncateForPrompt(value: string, maxLength: number): string {\n  if (value.length <= maxLength) {\n    return value;\n  }\n\n  return `${value.slice(0, maxLength)}... [truncated]`;\n}\n\nfunction legacyEvaluationToResult(\n  result: LegacyEvaluationResult,\n  screenshotCount: number,\n): EvaluationResult {\n  const outcomeSuccess = result.evaluation === \"YES\";\n  const invalid = result.evaluation === \"INVALID\";\n  const findings: VerifierFinding[] = invalid\n    ? [\n        {\n          category: \"verifier_uncertainty\",\n          severity: \"warning\",\n          description: result.reasoning,\n        },\n      ]\n    : [];\n\n  return {\n    outcomeSuccess,\n    explanation: result.reasoning,\n    ...(findings.length ? { findings } : {}),\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: result.evaluation,\n      screenshotCount,\n    },\n  };\n}\n\nfunction legacyInsufficientEvidenceResult(reason: string): EvaluationResult {\n  return {\n    outcomeSuccess: false,\n    explanation: reason,\n    findings: [\n      {\n        category: \"trajectory_capture\",\n        severity: \"blocking\",\n        description: reason,\n      },\n    ],\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: \"INVALID\",\n      screenshotCount: 0,\n    },\n  };\n}\n"]}

package/dist/esm/lib/v3LegacyEvaluator.js

@@ -40,6 +40,7 @@ export class LegacyV3Evaluator {
         if (Array.isArray(screenshot)) {
             return this._evaluateWithMultipleScreenshots({
                 question,
+                answer,
                 screenshots: screenshot,
                 systemPrompt,
                 agentReasoning,
@@ -159,7 +160,7 @@ export class LegacyV3Evaluator {
         }
     }
     async _evaluateWithMultipleScreenshots(options) {
-        const { question, screenshots, agentReasoning, systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.
+        const { question, answer, screenshots, agentReasoning, systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.
         ${agentReasoning ? "You also have access to the agent's detailed reasoning and thought process throughout the task." : ""}
         Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.
         Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).
@@ -188,6 +189,9 @@ export class LegacyV3Evaluator {
                                     ? `Question: ${question}\n\nAgent's reasoning and actions throughout the task:\n${agentReasoning}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`
                                     : `${question}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,
                             },
+                            ...(answer
+                                ? [{ type: "text", text: `the answer is ${answer}` }]
+                                : []),
                             ...imageContents,
                         ],
                     },

package/dist/esm/lib/v3LegacyEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"v3LegacyEvaluator.js","sourceRoot":"","sources":["../../../lib/v3LegacyEvaluator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAExD,MAAM,OAAO,iBAAiB;IACpB,EAAE,CAAK;IACP,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,YAAY,GAA+B,GAAG,EAAE,GAAE,CAAC,CAAC;IAE5D,YACE,EAAM,EACN,SAA0B,EAC1B,kBAAkC;QAElC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,IAAK,yBAA4C,CAAC;QAC5E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI;YAC9C,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,sEAAsE;QACtE,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,UAAU,GAAG,IAAI,EACjB,YAAY,EACZ,iBAAiB,GAAG,GAAG,EACvB,cAAc,GACf,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;YACxB,MAAM,IAAI,6BAA6B,CACrC,qDAAqD,CACtD,CAAC;QAEJ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,gCAAgC,CAAC;gBAC3C,QAAQ;gBACR,WAAW,EAAE,UAAU;gBACvB,YAAY;gBACZ,cAAc;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,mBAAmB,GAAG,kIAAkI,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,sDAAsD,8HAA8H,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC;QAElZ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE;oBAChE;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,6CAA6C,cAAc,EAAE;oCACpF,CAAC,CAAC,QAAQ;6BACb;4BACD,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,EACJ,SAAS,EACT,UAAU,GAAG,IAAI,EACjB,YAAY,GAAG,8EAA8E,EAC7F,iBAAiB,GAAG,GAAG,GACxB,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,SAAS,EAAE,MAAM;YACpB,MAAM,IAAI,6BAA6B,CACrC,iCAAiC,CAClC,CAAC;QAEJ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAG,SAAS;aACxB,GAAG,CACF,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF;aACA,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,GAAG,YAAY,2CAA2C,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE,6KAA6K;qBAChX;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;4BACjC,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,uBAAuB;oBAC7B,MAAM,EAAE,qBAAqB;iBAC9B;aACF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAExB,CAAC;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1B,UAAU,EAAE,SAAkB;gBAC9B,SAAS,EAAE,sCAAsC,YAAY,EAAE;aAChE,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAK9C;QACC,MAAM,EACJ,QAAQ,EACR,WAAW,EACX,cAAc,EACd,YAAY,GAAG;UACX,cAAc,CAAC,CAAC,CAAC,iGAAiG,CAAC,CAAC,CAAC,EAAE;;;UAGvH,cAAc,CAAC,CAAC,CAAC,kNAAkN,CAAC,CAAC,CAAC,EAAE;0BACxN,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,GACtD,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAC1C,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,CAC3C,CAAC;QAEJ,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,WAAoB;YAC1B,SAAS,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;SACrE,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;oBACzC;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,2DAA2D,cAAc,qBAAqB,WAAW,CAAC,MAAM,sKAAsK;oCAC7S,CAAC,CAAC,GAAG,QAAQ,qBAAqB,WAAW,CAAC,MAAM,mIAAmI;6BAC1L;4BACD,GAAG,aAAa;yBACjB;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;CACF","sourcesContent":["/**\n * Legacy V3 evaluator implementation.\n *\n * This is the behavior-preserving implementation that backs V3Evaluator when\n * STAGEHAND_EVALUATOR_BACKEND=legacy.\n */\n\nimport { z } from \"zod\";\nimport type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { LLMParsedResponse } from \"./inference.js\";\nimport { LLMResponse, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\nconst EvaluationSchema = z.object({\n  evaluation: z.enum([\"YES\", \"NO\"]),\n  reasoning: z.string(),\n});\n\nconst BatchEvaluationSchema = z.array(EvaluationSchema);\n\nexport class LegacyV3Evaluator {\n  private v3: V3;\n  private modelName: AvailableModel;\n  private modelClientOptions: ClientOptions | { apiKey: string };\n  private silentLogger: (message: LogLine) => void = () => {};\n\n  constructor(\n    v3: V3,\n    modelName?: AvailableModel,\n    modelClientOptions?: ClientOptions,\n  ) {\n    this.v3 = v3;\n    this.modelName = modelName || (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n  }\n\n  private getClient(): LLMClient {\n    // Prefer a dedicated provider so we can override model per-evaluation\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshot = true,\n      systemPrompt,\n      screenshotDelayMs = 250,\n      agentReasoning,\n    } = options;\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!answer && !screenshot)\n      throw new StagehandInvalidArgumentError(\n        \"Either answer (text) or screenshot must be provided\",\n      );\n\n    if (Array.isArray(screenshot)) {\n      return this._evaluateWithMultipleScreenshots({\n        question,\n        screenshots: screenshot,\n        systemPrompt,\n        agentReasoning,\n      });\n    }\n\n    const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? \"a screenshot\" : \"the agents reasoning and actions throughout the task\"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\\n          Today's date is ${new Date().toLocaleDateString()}`;\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt || defaultSystemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions taken:\\n${agentReasoning}`\n                  : question,\n              },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    const {\n      questions,\n      screenshot = true,\n      systemPrompt = \"You are an expert evaluator that returns YES or NO with a concise reasoning.\",\n      screenshotDelayMs = 250,\n    } = options;\n    if (!questions?.length)\n      throw new StagehandInvalidArgumentError(\n        \"Questions array cannot be empty\",\n      );\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const formatted = questions\n      .map(\n        (item, i) =>\n          `${i + 1}. ${item.question}${item.answer ? `\\n   Answer: ${item.answer}` : \"\"}`,\n      )\n      .join(\"\\n\\n\");\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          {\n            role: \"system\",\n            content: `${systemPrompt}\\n\\nYou will be given multiple questions${screenshot ? \" with a screenshot\" : \"\"}. ${questions.some((q) => q.answer) ? \"Some questions include answers to evaluate.\" : \"\"} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,\n          },\n          {\n            role: \"user\",\n            content: [\n              { type: \"text\", text: formatted },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n            ],\n          },\n        ],\n        response_model: {\n          name: \"BatchEvaluationResult\",\n          schema: BatchEvaluationSchema,\n        },\n      },\n    });\n\n    try {\n      const results = response.data as unknown as z.infer<\n        typeof BatchEvaluationSchema\n      >;\n      return results.map((r) => ({\n        evaluation: r.evaluation,\n        reasoning: r.reasoning,\n      }));\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return questions.map(() => ({\n        evaluation: \"INVALID\" as const,\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      }));\n    }\n  }\n\n  private async _evaluateWithMultipleScreenshots(options: {\n    question: string;\n    screenshots: Buffer[];\n    systemPrompt?: string;\n    agentReasoning?: string;\n  }): Promise<EvaluationResult> {\n    const {\n      question,\n      screenshots,\n      agentReasoning,\n      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.\n        ${agentReasoning ? \"You also have access to the agent's detailed reasoning and thought process throughout the task.\" : \"\"}\n        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.\n        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).\n        ${agentReasoning ? \"The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation.\" : \"\"}\n        Today's date is ${new Date().toLocaleDateString()}`,\n    } = options;\n\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!screenshots || screenshots.length === 0)\n      throw new StagehandInvalidArgumentError(\n        \"At least one screenshot must be provided\",\n      );\n\n    const llmClient = this.getClient();\n\n    const imageContents = screenshots.map((s) => ({\n      type: \"image_url\" as const,\n      image_url: { url: `data:image/jpeg;base64,${s.toString(\"base64\")}` },\n    }));\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions throughout the task:\\n${agentReasoning}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`\n                  : `${question}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,\n              },\n              ...imageContents,\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n}\n"]}
+{"version":3,"file":"v3LegacyEvaluator.js","sourceRoot":"","sources":["../../../lib/v3LegacyEvaluator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAExD,MAAM,OAAO,iBAAiB;IACpB,EAAE,CAAK;IACP,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,YAAY,GAA+B,GAAG,EAAE,GAAE,CAAC,CAAC;IAE5D,YACE,EAAM,EACN,SAA0B,EAC1B,kBAAkC;QAElC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,IAAK,yBAA4C,CAAC;QAC5E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI;YAC9C,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,sEAAsE;QACtE,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,UAAU,GAAG,IAAI,EACjB,YAAY,EACZ,iBAAiB,GAAG,GAAG,EACvB,cAAc,GACf,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;YACxB,MAAM,IAAI,6BAA6B,CACrC,qDAAqD,CACtD,CAAC;QAEJ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,gCAAgC,CAAC;gBAC3C,QAAQ;gBACR,MAAM;gBACN,WAAW,EAAE,UAAU;gBACvB,YAAY;gBACZ,cAAc;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,mBAAmB,GAAG,kIAAkI,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,sDAAsD,8HAA8H,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC;QAElZ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE;oBAChE;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,6CAA6C,cAAc,EAAE;oCACpF,CAAC,CAAC,QAAQ;6BACb;4BACD,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,EACJ,SAAS,EACT,UAAU,GAAG,IAAI,EACjB,YAAY,GAAG,8EAA8E,EAC7F,iBAAiB,GAAG,GAAG,GACxB,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,SAAS,EAAE,MAAM;YACpB,MAAM,IAAI,6BAA6B,CACrC,iCAAiC,CAClC,CAAC;QAEJ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAG,SAAS;aACxB,GAAG,CACF,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF;aACA,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,GAAG,YAAY,2CAA2C,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE,6KAA6K;qBAChX;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;4BACjC,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,uBAAuB;oBAC7B,MAAM,EAAE,qBAAqB;iBAC9B;aACF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAExB,CAAC;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1B,UAAU,EAAE,SAAkB;gBAC9B,SAAS,EAAE,sCAAsC,YAAY,EAAE;aAChE,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAM9C;QACC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,WAAW,EACX,cAAc,EACd,YAAY,GAAG;UACX,cAAc,CAAC,CAAC,CAAC,iGAAiG,CAAC,CAAC,CAAC,EAAE;;;UAGvH,cAAc,CAAC,CAAC,CAAC,kNAAkN,CAAC,CAAC,CAAC,EAAE;0BACxN,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,GACtD,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAC1C,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,CAC3C,CAAC;QAEJ,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,WAAoB;YAC1B,SAAS,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;SACrE,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;oBACzC;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,2DAA2D,cAAc,qBAAqB,WAAW,CAAC,MAAM,sKAAsK;oCAC7S,CAAC,CAAC,GAAG,QAAQ,qBAAqB,WAAW,CAAC,MAAM,mIAAmI;6BAC1L;4BACD,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,aAAa;yBACjB;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;CACF","sourcesContent":["/**\n * Legacy V3 evaluator implementation.\n *\n * This is the behavior-preserving implementation that backs V3Evaluator when\n * STAGEHAND_EVALUATOR_BACKEND=legacy.\n */\n\nimport { z } from \"zod\";\nimport type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { LLMParsedResponse } from \"./inference.js\";\nimport { LLMResponse, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\nconst EvaluationSchema = z.object({\n  evaluation: z.enum([\"YES\", \"NO\"]),\n  reasoning: z.string(),\n});\n\nconst BatchEvaluationSchema = z.array(EvaluationSchema);\n\nexport class LegacyV3Evaluator {\n  private v3: V3;\n  private modelName: AvailableModel;\n  private modelClientOptions: ClientOptions | { apiKey: string };\n  private silentLogger: (message: LogLine) => void = () => {};\n\n  constructor(\n    v3: V3,\n    modelName?: AvailableModel,\n    modelClientOptions?: ClientOptions,\n  ) {\n    this.v3 = v3;\n    this.modelName = modelName || (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n  }\n\n  private getClient(): LLMClient {\n    // Prefer a dedicated provider so we can override model per-evaluation\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshot = true,\n      systemPrompt,\n      screenshotDelayMs = 250,\n      agentReasoning,\n    } = options;\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!answer && !screenshot)\n      throw new StagehandInvalidArgumentError(\n        \"Either answer (text) or screenshot must be provided\",\n      );\n\n    if (Array.isArray(screenshot)) {\n      return this._evaluateWithMultipleScreenshots({\n        question,\n        answer,\n        screenshots: screenshot,\n        systemPrompt,\n        agentReasoning,\n      });\n    }\n\n    const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? \"a screenshot\" : \"the agents reasoning and actions throughout the task\"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\\n          Today's date is ${new Date().toLocaleDateString()}`;\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt || defaultSystemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions taken:\\n${agentReasoning}`\n                  : question,\n              },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    const {\n      questions,\n      screenshot = true,\n      systemPrompt = \"You are an expert evaluator that returns YES or NO with a concise reasoning.\",\n      screenshotDelayMs = 250,\n    } = options;\n    if (!questions?.length)\n      throw new StagehandInvalidArgumentError(\n        \"Questions array cannot be empty\",\n      );\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const formatted = questions\n      .map(\n        (item, i) =>\n          `${i + 1}. ${item.question}${item.answer ? `\\n   Answer: ${item.answer}` : \"\"}`,\n      )\n      .join(\"\\n\\n\");\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          {\n            role: \"system\",\n            content: `${systemPrompt}\\n\\nYou will be given multiple questions${screenshot ? \" with a screenshot\" : \"\"}. ${questions.some((q) => q.answer) ? \"Some questions include answers to evaluate.\" : \"\"} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,\n          },\n          {\n            role: \"user\",\n            content: [\n              { type: \"text\", text: formatted },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n            ],\n          },\n        ],\n        response_model: {\n          name: \"BatchEvaluationResult\",\n          schema: BatchEvaluationSchema,\n        },\n      },\n    });\n\n    try {\n      const results = response.data as unknown as z.infer<\n        typeof BatchEvaluationSchema\n      >;\n      return results.map((r) => ({\n        evaluation: r.evaluation,\n        reasoning: r.reasoning,\n      }));\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return questions.map(() => ({\n        evaluation: \"INVALID\" as const,\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      }));\n    }\n  }\n\n  private async _evaluateWithMultipleScreenshots(options: {\n    question: string;\n    answer?: string;\n    screenshots: Buffer[];\n    systemPrompt?: string;\n    agentReasoning?: string;\n  }): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshots,\n      agentReasoning,\n      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.\n        ${agentReasoning ? \"You also have access to the agent's detailed reasoning and thought process throughout the task.\" : \"\"}\n        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.\n        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).\n        ${agentReasoning ? \"The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation.\" : \"\"}\n        Today's date is ${new Date().toLocaleDateString()}`,\n    } = options;\n\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!screenshots || screenshots.length === 0)\n      throw new StagehandInvalidArgumentError(\n        \"At least one screenshot must be provided\",\n      );\n\n    const llmClient = this.getClient();\n\n    const imageContents = screenshots.map((s) => ({\n      type: \"image_url\" as const,\n      image_url: { url: `data:image/jpeg;base64,${s.toString(\"base64\")}` },\n    }));\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions throughout the task:\\n${agentReasoning}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`\n                  : `${question}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,\n              },\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n              ...imageContents,\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n}\n"]}

package/dist/esm/lib/version.d.ts

@@ -2,4 +2,4 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export declare const STAGEHAND_VERSION: "3.4.0";
+export declare const STAGEHAND_VERSION: "3.6.0";

package/dist/esm/lib/version.js

@@ -2,5 +2,5 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export const STAGEHAND_VERSION = "3.4.0";
+export const STAGEHAND_VERSION = "3.6.0";
 //# sourceMappingURL=version.js.map

package/dist/esm/lib/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.4.0\" as const;\n"]}
+{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.6.0\" as const;\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserbasehq/orca",
-  "version": "3.5.0-vertex-test",
+  "version": "3.6.0-gemini-cua",
   "description": "An AI web browsing framework focused on simplicity and extensibility.",
   "type": "module",
   "main": "./dist/cjs/index.js",
@@ -36,6 +36,7 @@
     "build:cjs": "tsx scripts/build-cjs.ts",
     "build:esm": "tsx scripts/build-esm.ts",
     "build": "pnpm --filter @browserbasehq/orca run --parallel \"/^build:(esm|cjs)$/\"",
+    "prepack": "pnpm -w --dir ../.. exec turbo run build --filter=@browserbasehq/orca",
     "example": "node --import tsx -e \"const args=process.argv.slice(1).filter(a=>a!=='--'); const [p]=args; const n=(p||'example').replace(/^\\.\\//,'').replace(/\\.ts$/i,''); import('node:path').then(path=>import(new URL(path.resolve('examples', n + '.ts'), 'file:')));\" --",
     "test": "pnpm -w --dir ../.. exec turbo run test:core test:e2e --filter=@browserbasehq/orca --",
     "test:core": "tsx scripts/test-core.ts",
@@ -46,10 +47,15 @@
     "lint": "cd ../.. && prettier --check packages/core && cd packages/core && pnpm run eslint && pnpm run typecheck"
   },
   "files": [
-    "dist/esm",
-    "dist/cjs",
-    "!dist/esm/tests",
-    "!dist/cjs/tests"
+    "dist/esm/index.d.ts",
+    "dist/esm/index.js",
+    "dist/esm/lib",
+    "dist/esm/package.json",
+    "dist/cjs/cli.js",
+    "dist/cjs/index.d.ts",
+    "dist/cjs/index.js",
+    "dist/cjs/lib",
+    "dist/cjs/package.json"
   ],
   "keywords": [
     "ai",
@@ -84,19 +90,19 @@
     "@google/genai": "^1.22.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "ai": "^5.0.133",
-    "devtools-protocol": "^0.0.1464554",
+    "devtools-protocol": "^0.0.1642743",
     "fetch-cookie": "^3.1.0",
     "openai": "^4.104.0",
     "pino": "^9.6.0",
     "pino-pretty": "^13.0.0",
     "uuid": "^11.1.1",
-    "ws": "^8.18.0",
+    "ws": "^8.21.0",
     "zod-to-json-schema": "^3.25.0"
   },
   "optionalDependencies": {
     "@ai-sdk/amazon-bedrock": "^3.0.73",
-    "@ai-sdk/anthropic": "^2.0.34",
-    "@ai-sdk/azure": "^2.0.54",
+    "@ai-sdk/anthropic": "^2.0.81",
+    "@ai-sdk/azure": "^2.0.109",
     "@ai-sdk/cerebras": "^1.0.25",
     "@ai-sdk/deepseek": "^1.0.23",
     "@ai-sdk/google": "^2.0.53",
@@ -117,7 +123,7 @@
     "@types/jsdom": "^27.0.0",
     "@types/node": "^20.11.30",
     "@types/ws": "^8.5.13",
-    "@vitest/coverage-v8": "^4.0.8",
+    "@vitest/coverage-v8": "^4.1.9",
     "adm-zip": "^0.5.16",
     "chalk": "^5.4.1",
     "eslint": "10.0.2",
@@ -128,8 +134,8 @@
     "prettier": "^3.2.5",
     "puppeteer-core": "^24.43.0",
     "tsx": "*",
-    "vite": "^7.3.3",
-    "vitest": "^4.1.6",
+    "vite": "^7.3.5",
+    "vitest": "^4.1.9",
     "zod": "^3.25.76 || ^4.2.0"
   },
   "repository": {