@browserbasehq/orca 3.5.1-preview.0 → 3.6.0-gemini-cua

package/dist/esm/lib/v3/verifier/types.d.ts

@@ -5,6 +5,8 @@
  * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve
  * screenshots sent to the provider plus independent harness probes.
  */
+import type { LLMClient } from "../llm/LLMClient.js";
+import type { LogLine } from "../types/public/logs.js";
 /** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */
 export interface TrajectoryUsage {
     input_tokens: number;
@@ -123,12 +125,15 @@ export type TrajectoryStatus = "complete" | "aborted" | "stalled" | "error";
  * The on-disk layout is one directory per task:
  *
  *   .trajectories/<run-id>/<task-id>/
- *     ├── task_data.json    — TaskSpec + result metadata
- *     ├── trajectory.json   — this object, with screenshotPath instead of bytes
- *     ├── screenshots/      — step probe/agent images plus final observation
+ *     ├── task_data.json           — TaskSpec + result metadata
+ *     ├── trajectory.json          — this object, with image paths instead of bytes
+ *     ├── screenshots/
+ *     │   ├── probe/<N>.png        — tier-2 probe screenshot per step
+ *     │   ├── probe/final.png      — final terminal observation screenshot
+ *     │   └── agent/<N>.png        — tier-1 image the model received per step
  *     ├── scores/
- *     │   └── result.json       — Result from V3Evaluator.verify()
- *     └── core.log          — captured action log
+ *     │   └── result.json          — Result from V3Evaluator.verify()
+ *     └── core.log                 — captured action log
  */
 export interface Trajectory {
     task: TaskSpec;
@@ -139,6 +144,70 @@ export interface Trajectory {
     status: TrajectoryStatus;
     usage: TrajectoryUsage;
 }
+/** A single screenshot kept by Step 1, ready for downstream relevance scoring. */
+export interface CanonicalScreenshot {
+    /** 0-based position in the kept-screenshots array. Stable across the pipeline. */
+    canonicalIndex: number;
+    /**
+     * 0-based position in `Trajectory.steps` this screenshot came from
+     * (steps.length for the final observation). Lets downstream prompts
+     * cross-reference the action history.
+     */
+    originalStepIndex: number;
+    /** The resized PNG/JPEG buffer (or native bytes if sharp unavailable). */
+    bytes: Buffer;
+    /** MIME media type. Always "image/png" after the optional resize. */
+    mediaType: string;
+    /** Reason this frame was kept: "first" / "last" / "diverges". */
+    keptReason: "first" | "last" | "diverges" | "no-dedup";
+}
+/**
+ * A text evidence point sourced from tier-2 probes or tier-1 tool outputs.
+ * These feed the same relevance + scoring path as screenshots, letting DOM
+ * and hybrid agents preserve extract/aria/tool-return evidence without a
+ * separate verifier architecture.
+ */
+export interface CanonicalTextEvidence {
+    /** 0-based position in the combined evidence-point array. */
+    canonicalIndex: number;
+    originalStepIndex: number;
+    /** Where the text came from. */
+    source: "probe-aria" | "agent-text" | "agent-json" | "tool-output";
+    /** The text payload, already truncated. */
+    content: string;
+}
+export type CanonicalEvidence = CanonicalScreenshot | CanonicalTextEvidence;
+/** Result of Step 1 evidence loading. */
+export interface EvidenceLoadResult {
+    /** Kept frames, in chronological order. */
+    screenshots: CanonicalScreenshot[];
+    /**
+     * Maps trajectory step position → canonical index in `screenshots`. Steps
+     * whose screenshots were deduplicated point to the surviving canonical frame
+     * (typically the prior kept frame). Useful for "find me the screenshot for
+     * step K" lookups in downstream prompts.
+     */
+    stepIndexToCanonical: Map<number, number>;
+    /** Number of original frames considered. */
+    originalCount: number;
+    /** Number of frames kept post-dedup (== screenshots.length). */
+    keptCount: number;
+    /** Effective thresholds used (resolved from env). */
+    thresholds: {
+        ssim: number;
+        mse: number;
+        resize: number;
+    };
+}
+/** Options for evidence loading; primarily test seams over the defaults. */
+export interface EvidenceLoadOptions {
+    /** SSIM similarity threshold for dedup (default 0.75). */
+    ssimThreshold?: number;
+    /** MSE similarity threshold for dedup (default 30). */
+    mseThreshold?: number;
+    /** Scale factor applied before relevance scoring (default 0.7). */
+    imageResize?: number;
+}
 /** Score for a single rubric criterion after evidence analysis + rescoring. */
 export interface CriterionScore {
     /** Matches RubricCriterion.criterion (the criterion's short name). */
@@ -146,9 +215,9 @@ export interface CriterionScore {
     /** Maximum possible points for this criterion. */
     maxPoints: number;
     /**
-     * Points earned post-evidence-analysis (paper's post_image_earned_points).
-     * Null if the criterion was conditional and its condition wasn't met (excluded
-     * from both numerator and denominator in the process score).
+     * Points earned after evidence analysis. Null when the criterion is
+     * conditional and its condition was not met — excluded from both numerator
+     * and denominator in the process score.
      */
     earnedPoints: number | null;
     /** Verifier's explanation for the score. */
@@ -160,15 +229,14 @@ export interface CriterionScore {
     conditionMet?: boolean;
     /**
      * Set when the verifier had no evidence to ground this criterion in either
-     * tier. Per paper §2, treated as uncontrollable failure → full credit, but
-     * surfaced here so dashboards can flag low-confidence results.
+     * tier. Treated as uncontrollable failure (full credit) but surfaced here
+     * so dashboards can flag low-confidence results.
      */
     evidenceInsufficient?: boolean;
 }
 /**
- * First-point-of-failure analysis (paper Step 9a). Identifies the earliest
- * step where the agent's trajectory went off-track, using a structured error
- * taxonomy (7 top-level categories, 1.1–7.4 sub-codes).
+ * Earliest step where the agent's trajectory went off-track, classified
+ * against the error taxonomy (7 top-level categories, 1.1–7.4 sub-codes).
  */
 export interface FirstPointOfFailure {
     stepIndex: number;
@@ -212,25 +280,31 @@ export interface VerifierFinding {
 /** Stable debugging summary emitted by verifier backends. */
 export interface VerifierRawSteps {
     backend?: "legacy" | "verifier";
+    reason?: string;
     primaryIntent?: string;
     reasoning?: string;
     rubricSource?: "precomputed" | "generated" | "none";
-    approach?: "a" | "b";
+    approach?: "a" | "b" | "outcome-only";
     optionalsMode?: "folded" | "separate" | "skip";
     totalEarned?: number;
     totalMax?: number;
     evidenceImages?: number;
     evidenceTexts?: number;
     evidenceOriginalScreenshots?: number;
+    screenshotsAttached?: number;
     legacyEvaluation?: string;
     screenshotCount?: number;
 }
-/** Task-validity classification (paper Step 10). */
+/** Task-validity classification: whether the task is even answerable. */
 export interface TaskValidity {
     /** True if the task is underspecified / has multiple valid interpretations. */
     isAmbiguous: boolean;
+    /** Explanation for why the task is ambiguous, when available. */
+    ambiguityReason?: string;
     /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */
     isInvalid: boolean;
+    /** Explanation for why the task is invalid, when available. */
+    invalidReason?: string;
     /** Optional sub-codes from the task-classification taxonomy. */
     ambiguityCodes?: string[];
     invalidTaskCodes?: string[];
@@ -279,3 +353,77 @@ export interface EvaluationResult {
 export interface Verifier {
     verify(trajectory: Trajectory): Promise<EvaluationResult>;
 }
+export interface RubricVerifierOptions {
+    /** Factory that returns a configured LLMClient. Called per pipeline step so callers can supply step-specific clients. */
+    getClient: () => LLMClient;
+    /** Optional factory for rubric generation so callers can route it to a stronger model. */
+    getRubricGenClient?: () => LLMClient;
+    /** Logger; defaults to a no-op so the verifier stays quiet inside V3Evaluator. */
+    logger?: (line: LogLine) => void;
+    /**
+     * Override any verifier knob. Env vars supply the defaults; values here win.
+     * Useful for tests and for cross-verify sweeps that want different budgets
+     * per run.
+     */
+    config?: Partial<VerifierConfig>;
+}
+/**
+ * Resolved verifier knobs. Constructed once from env (and optional overrides)
+ * by RubricVerifier's constructor; subsequent verify() calls can pass a
+ * Partial to shift any field.
+ */
+export interface VerifierConfig {
+    /** Which pipeline path to take: per-criterion (a), fused (b), or skip rubric entirely (outcome-only). */
+    approach: "a" | "b" | "outcome-only";
+    /** Folded (in fused call), separate (own calls), or skip (omit). */
+    optionalSteps: "folded" | "separate" | "skip";
+    /** Top-K evidence points selected per criterion. */
+    topK: number;
+    /** Batch size for the relevance-scoring LLM call. */
+    relevanceBatchSize: number;
+    /** Image cap on the outcome-only path. */
+    outcomeMaxImages: number;
+    /** Concurrent LLM calls across batches / criteria. */
+    maxParallel: number;
+    /** Token budgets for the three evidence channels. */
+    evidenceTokenBudget: number;
+    outcomeEvidenceTokenBudget: number;
+    actionHistoryTokenBudget: number;
+    /** Per-section character limits applied during evidence-text assembly. */
+    truncation: {
+        /** Master switch: when true, all per-section limits go to MAX_SAFE_INTEGER. */
+        disabled: boolean;
+        evidenceTextPreview: number;
+        groupedEvidenceText: number;
+        buildEvidenceText: number;
+        buildEvidenceAria: number;
+        actionHistoryReasoning: number;
+    };
+}
+export interface ErrorTaxonomySubCategory {
+    /** Sub-code (e.g., "2.3"). */
+    code: string;
+    /** Human-readable name (e.g., "Output fabrication"). */
+    name: string;
+    /** Detailed description ported from the .md. Markdown formatting preserved. */
+    description: string;
+}
+export interface ErrorTaxonomyCategory {
+    /** Top-level number (1-8). */
+    number: number;
+    /** Top-level name (e.g., "Hallucination Errors"). */
+    name: string;
+    /** One-sentence summary of the category. */
+    summary: string;
+    /** Sub-categories. The last one is always an "Other" catch-all. */
+    subCategories: ErrorTaxonomySubCategory[];
+}
+export interface ParseFailureStepNumbersOptions {
+    /**
+     * Maximum unique step numbers to expand from ranges. Protects the verifier
+     * from malformed model output such as "0-2147483647".
+     */
+    maxExpandedSteps?: number;
+    /** Optional inclusive upper bound for accepted step numbers. */
+    maxStep?: number;
+}

package/dist/esm/lib/v3/verifier/types.js

@@ -1,9 +1,2 @@
-/**
- * Shared verifier types for trajectories, rubrics, evidence, and results.
- *
- * The verifier consumes saved trajectories instead of a live browser. DOM and
- * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve
- * screenshots sent to the provider plus independent harness probes.
- */
 export {};
 //# sourceMappingURL=types.js.map

package/dist/esm/lib/v3/verifier/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG","sourcesContent":["/**\n * Shared verifier types for trajectories, rubrics, evidence, and results.\n *\n * The verifier consumes saved trajectories instead of a live browser. DOM and\n * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve\n * screenshots sent to the provider plus independent harness probes.\n */\n\n/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */\nexport interface TrajectoryUsage {\n  input_tokens: number;\n  output_tokens: number;\n  reasoning_tokens?: number;\n  cached_input_tokens?: number;\n  inference_time_ms?: number;\n}\n\n/** A single criterion in a Stagehand rubric. */\nexport interface RubricCriterion {\n  /** Short name of the criterion (e.g., \"Add ground beef to cart\"). */\n  criterion: string;\n  /** What to evaluate and how to award partial credit. */\n  description: string;\n  /** Maximum points for this criterion. */\n  maxPoints: number;\n  /**\n   * Applicability rule for situational criteria. When this condition is not\n   * met, the criterion is excluded from scoring rather than counted as failed.\n   */\n  condition?: string;\n}\n\n/** A rubric — list of criteria for a task. */\nexport interface Rubric {\n  items: RubricCriterion[];\n}\n\n/**\n * Spec for a single task being verified. Carried both at runtime and into the\n * verifier alongside the trajectory.\n */\nexport interface TaskSpec {\n  /** Stable identifier (e.g., \"united_13\" for WebTailBench, task_id for Mind2Web). */\n  id: string;\n  /** Task instruction shown to the agent. */\n  instruction: string;\n  /** Starting URL, if any. */\n  initUrl?: string;\n  /** Rubric carried by the dataset or generated by a verifier backend. */\n  precomputedRubric?: Rubric;\n  /** Optional reference answer (set when dataset ships one). */\n  expectedAnswer?: string;\n}\n\n/**\n * A single modality unit in tier-1 agent evidence. Mirrors the shape of\n * ModelMessage content parts so we can reproduce what the LLM ingested.\n */\nexport type AgentEvidenceModality =\n  | { type: \"text\"; content: string }\n  | { type: \"image\"; bytes: Buffer; mediaType: string }\n  | { type: \"json\"; content: unknown };\n\n/**\n * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the\n * tool result for this step.\n *\n * Modes:\n *   - CUA: usually a single image modality (the screenshot sent to the provider).\n *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.\n *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.\n */\nexport interface AgentEvidence {\n  modalities: AgentEvidenceModality[];\n}\n\n/**\n * Tier 2 — independent harness probes around this step.\n *\n * If a probe wasn't captured, the field is absent (not null).\n */\nexport interface ProbeEvidence {\n  /** URL after the step's tool execution. */\n  url?: string;\n  /**\n   * Bus screenshot captured after the step. Path on disk is preferred once\n   * persisted; in-memory Buffer is used during a live run.\n   */\n  screenshot?: Buffer;\n  /** Reference to the persisted screenshot file under the trajectory dir. */\n  screenshotPath?: string;\n  /** Accessibility tree snapshot. */\n  ariaTree?: string;\n  /** Verifier-requested probes, keyed by criterion id. */\n  onDemand?: Record<string, unknown>;\n}\n\n/** Outcome of a single tool execution as seen by the harness. */\nexport interface ToolOutput {\n  ok: boolean;\n  /**\n   * The tool's return value. Same payload that flowed into agentEvidence\n   * modalities, but in its native shape (e.g., the extract result, the act\n   * describe-string) rather than serialized for the LLM.\n   */\n  result: unknown;\n  error?: string;\n}\n\n/** One step in a trajectory: action + reasoning + evidence + outcome. */\nexport interface TrajectoryStep {\n  actionName: string;\n  actionArgs: Record<string, unknown>;\n  /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */\n  reasoning: string;\n  agentEvidence: AgentEvidence;\n  probeEvidence: ProbeEvidence;\n  toolOutput: ToolOutput;\n}\n\n/** Terminal status of the agent run. */\nexport type TrajectoryStatus = \"complete\" | \"aborted\" | \"stalled\" | \"error\";\n\n/**\n * Full trajectory for one task run.\n *\n * The on-disk layout is one directory per task:\n *\n *   .trajectories/<run-id>/<task-id>/\n *     ├── task_data.json    — TaskSpec + result metadata\n *     ├── trajectory.json   — this object, with screenshotPath instead of bytes\n *     ├── screenshots/      — step probe/agent images plus final observation\n *     ├── scores/\n *     │   └── result.json       — Result from V3Evaluator.verify()\n *     └── core.log          — captured action log\n */\nexport interface Trajectory {\n  task: TaskSpec;\n  steps: TrajectoryStep[];\n  finalAnswer?: string;\n  /** Terminal page observation captured after the agent finishes. */\n  finalObservation?: ProbeEvidence;\n  status: TrajectoryStatus;\n  usage: TrajectoryUsage;\n}\n\n/** Score for a single rubric criterion after evidence analysis + rescoring. */\nexport interface CriterionScore {\n  /** Matches RubricCriterion.criterion (the criterion's short name). */\n  criterion: string;\n  /** Maximum possible points for this criterion. */\n  maxPoints: number;\n  /**\n   * Points earned post-evidence-analysis (paper's post_image_earned_points).\n   * Null if the criterion was conditional and its condition wasn't met (excluded\n   * from both numerator and denominator in the process score).\n   */\n  earnedPoints: number | null;\n  /** Verifier's explanation for the score. */\n  explanation: string;\n  /**\n   * True if the criterion is conditional and its condition was determined to\n   * be met. Absent for non-conditional criteria.\n   */\n  conditionMet?: boolean;\n  /**\n   * Set when the verifier had no evidence to ground this criterion in either\n   * tier. Per paper §2, treated as uncontrollable failure → full credit, but\n   * surfaced here so dashboards can flag low-confidence results.\n   */\n  evidenceInsufficient?: boolean;\n}\n\n/**\n * First-point-of-failure analysis (paper Step 9a). Identifies the earliest\n * step where the agent's trajectory went off-track, using a structured error\n * taxonomy (7 top-level categories, 1.1–7.4 sub-codes).\n */\nexport interface FirstPointOfFailure {\n  stepIndex: number;\n  /** Sub-code from the error taxonomy (e.g., \"2.3\" for a specific hallucination type). */\n  errorCode: string;\n  /** Top-level category name (Selection, Hallucination, etc.). */\n  category: string;\n  /** Verifier's reasoning for selecting this point. */\n  description?: string;\n}\n\n/**\n * Structured observation surfaced by the verifier that another agent or\n * tooling could act on. Findings are emitted opportunistically by Step 8\n * (outcome verification) when the verifier notices actionable patterns —\n * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.\n *\n * Not produced for every task: when nothing actionable surfaces, the\n * `findings` array on the EvaluationResult is empty. Consumers should treat the\n * field as advisory, not as part of the formal score.\n */\nexport interface VerifierFinding {\n  /**\n   * Category of the observation. Open-ended enum — additional categories may\n   * be added as verifier backends surface new failure modes.\n   */\n  category:\n    | \"agent_tool_usage\" // agent's tool calls had repeated issues (misclicks, wrong args, retries)\n    | \"agent_strategy\" // higher-level planning / decision-making problems\n    | \"rubric_quality\" // criteria were overly strict, ambiguous, or contradictory\n    | \"trajectory_capture\" // gaps in evidence (missing screenshots, empty steps)\n    | \"task_specification\" // task instruction was ambiguous / under- or over-specified\n    | \"verifier_uncertainty\" // verifier itself couldn't confidently decide\n    | \"other\";\n  /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */\n  severity: \"info\" | \"warning\" | \"blocking\";\n  /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */\n  description: string;\n  /**\n   * Optional concrete next action another agent could take. Should be\n   * specific enough that it can be acted on without further reasoning —\n   * e.g., \"Try double_click instead of triple_click to clear placeholder\n   * text on this form field.\"\n   */\n  suggestedAction?: string;\n  /** Step indices in the trajectory where this pattern showed up. */\n  relatedSteps?: number[];\n}\n\n/** Stable debugging summary emitted by verifier backends. */\nexport interface VerifierRawSteps {\n  backend?: \"legacy\" | \"verifier\";\n  primaryIntent?: string;\n  reasoning?: string;\n  rubricSource?: \"precomputed\" | \"generated\" | \"none\";\n  approach?: \"a\" | \"b\";\n  optionalsMode?: \"folded\" | \"separate\" | \"skip\";\n  totalEarned?: number;\n  totalMax?: number;\n  evidenceImages?: number;\n  evidenceTexts?: number;\n  evidenceOriginalScreenshots?: number;\n  legacyEvaluation?: string;\n  screenshotCount?: number;\n}\n\n/** Task-validity classification (paper Step 10). */\nexport interface TaskValidity {\n  /** True if the task is underspecified / has multiple valid interpretations. */\n  isAmbiguous: boolean;\n  /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */\n  isInvalid: boolean;\n  /** Optional sub-codes from the task-classification taxonomy. */\n  ambiguityCodes?: string[];\n  invalidTaskCodes?: string[];\n}\n\n/**\n * Evaluator output. Legacy evaluation may only populate outcome fields; richer\n * verifier backends can also populate process scoring and diagnostics.\n *\n * Process and outcome are deliberately independent when both are present:\n * an agent can follow the right steps but get blocked (high process, low\n * outcome), or succeed through an unexpected path (variable process, high\n * outcome).\n */\nexport interface EvaluationResult {\n  /** Did the agent accomplish the task from the user's perspective? */\n  outcomeSuccess: boolean;\n  /** Human-readable explanation for the outcome. */\n  explanation?: string;\n  /** Aggregated earned/max across applicable criteria, in [0, 1]. */\n  processScore?: number;\n  /** Per-criterion breakdown after rescoring. */\n  perCriterion?: CriterionScore[];\n  /** Step 9a — first step where the trajectory went off-track, if any. */\n  firstPointOfFailure?: FirstPointOfFailure;\n  /** Step 10 — task-itself ambiguity / validity. */\n  taskValidity?: TaskValidity;\n  /**\n   * Ids (RubricCriterion.criterion strings) of criteria where neither tier of\n   * evidence resolved the question. Treated as uncontrollable → full credit,\n   * but flagged here so consumers can decide whether to discount the score.\n   */\n  evidenceInsufficient?: string[];\n  /**\n   * Structured observations from the verifier that a downstream tool or\n   * follow-up agent could act on. Opportunistic — empty when the verifier\n   * doesn't notice anything actionable. Not part of the score; advisory.\n   */\n  findings?: VerifierFinding[];\n  /** Debugging summary from the active evaluator backend. */\n  rawSteps?: VerifierRawSteps;\n}\n\n/**\n * Verifier interface. Implementations consume a Trajectory and return an\n * EvaluationResult — they MUST NOT touch a live browser.\n */\nexport interface Verifier {\n  verify(trajectory: Trajectory): Promise<EvaluationResult>;\n}\n"]}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/types.ts"],"names":[],"mappings":"","sourcesContent":["/**\n * Shared verifier types for trajectories, rubrics, evidence, and results.\n *\n * The verifier consumes saved trajectories instead of a live browser. DOM and\n * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve\n * screenshots sent to the provider plus independent harness probes.\n */\nimport type { LLMClient } from \"../llm/LLMClient.js\";\nimport type { LogLine } from \"../types/public/logs.js\";\n\n/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */\nexport interface TrajectoryUsage {\n  input_tokens: number;\n  output_tokens: number;\n  reasoning_tokens?: number;\n  cached_input_tokens?: number;\n  inference_time_ms?: number;\n}\n\n/** A single criterion in a Stagehand rubric. */\nexport interface RubricCriterion {\n  /** Short name of the criterion (e.g., \"Add ground beef to cart\"). */\n  criterion: string;\n  /** What to evaluate and how to award partial credit. */\n  description: string;\n  /** Maximum points for this criterion. */\n  maxPoints: number;\n  /**\n   * Applicability rule for situational criteria. When this condition is not\n   * met, the criterion is excluded from scoring rather than counted as failed.\n   */\n  condition?: string;\n}\n\n/** A rubric — list of criteria for a task. */\nexport interface Rubric {\n  items: RubricCriterion[];\n}\n\n/**\n * Spec for a single task being verified. Carried both at runtime and into the\n * verifier alongside the trajectory.\n */\nexport interface TaskSpec {\n  /** Stable identifier (e.g., \"united_13\" for WebTailBench, task_id for Mind2Web). */\n  id: string;\n  /** Task instruction shown to the agent. */\n  instruction: string;\n  /** Starting URL, if any. */\n  initUrl?: string;\n  /** Rubric carried by the dataset or generated by a verifier backend. */\n  precomputedRubric?: Rubric;\n  /** Optional reference answer (set when dataset ships one). */\n  expectedAnswer?: string;\n}\n\n/**\n * A single modality unit in tier-1 agent evidence. Mirrors the shape of\n * ModelMessage content parts so we can reproduce what the LLM ingested.\n */\nexport type AgentEvidenceModality =\n  | { type: \"text\"; content: string }\n  | { type: \"image\"; bytes: Buffer; mediaType: string }\n  | { type: \"json\"; content: unknown };\n\n/**\n * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the\n * tool result for this step.\n *\n * Modes:\n *   - CUA: usually a single image modality (the screenshot sent to the provider).\n *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.\n *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.\n */\nexport interface AgentEvidence {\n  modalities: AgentEvidenceModality[];\n}\n\n/**\n * Tier 2 — independent harness probes around this step.\n *\n * If a probe wasn't captured, the field is absent (not null).\n */\nexport interface ProbeEvidence {\n  /** URL after the step's tool execution. */\n  url?: string;\n  /**\n   * Bus screenshot captured after the step. Path on disk is preferred once\n   * persisted; in-memory Buffer is used during a live run.\n   */\n  screenshot?: Buffer;\n  /** Reference to the persisted screenshot file under the trajectory dir. */\n  screenshotPath?: string;\n  /** Accessibility tree snapshot. */\n  ariaTree?: string;\n  /** Verifier-requested probes, keyed by criterion id. */\n  onDemand?: Record<string, unknown>;\n}\n\n/** Outcome of a single tool execution as seen by the harness. */\nexport interface ToolOutput {\n  ok: boolean;\n  /**\n   * The tool's return value. Same payload that flowed into agentEvidence\n   * modalities, but in its native shape (e.g., the extract result, the act\n   * describe-string) rather than serialized for the LLM.\n   */\n  result: unknown;\n  error?: string;\n}\n\n/** One step in a trajectory: action + reasoning + evidence + outcome. */\nexport interface TrajectoryStep {\n  actionName: string;\n  actionArgs: Record<string, unknown>;\n  /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */\n  reasoning: string;\n  agentEvidence: AgentEvidence;\n  probeEvidence: ProbeEvidence;\n  toolOutput: ToolOutput;\n}\n\n/** Terminal status of the agent run. */\nexport type TrajectoryStatus = \"complete\" | \"aborted\" | \"stalled\" | \"error\";\n\n/**\n * Full trajectory for one task run.\n *\n * The on-disk layout is one directory per task:\n *\n *   .trajectories/<run-id>/<task-id>/\n *     ├── task_data.json           — TaskSpec + result metadata\n *     ├── trajectory.json          — this object, with image paths instead of bytes\n *     ├── screenshots/\n *     │   ├── probe/<N>.png        — tier-2 probe screenshot per step\n *     │   ├── probe/final.png      — final terminal observation screenshot\n *     │   └── agent/<N>.png        — tier-1 image the model received per step\n *     ├── scores/\n *     │   └── result.json          — Result from V3Evaluator.verify()\n *     └── core.log                 — captured action log\n */\nexport interface Trajectory {\n  task: TaskSpec;\n  steps: TrajectoryStep[];\n  finalAnswer?: string;\n  /** Terminal page observation captured after the agent finishes. */\n  finalObservation?: ProbeEvidence;\n  status: TrajectoryStatus;\n  usage: TrajectoryUsage;\n}\n\n/** A single screenshot kept by Step 1, ready for downstream relevance scoring. */\nexport interface CanonicalScreenshot {\n  /** 0-based position in the kept-screenshots array. Stable across the pipeline. */\n  canonicalIndex: number;\n  /**\n   * 0-based position in `Trajectory.steps` this screenshot came from\n   * (steps.length for the final observation). Lets downstream prompts\n   * cross-reference the action history.\n   */\n  originalStepIndex: number;\n  /** The resized PNG/JPEG buffer (or native bytes if sharp unavailable). */\n  bytes: Buffer;\n  /** MIME media type. Always \"image/png\" after the optional resize. */\n  mediaType: string;\n  /** Reason this frame was kept: \"first\" / \"last\" / \"diverges\". */\n  keptReason: \"first\" | \"last\" | \"diverges\" | \"no-dedup\";\n}\n\n/**\n * A text evidence point sourced from tier-2 probes or tier-1 tool outputs.\n * These feed the same relevance + scoring path as screenshots, letting DOM\n * and hybrid agents preserve extract/aria/tool-return evidence without a\n * separate verifier architecture.\n */\nexport interface CanonicalTextEvidence {\n  /** 0-based position in the combined evidence-point array. */\n  canonicalIndex: number;\n  originalStepIndex: number;\n  /** Where the text came from. */\n  source: \"probe-aria\" | \"agent-text\" | \"agent-json\" | \"tool-output\";\n  /** The text payload, already truncated. */\n  content: string;\n}\n\nexport type CanonicalEvidence = CanonicalScreenshot | CanonicalTextEvidence;\n\n/** Result of Step 1 evidence loading. */\nexport interface EvidenceLoadResult {\n  /** Kept frames, in chronological order. */\n  screenshots: CanonicalScreenshot[];\n  /**\n   * Maps trajectory step position → canonical index in `screenshots`. Steps\n   * whose screenshots were deduplicated point to the surviving canonical frame\n   * (typically the prior kept frame). Useful for \"find me the screenshot for\n   * step K\" lookups in downstream prompts.\n   */\n  stepIndexToCanonical: Map<number, number>;\n  /** Number of original frames considered. */\n  originalCount: number;\n  /** Number of frames kept post-dedup (== screenshots.length). */\n  keptCount: number;\n  /** Effective thresholds used (resolved from env). */\n  thresholds: {\n    ssim: number;\n    mse: number;\n    resize: number;\n  };\n}\n\n/** Options for evidence loading; primarily test seams over the defaults. */\nexport interface EvidenceLoadOptions {\n  /** SSIM similarity threshold for dedup (default 0.75). */\n  ssimThreshold?: number;\n  /** MSE similarity threshold for dedup (default 30). */\n  mseThreshold?: number;\n  /** Scale factor applied before relevance scoring (default 0.7). */\n  imageResize?: number;\n}\n\n/** Score for a single rubric criterion after evidence analysis + rescoring. */\nexport interface CriterionScore {\n  /** Matches RubricCriterion.criterion (the criterion's short name). */\n  criterion: string;\n  /** Maximum possible points for this criterion. */\n  maxPoints: number;\n  /**\n   * Points earned after evidence analysis. Null when the criterion is\n   * conditional and its condition was not met — excluded from both numerator\n   * and denominator in the process score.\n   */\n  earnedPoints: number | null;\n  /** Verifier's explanation for the score. */\n  explanation: string;\n  /**\n   * True if the criterion is conditional and its condition was determined to\n   * be met. Absent for non-conditional criteria.\n   */\n  conditionMet?: boolean;\n  /**\n   * Set when the verifier had no evidence to ground this criterion in either\n   * tier. Treated as uncontrollable failure (full credit) but surfaced here\n   * so dashboards can flag low-confidence results.\n   */\n  evidenceInsufficient?: boolean;\n}\n\n/**\n * Earliest step where the agent's trajectory went off-track, classified\n * against the error taxonomy (7 top-level categories, 1.1–7.4 sub-codes).\n */\nexport interface FirstPointOfFailure {\n  stepIndex: number;\n  /** Sub-code from the error taxonomy (e.g., \"2.3\" for a specific hallucination type). */\n  errorCode: string;\n  /** Top-level category name (Selection, Hallucination, etc.). */\n  category: string;\n  /** Verifier's reasoning for selecting this point. */\n  description?: string;\n}\n\n/**\n * Structured observation surfaced by the verifier that another agent or\n * tooling could act on. Findings are emitted opportunistically by Step 8\n * (outcome verification) when the verifier notices actionable patterns —\n * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.\n *\n * Not produced for every task: when nothing actionable surfaces, the\n * `findings` array on the EvaluationResult is empty. Consumers should treat the\n * field as advisory, not as part of the formal score.\n */\nexport interface VerifierFinding {\n  /**\n   * Category of the observation. Open-ended enum — additional categories may\n   * be added as verifier backends surface new failure modes.\n   */\n  category:\n    | \"agent_tool_usage\" // agent's tool calls had repeated issues (misclicks, wrong args, retries)\n    | \"agent_strategy\" // higher-level planning / decision-making problems\n    | \"rubric_quality\" // criteria were overly strict, ambiguous, or contradictory\n    | \"trajectory_capture\" // gaps in evidence (missing screenshots, empty steps)\n    | \"task_specification\" // task instruction was ambiguous / under- or over-specified\n    | \"verifier_uncertainty\" // verifier itself couldn't confidently decide\n    | \"other\";\n  /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */\n  severity: \"info\" | \"warning\" | \"blocking\";\n  /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */\n  description: string;\n  /**\n   * Optional concrete next action another agent could take. Should be\n   * specific enough that it can be acted on without further reasoning —\n   * e.g., \"Try double_click instead of triple_click to clear placeholder\n   * text on this form field.\"\n   */\n  suggestedAction?: string;\n  /** Step indices in the trajectory where this pattern showed up. */\n  relatedSteps?: number[];\n}\n\n/** Stable debugging summary emitted by verifier backends. */\nexport interface VerifierRawSteps {\n  backend?: \"legacy\" | \"verifier\";\n  reason?: string;\n  primaryIntent?: string;\n  reasoning?: string;\n  rubricSource?: \"precomputed\" | \"generated\" | \"none\";\n  approach?: \"a\" | \"b\" | \"outcome-only\";\n  optionalsMode?: \"folded\" | \"separate\" | \"skip\";\n  totalEarned?: number;\n  totalMax?: number;\n  evidenceImages?: number;\n  evidenceTexts?: number;\n  evidenceOriginalScreenshots?: number;\n  screenshotsAttached?: number;\n  legacyEvaluation?: string;\n  screenshotCount?: number;\n}\n\n/** Task-validity classification: whether the task is even answerable. */\nexport interface TaskValidity {\n  /** True if the task is underspecified / has multiple valid interpretations. */\n  isAmbiguous: boolean;\n  /** Explanation for why the task is ambiguous, when available. */\n  ambiguityReason?: string;\n  /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */\n  isInvalid: boolean;\n  /** Explanation for why the task is invalid, when available. */\n  invalidReason?: string;\n  /** Optional sub-codes from the task-classification taxonomy. */\n  ambiguityCodes?: string[];\n  invalidTaskCodes?: string[];\n}\n\n/**\n * Evaluator output. Legacy evaluation may only populate outcome fields; richer\n * verifier backends can also populate process scoring and diagnostics.\n *\n * Process and outcome are deliberately independent when both are present:\n * an agent can follow the right steps but get blocked (high process, low\n * outcome), or succeed through an unexpected path (variable process, high\n * outcome).\n */\nexport interface EvaluationResult {\n  /** Did the agent accomplish the task from the user's perspective? */\n  outcomeSuccess: boolean;\n  /** Human-readable explanation for the outcome. */\n  explanation?: string;\n  /** Aggregated earned/max across applicable criteria, in [0, 1]. */\n  processScore?: number;\n  /** Per-criterion breakdown after rescoring. */\n  perCriterion?: CriterionScore[];\n  /** Step 9a — first step where the trajectory went off-track, if any. */\n  firstPointOfFailure?: FirstPointOfFailure;\n  /** Step 10 — task-itself ambiguity / validity. */\n  taskValidity?: TaskValidity;\n  /**\n   * Ids (RubricCriterion.criterion strings) of criteria where neither tier of\n   * evidence resolved the question. Treated as uncontrollable → full credit,\n   * but flagged here so consumers can decide whether to discount the score.\n   */\n  evidenceInsufficient?: string[];\n  /**\n   * Structured observations from the verifier that a downstream tool or\n   * follow-up agent could act on. Opportunistic — empty when the verifier\n   * doesn't notice anything actionable. Not part of the score; advisory.\n   */\n  findings?: VerifierFinding[];\n  /** Debugging summary from the active evaluator backend. */\n  rawSteps?: VerifierRawSteps;\n}\n\n/**\n * Verifier interface. Implementations consume a Trajectory and return an\n * EvaluationResult — they MUST NOT touch a live browser.\n */\nexport interface Verifier {\n  verify(trajectory: Trajectory): Promise<EvaluationResult>;\n}\n\nexport interface RubricVerifierOptions {\n  /** Factory that returns a configured LLMClient. Called per pipeline step so callers can supply step-specific clients. */\n  getClient: () => LLMClient;\n  /** Optional factory for rubric generation so callers can route it to a stronger model. */\n  getRubricGenClient?: () => LLMClient;\n  /** Logger; defaults to a no-op so the verifier stays quiet inside V3Evaluator. */\n  logger?: (line: LogLine) => void;\n  /**\n   * Override any verifier knob. Env vars supply the defaults; values here win.\n   * Useful for tests and for cross-verify sweeps that want different budgets\n   * per run.\n   */\n  config?: Partial<VerifierConfig>;\n}\n\n/**\n * Resolved verifier knobs. Constructed once from env (and optional overrides)\n * by RubricVerifier's constructor; subsequent verify() calls can pass a\n * Partial to shift any field.\n */\nexport interface VerifierConfig {\n  /** Which pipeline path to take: per-criterion (a), fused (b), or skip rubric entirely (outcome-only). */\n  approach: \"a\" | \"b\" | \"outcome-only\";\n  /** Folded (in fused call), separate (own calls), or skip (omit). */\n  optionalSteps: \"folded\" | \"separate\" | \"skip\";\n  /** Top-K evidence points selected per criterion. */\n  topK: number;\n  /** Batch size for the relevance-scoring LLM call. */\n  relevanceBatchSize: number;\n  /** Image cap on the outcome-only path. */\n  outcomeMaxImages: number;\n  /** Concurrent LLM calls across batches / criteria. */\n  maxParallel: number;\n  /** Token budgets for the three evidence channels. */\n  evidenceTokenBudget: number;\n  outcomeEvidenceTokenBudget: number;\n  actionHistoryTokenBudget: number;\n  /** Per-section character limits applied during evidence-text assembly. */\n  truncation: {\n    /** Master switch: when true, all per-section limits go to MAX_SAFE_INTEGER. */\n    disabled: boolean;\n    evidenceTextPreview: number;\n    groupedEvidenceText: number;\n    buildEvidenceText: number;\n    buildEvidenceAria: number;\n    actionHistoryReasoning: number;\n  };\n}\n\nexport interface ErrorTaxonomySubCategory {\n  /** Sub-code (e.g., \"2.3\"). */\n  code: string;\n  /** Human-readable name (e.g., \"Output fabrication\"). */\n  name: string;\n  /** Detailed description ported from the .md. Markdown formatting preserved. */\n  description: string;\n}\n\nexport interface ErrorTaxonomyCategory {\n  /** Top-level number (1-8). */\n  number: number;\n  /** Top-level name (e.g., \"Hallucination Errors\"). */\n  name: string;\n  /** One-sentence summary of the category. */\n  summary: string;\n  /** Sub-categories. The last one is always an \"Other\" catch-all. */\n  subCategories: ErrorTaxonomySubCategory[];\n}\n\nexport interface ParseFailureStepNumbersOptions {\n  /**\n   * Maximum unique step numbers to expand from ranges. Protects the verifier\n   * from malformed model output such as \"0-2147483647\".\n   */\n  maxExpandedSteps?: number;\n  /** Optional inclusive upper bound for accepted step numbers. */\n  maxStep?: number;\n}\n"]}

package/dist/esm/lib/v3Evaluator.d.ts

@@ -19,7 +19,10 @@ export type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {
     modelClientOptions?: ClientOptions;
 };
 export declare class V3Evaluator implements Verifier {
+    private readonly v3;
     private readonly backend;
+    private readonly modelName;
+    private readonly modelClientOptions;
     private readonly legacyEvaluator;
     constructor(v3: V3, modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions, modelClientOptions?: ClientOptions, options?: V3EvaluatorOptions);
     ask(options: EvaluateOptions): Promise<LegacyEvaluationResult>;
@@ -28,5 +31,7 @@ export declare class V3Evaluator implements Verifier {
     generateRubric(taskSpec: TaskSpec): Promise<Rubric>;
     private getLegacyBackend;
     private unavailableVerifierBackend;
+    private getClient;
+    private getRubricGenClient;
     private verifyTrajectoryWithLegacyEvaluator;
 }

package/dist/esm/lib/v3Evaluator.js

@@ -1,13 +1,27 @@
+import { LLMProvider } from "./v3/llm/LLMProvider.js";
 import { StagehandInvalidArgumentError } from "./v3/types/public/sdkErrors.js";
 import { LegacyV3Evaluator } from "./v3LegacyEvaluator.js";
+import { RubricVerifier } from "./v3/verifier/rubricVerifier.js";
 const EVALUATOR_BACKEND_ENV = "STAGEHAND_EVALUATOR_BACKEND";
 const DEFAULT_EVALUATOR_BACKEND = "legacy";
 export class V3Evaluator {
+    v3;
     backend;
+    modelName;
+    modelClientOptions;
     legacyEvaluator;
     constructor(v3, modelNameOrOptions, modelClientOptions, options) {
         const normalizedOptions = normalizeConstructorOptions(modelNameOrOptions, modelClientOptions, options);
         this.backend = resolveEvaluatorBackend(normalizedOptions.backend);
+        this.v3 = v3;
+        this.modelName =
+            normalizedOptions.modelName ||
+                "google/gemini-2.5-flash";
+        this.modelClientOptions = normalizedOptions.modelClientOptions || {
+            apiKey: process.env.GEMINI_API_KEY ||
+                process.env.GOOGLE_GENERATIVE_AI_API_KEY ||
+                "",
+        };
         this.legacyEvaluator = new LegacyV3Evaluator(v3, normalizedOptions.modelName, normalizedOptions.modelClientOptions);
     }
     async ask(options) {
@@ -21,18 +35,26 @@ export class V3Evaluator {
         if (this.backend === "legacy") {
             return this.verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec);
         }
-        return this.unavailableVerifierBackend("verify");
+        const verifier = new RubricVerifier({
+            getClient: () => this.getClient(),
+            getRubricGenClient: () => this.getRubricGenClient(),
+        });
+        return verifier.verify(trajectory);
     }
     async generateRubric(taskSpec) {
         if (!taskSpec?.id) {
             throw new StagehandInvalidArgumentError("TaskSpec.id is required for rubric generation");
         }
-        if (this.backend === "verifier") {
-            return this.unavailableVerifierBackend("generateRubric");
+        if (this.backend === "legacy") {
+            return {
+                items: [legacyTaskCompletionCriterion(taskSpec)],
+            };
         }
-        return {
-            items: [legacyTaskCompletionCriterion(taskSpec)],
-        };
+        const verifier = new RubricVerifier({
+            getClient: () => this.getClient(),
+            getRubricGenClient: () => this.getRubricGenClient(),
+        });
+        return verifier.generateRubric(taskSpec);
     }
     getLegacyBackend(methodName) {
         if (this.backend === "legacy") {
@@ -41,7 +63,25 @@ export class V3Evaluator {
         return this.unavailableVerifierBackend(methodName);
     }
     unavailableVerifierBackend(methodName) {
-        throw new StagehandInvalidArgumentError(`V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use "legacy" or install the verifier backend PR.`);
+        throw new StagehandInvalidArgumentError(`V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend only supports verify() and generateRubric(). Use "legacy" for ask()/batchAsk().`);
+    }
+    getClient() {
+        const provider = new LLMProvider(this.v3.logger);
+        return provider.getClient(this.modelName, this.modelClientOptions);
+    }
+    getRubricGenClient() {
+        const override = process.env.VERIFIER_RUBRIC_MODEL;
+        if (!override)
+            return this.getClient();
+        const provider = new LLMProvider(this.v3.logger);
+        const overrideProvider = override.includes("/")
+            ? override.slice(0, override.indexOf("/"))
+            : undefined;
+        const defaultProvider = this.modelName.includes("/")
+            ? this.modelName.slice(0, this.modelName.indexOf("/"))
+            : undefined;
+        const sameProvider = overrideProvider !== undefined && overrideProvider === defaultProvider;
+        return provider.getClient(override, sameProvider ? this.modelClientOptions : undefined);
     }
     async verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec) {
         const screenshots = collectLegacyScreenshots(trajectory);

package/dist/esm/lib/v3Evaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAW3D,MAAM,qBAAqB,GAAG,6BAA6B,CAAC;AAC5D,MAAM,yBAAyB,GAAuB,QAAQ,CAAC;AA2B/D,MAAM,OAAO,WAAW;IACL,OAAO,CAAqB;IAC5B,eAAe,CAAoB;IAEpD,YACE,EAAM,EACN,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;QAE5B,MAAM,iBAAiB,GAAG,2BAA2B,CACnD,kBAAkB,EAClB,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAC1C,EAAE,EACF,iBAAiB,CAAC,SAAS,EAC3B,iBAAiB,CAAC,kBAAkB,CACrC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAsB;QACjC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,mCAAmC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAED,OAAO,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,6BAA6B,CACrC,+CAA+C,CAChD,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO;YACL,KAAK,EAAE,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC;SACjD,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAEO,0BAA0B,CAAC,UAAkB;QACnD,MAAM,IAAI,6BAA6B,CACrC,eAAe,UAAU,0BAA0B,qBAAqB,sHAAsH,CAC/L,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mCAAmC,CAC/C,UAAsB,EACtB,QAAkB;QAElB,MAAM,WAAW,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC;QAEtC,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,OAAO,gCAAgC,CACrC,qFAAqF,CACtF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;YAC5C,QAAQ,EAAE,QAAQ,CAAC,WAAW;YAC9B,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK;YACpD,MAAM;YACN,cAAc;SACf,CAAC,CAAC;QAEH,OAAO,wBAAwB,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9D,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;IAE5B,IACE,kBAAkB;QAClB,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,CAAC;QACD,OAAO;YACL,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,kBAAkB,EAAE,kBAAkB,CAAC,kBAAkB;YACzD,OAAO,EAAE,kBAAkB,CAAC,OAAO,IAAI,OAAO,EAAE,OAAO;SACxD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAgD;QAC3D,kBAAkB;QAClB,OAAO,EAAE,OAAO,EAAE,OAAO;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,eAAoC;IAEpC,MAAM,iBAAiB,GACrB,eAAe;QACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;QAClC,yBAAyB,CAAC;IAC5B,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEjE,IAAI,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,KAAK,UAAU,EAAE,CAAC;QACvE,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,6BAA6B,CACrC,WAAW,qBAAqB,KAAK,iBAAiB,qCAAqC,CAC5F,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAsB;IACjD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,6BAA6B,CACrC,yCAAyC,CAC1C,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;QACzB,MAAM,IAAI,6BAA6B,CACrC,iDAAiD,CAClD,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC;AACzB,CAAC;AAED,SAAS,6BAA6B,CAAC,QAAkB;IACvD,OAAO;QACL,SAAS,EAAE,wBAAwB;QACnC,WAAW,EAAE,yDAAyD,QAAQ,CAAC,WAAW,EAAE;QAC5F,SAAS,EAAE,CAAC;KACb,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAsB;IACtD,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,CAAC;YACpD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,IAAI,CACrD,CACE,QAAQ,EACuD,EAAE,CACjE,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC/D,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAAE,CAAC;QAC7D,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,0BAA0B,CACjC,UAAsB;IAEtB,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK;YACnC,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE;YACxC,CAAC,CAAC,gBAAgB,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;QAClE,OAAO;YACL,QAAQ,CAAC,KAAK,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;YAC3D,MAAM,IAAI,SAAS;YACnB,MAAM;SACP;aACE,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,iBAAiB,CACtB,sBAAsB,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAC9C,KAAK,CACN,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAE,SAAiB;IACzD,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;AACvD,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAA8B,EAC9B,eAAuB;IAEvB,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC;IAChD,MAAM,QAAQ,GAAsB,OAAO;QACzC,CAAC,CAAC;YACE;gBACE,QAAQ,EAAE,sBAAsB;gBAChC,QAAQ,EAAE,SAAS;gBACnB,WAAW,EAAE,MAAM,CAAC,SAAS;aAC9B;SACF;QACH,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;QACL,cAAc;QACd,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,MAAM,CAAC,UAAU;YACnC,eAAe;SAChB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,gCAAgC,CAAC,MAAc;IACtD,OAAO;QACL,cAAc,EAAE,KAAK;QACrB,WAAW,EAAE,MAAM;QACnB,QAAQ,EAAE;YACR;gBACE,QAAQ,EAAE,oBAAoB;gBAC9B,QAAQ,EAAE,UAAU;gBACpB,WAAW,EAAE,MAAM;aACpB;SACF;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,SAAS;YAC3B,eAAe,EAAE,CAAC;SACnB;KACF,CAAC;AACJ,CAAC","sourcesContent":["import type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult as LegacyEvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\nimport { LegacyV3Evaluator } from \"./v3LegacyEvaluator.js\";\nimport type {\n  Trajectory,\n  TaskSpec,\n  EvaluationResult,\n  Rubric,\n  Verifier,\n  AgentEvidenceModality,\n  VerifierFinding,\n} from \"./v3/verifier/index.js\";\n\nconst EVALUATOR_BACKEND_ENV = \"STAGEHAND_EVALUATOR_BACKEND\";\nconst DEFAULT_EVALUATOR_BACKEND: V3EvaluatorBackend = \"legacy\";\n\nexport type V3EvaluatorBackend = \"legacy\" | \"verifier\";\n\nexport type V3EvaluatorOptions = {\n  /**\n   * Selects the evaluator implementation.\n   *\n   * \"legacy\" preserves the existing screenshot/text YES/NO evaluator.\n   * \"verifier\" is reserved for the rubric verifier backend.\n   *\n   * @default process.env.STAGEHAND_EVALUATOR_BACKEND || \"legacy\"\n   */\n  backend?: V3EvaluatorBackend;\n};\n\nexport type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n};\n\ntype NormalizedConstructorOptions = {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n  backend?: V3EvaluatorBackend;\n};\n\nexport class V3Evaluator implements Verifier {\n  private readonly backend: V3EvaluatorBackend;\n  private readonly legacyEvaluator: LegacyV3Evaluator;\n\n  constructor(\n    v3: V3,\n    modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n    modelClientOptions?: ClientOptions,\n    options?: V3EvaluatorOptions,\n  ) {\n    const normalizedOptions = normalizeConstructorOptions(\n      modelNameOrOptions,\n      modelClientOptions,\n      options,\n    );\n\n    this.backend = resolveEvaluatorBackend(normalizedOptions.backend);\n    this.legacyEvaluator = new LegacyV3Evaluator(\n      v3,\n      normalizedOptions.modelName,\n      normalizedOptions.modelClientOptions,\n    );\n  }\n\n  async ask(options: EvaluateOptions): Promise<LegacyEvaluationResult> {\n    return this.getLegacyBackend(\"ask\").ask(options);\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<LegacyEvaluationResult[]> {\n    return this.getLegacyBackend(\"batchAsk\").batchAsk(options);\n  }\n\n  async verify(trajectory: Trajectory): Promise<EvaluationResult> {\n    const taskSpec = assertVerifierInput(trajectory);\n\n    if (this.backend === \"legacy\") {\n      return this.verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec);\n    }\n\n    return this.unavailableVerifierBackend(\"verify\");\n  }\n\n  async generateRubric(taskSpec: TaskSpec): Promise<Rubric> {\n    if (!taskSpec?.id) {\n      throw new StagehandInvalidArgumentError(\n        \"TaskSpec.id is required for rubric generation\",\n      );\n    }\n\n    if (this.backend === \"verifier\") {\n      return this.unavailableVerifierBackend(\"generateRubric\");\n    }\n\n    return {\n      items: [legacyTaskCompletionCriterion(taskSpec)],\n    };\n  }\n\n  private getLegacyBackend(methodName: string): LegacyV3Evaluator {\n    if (this.backend === \"legacy\") {\n      return this.legacyEvaluator;\n    }\n\n    return this.unavailableVerifierBackend(methodName);\n  }\n\n  private unavailableVerifierBackend(methodName: string): never {\n    throw new StagehandInvalidArgumentError(\n      `V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend is not available in this build. Use \"legacy\" or install the verifier backend PR.`,\n    );\n  }\n\n  private async verifyTrajectoryWithLegacyEvaluator(\n    trajectory: Trajectory,\n    taskSpec: TaskSpec,\n  ): Promise<EvaluationResult> {\n    const screenshots = collectLegacyScreenshots(trajectory);\n    const agentReasoning = renderLegacyAgentReasoning(trajectory);\n    const answer = trajectory.finalAnswer;\n\n    if (!screenshots.length && !answer) {\n      return legacyInsufficientEvidenceResult(\n        \"Legacy evaluator compatibility mode had no screenshots or final answer to evaluate.\",\n      );\n    }\n\n    const result = await this.legacyEvaluator.ask({\n      question: taskSpec.instruction,\n      screenshot: screenshots.length ? screenshots : false,\n      answer,\n      agentReasoning,\n    });\n\n    return legacyEvaluationToResult(result, screenshots.length);\n  }\n}\n\nfunction normalizeConstructorOptions(\n  modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n  modelClientOptions?: ClientOptions,\n  options?: V3EvaluatorOptions,\n): NormalizedConstructorOptions {\n  if (\n    modelNameOrOptions &&\n    typeof modelNameOrOptions === \"object\" &&\n    !Array.isArray(modelNameOrOptions)\n  ) {\n    return {\n      modelName: modelNameOrOptions.modelName,\n      modelClientOptions: modelNameOrOptions.modelClientOptions,\n      backend: modelNameOrOptions.backend ?? options?.backend,\n    };\n  }\n\n  return {\n    modelName: modelNameOrOptions as AvailableModel | undefined,\n    modelClientOptions,\n    backend: options?.backend,\n  };\n}\n\nfunction resolveEvaluatorBackend(\n  explicitBackend?: V3EvaluatorBackend,\n): V3EvaluatorBackend {\n  const configuredBackend =\n    explicitBackend ??\n    process.env[EVALUATOR_BACKEND_ENV] ??\n    DEFAULT_EVALUATOR_BACKEND;\n  const normalizedBackend = configuredBackend.trim().toLowerCase();\n\n  if (normalizedBackend === \"legacy\" || normalizedBackend === \"verifier\") {\n    return normalizedBackend;\n  }\n\n  throw new StagehandInvalidArgumentError(\n    `Invalid ${EVALUATOR_BACKEND_ENV}=\"${configuredBackend}\". Expected \"legacy\" or \"verifier\".`,\n  );\n}\n\nfunction assertVerifierInput(trajectory: Trajectory): TaskSpec {\n  if (!trajectory) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory is required for verification\",\n    );\n  }\n  if (!trajectory.task?.id) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory.task.id is required for verification\",\n    );\n  }\n  return trajectory.task;\n}\n\nfunction legacyTaskCompletionCriterion(taskSpec: TaskSpec) {\n  return {\n    criterion: \"legacy-task-completion\",\n    description: `Evaluate whether the task was completed successfully: ${taskSpec.instruction}`,\n    maxPoints: 1,\n  };\n}\n\nfunction collectLegacyScreenshots(trajectory: Trajectory): Buffer[] {\n  const screenshots: Buffer[] = [];\n\n  for (const step of trajectory.steps ?? []) {\n    if (Buffer.isBuffer(step.probeEvidence?.screenshot)) {\n      screenshots.push(step.probeEvidence.screenshot);\n      continue;\n    }\n\n    const agentImage = step.agentEvidence?.modalities?.find(\n      (\n        modality,\n      ): modality is Extract<AgentEvidenceModality, { type: \"image\" }> =>\n        modality.type === \"image\" && Buffer.isBuffer(modality.bytes),\n    );\n\n    if (agentImage) {\n      screenshots.push(agentImage.bytes);\n    }\n  }\n\n  if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {\n    screenshots.push(trajectory.finalObservation.screenshot);\n  }\n\n  return screenshots;\n}\n\nfunction renderLegacyAgentReasoning(\n  trajectory: Trajectory,\n): string | undefined {\n  const stepLines = (trajectory.steps ?? []).map((step, i) => {\n    const status = step.toolOutput?.ok === false ? \"Tool status: failed\" : \"\";\n    const output = step.toolOutput?.error\n      ? `Tool error: ${step.toolOutput.error}`\n      : `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;\n    return [\n      `Step ${i}: ${step.actionName}`,\n      step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,\n      status || undefined,\n      output,\n    ]\n      .filter(Boolean)\n      .join(\"\\n\");\n  });\n\n  if (!stepLines.length) {\n    return undefined;\n  }\n\n  return truncateForPrompt(\n    `Agent trajectory:\\n${stepLines.join(\"\\n\\n\")}`,\n    16000,\n  );\n}\n\nfunction stringifyForPrompt(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n\n  try {\n    const serialized = JSON.stringify(value);\n    return serialized ?? String(value);\n  } catch {\n    return String(value);\n  }\n}\n\nfunction truncateForPrompt(value: string, maxLength: number): string {\n  if (value.length <= maxLength) {\n    return value;\n  }\n\n  return `${value.slice(0, maxLength)}... [truncated]`;\n}\n\nfunction legacyEvaluationToResult(\n  result: LegacyEvaluationResult,\n  screenshotCount: number,\n): EvaluationResult {\n  const outcomeSuccess = result.evaluation === \"YES\";\n  const invalid = result.evaluation === \"INVALID\";\n  const findings: VerifierFinding[] = invalid\n    ? [\n        {\n          category: \"verifier_uncertainty\",\n          severity: \"warning\",\n          description: result.reasoning,\n        },\n      ]\n    : [];\n\n  return {\n    outcomeSuccess,\n    explanation: result.reasoning,\n    ...(findings.length ? { findings } : {}),\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: result.evaluation,\n      screenshotCount,\n    },\n  };\n}\n\nfunction legacyInsufficientEvidenceResult(reason: string): EvaluationResult {\n  return {\n    outcomeSuccess: false,\n    explanation: reason,\n    findings: [\n      {\n        category: \"trajectory_capture\",\n        severity: \"blocking\",\n        description: reason,\n      },\n    ],\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: \"INVALID\",\n      screenshotCount: 0,\n    },\n  };\n}\n"]}
+{"version":3,"file":"v3Evaluator.js","sourceRoot":"","sources":["../../../lib/v3Evaluator.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAWjE,MAAM,qBAAqB,GAAG,6BAA6B,CAAC;AAC5D,MAAM,yBAAyB,GAAuB,QAAQ,CAAC;AA2B/D,MAAM,OAAO,WAAW;IACL,EAAE,CAAK;IACP,OAAO,CAAqB;IAC5B,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,eAAe,CAAoB;IAEpD,YACE,EAAM,EACN,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;QAE5B,MAAM,iBAAiB,GAAG,2BAA2B,CACnD,kBAAkB,EAClB,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS;YACZ,iBAAiB,CAAC,SAAS;gBAC1B,yBAA4C,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC,kBAAkB,IAAI;YAChE,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;QACF,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAC1C,EAAE,EACF,iBAAiB,CAAC,SAAS,EAC3B,iBAAiB,CAAC,kBAAkB,CACrC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAsB;QACjC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,mCAAmC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE;YACjC,kBAAkB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE;SACpD,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,6BAA6B,CACrC,+CAA+C,CAChD,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO;gBACL,KAAK,EAAE,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE;YACjC,kBAAkB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE;SACpD,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAEO,0BAA0B,CAAC,UAAkB;QACnD,MAAM,IAAI,6BAA6B,CACrC,eAAe,UAAU,0BAA0B,qBAAqB,qHAAqH,CAC9L,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACxB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,qBAEhB,CAAC;QACd,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;YAC7C,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC1C,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACtD,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,YAAY,GAChB,gBAAgB,KAAK,SAAS,IAAI,gBAAgB,KAAK,eAAe,CAAC;QAEzE,OAAO,QAAQ,CAAC,SAAS,CACvB,QAAQ,EACR,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CACnD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mCAAmC,CAC/C,UAAsB,EACtB,QAAkB;QAElB,MAAM,WAAW,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC;QAEtC,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,OAAO,gCAAgC,CACrC,qFAAqF,CACtF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;YAC5C,QAAQ,EAAE,QAAQ,CAAC,WAAW;YAC9B,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK;YACpD,MAAM;YACN,cAAc;SACf,CAAC,CAAC;QAEH,OAAO,wBAAwB,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9D,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,kBAAmE,EACnE,kBAAkC,EAClC,OAA4B;IAE5B,IACE,kBAAkB;QAClB,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,CAAC;QACD,OAAO;YACL,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,kBAAkB,EAAE,kBAAkB,CAAC,kBAAkB;YACzD,OAAO,EAAE,kBAAkB,CAAC,OAAO,IAAI,OAAO,EAAE,OAAO;SACxD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAgD;QAC3D,kBAAkB;QAClB,OAAO,EAAE,OAAO,EAAE,OAAO;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,eAAoC;IAEpC,MAAM,iBAAiB,GACrB,eAAe;QACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;QAClC,yBAAyB,CAAC;IAC5B,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEjE,IAAI,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,KAAK,UAAU,EAAE,CAAC;QACvE,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,6BAA6B,CACrC,WAAW,qBAAqB,KAAK,iBAAiB,qCAAqC,CAC5F,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAsB;IACjD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,6BAA6B,CACrC,yCAAyC,CAC1C,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;QACzB,MAAM,IAAI,6BAA6B,CACrC,iDAAiD,CAClD,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC;AACzB,CAAC;AAED,SAAS,6BAA6B,CAAC,QAAkB;IACvD,OAAO;QACL,SAAS,EAAE,wBAAwB;QACnC,WAAW,EAAE,yDAAyD,QAAQ,CAAC,WAAW,EAAE;QAC5F,SAAS,EAAE,CAAC;KACb,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAsB;IACtD,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,CAAC;YACpD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,IAAI,CACrD,CACE,QAAQ,EACuD,EAAE,CACjE,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC/D,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAAE,CAAC;QAC7D,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,0BAA0B,CACjC,UAAsB;IAEtB,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK;YACnC,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE;YACxC,CAAC,CAAC,gBAAgB,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;QAClE,OAAO;YACL,QAAQ,CAAC,KAAK,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;YAC3D,MAAM,IAAI,SAAS;YACnB,MAAM;SACP;aACE,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,iBAAiB,CACtB,sBAAsB,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAC9C,KAAK,CACN,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAE,SAAiB;IACzD,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;AACvD,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAA8B,EAC9B,eAAuB;IAEvB,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC;IAChD,MAAM,QAAQ,GAAsB,OAAO;QACzC,CAAC,CAAC;YACE;gBACE,QAAQ,EAAE,sBAAsB;gBAChC,QAAQ,EAAE,SAAS;gBACnB,WAAW,EAAE,MAAM,CAAC,SAAS;aAC9B;SACF;QACH,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;QACL,cAAc;QACd,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,MAAM,CAAC,UAAU;YACnC,eAAe;SAChB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,gCAAgC,CAAC,MAAc;IACtD,OAAO;QACL,cAAc,EAAE,KAAK;QACrB,WAAW,EAAE,MAAM;QACnB,QAAQ,EAAE;YACR;gBACE,QAAQ,EAAE,oBAAoB;gBAC9B,QAAQ,EAAE,UAAU;gBACpB,WAAW,EAAE,MAAM;aACpB;SACF;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,QAAQ;YACjB,gBAAgB,EAAE,SAAS;YAC3B,eAAe,EAAE,CAAC;SACnB;KACF,CAAC;AACJ,CAAC","sourcesContent":["import type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult as LegacyEvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport type { LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\nimport { LegacyV3Evaluator } from \"./v3LegacyEvaluator.js\";\nimport { RubricVerifier } from \"./v3/verifier/rubricVerifier.js\";\nimport type {\n  Trajectory,\n  TaskSpec,\n  EvaluationResult,\n  Rubric,\n  Verifier,\n  AgentEvidenceModality,\n  VerifierFinding,\n} from \"./v3/verifier/index.js\";\n\nconst EVALUATOR_BACKEND_ENV = \"STAGEHAND_EVALUATOR_BACKEND\";\nconst DEFAULT_EVALUATOR_BACKEND: V3EvaluatorBackend = \"legacy\";\n\nexport type V3EvaluatorBackend = \"legacy\" | \"verifier\";\n\nexport type V3EvaluatorOptions = {\n  /**\n   * Selects the evaluator implementation.\n   *\n   * \"legacy\" preserves the existing screenshot/text YES/NO evaluator.\n   * \"verifier\" is reserved for the rubric verifier backend.\n   *\n   * @default process.env.STAGEHAND_EVALUATOR_BACKEND || \"legacy\"\n   */\n  backend?: V3EvaluatorBackend;\n};\n\nexport type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n};\n\ntype NormalizedConstructorOptions = {\n  modelName?: AvailableModel;\n  modelClientOptions?: ClientOptions;\n  backend?: V3EvaluatorBackend;\n};\n\nexport class V3Evaluator implements Verifier {\n  private readonly v3: V3;\n  private readonly backend: V3EvaluatorBackend;\n  private readonly modelName: AvailableModel;\n  private readonly modelClientOptions: ClientOptions | { apiKey: string };\n  private readonly legacyEvaluator: LegacyV3Evaluator;\n\n  constructor(\n    v3: V3,\n    modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n    modelClientOptions?: ClientOptions,\n    options?: V3EvaluatorOptions,\n  ) {\n    const normalizedOptions = normalizeConstructorOptions(\n      modelNameOrOptions,\n      modelClientOptions,\n      options,\n    );\n\n    this.backend = resolveEvaluatorBackend(normalizedOptions.backend);\n    this.v3 = v3;\n    this.modelName =\n      normalizedOptions.modelName ||\n      (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = normalizedOptions.modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n    this.legacyEvaluator = new LegacyV3Evaluator(\n      v3,\n      normalizedOptions.modelName,\n      normalizedOptions.modelClientOptions,\n    );\n  }\n\n  async ask(options: EvaluateOptions): Promise<LegacyEvaluationResult> {\n    return this.getLegacyBackend(\"ask\").ask(options);\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<LegacyEvaluationResult[]> {\n    return this.getLegacyBackend(\"batchAsk\").batchAsk(options);\n  }\n\n  async verify(trajectory: Trajectory): Promise<EvaluationResult> {\n    const taskSpec = assertVerifierInput(trajectory);\n\n    if (this.backend === \"legacy\") {\n      return this.verifyTrajectoryWithLegacyEvaluator(trajectory, taskSpec);\n    }\n\n    const verifier = new RubricVerifier({\n      getClient: () => this.getClient(),\n      getRubricGenClient: () => this.getRubricGenClient(),\n    });\n    return verifier.verify(trajectory);\n  }\n\n  async generateRubric(taskSpec: TaskSpec): Promise<Rubric> {\n    if (!taskSpec?.id) {\n      throw new StagehandInvalidArgumentError(\n        \"TaskSpec.id is required for rubric generation\",\n      );\n    }\n\n    if (this.backend === \"legacy\") {\n      return {\n        items: [legacyTaskCompletionCriterion(taskSpec)],\n      };\n    }\n\n    const verifier = new RubricVerifier({\n      getClient: () => this.getClient(),\n      getRubricGenClient: () => this.getRubricGenClient(),\n    });\n    return verifier.generateRubric(taskSpec);\n  }\n\n  private getLegacyBackend(methodName: string): LegacyV3Evaluator {\n    if (this.backend === \"legacy\") {\n      return this.legacyEvaluator;\n    }\n\n    return this.unavailableVerifierBackend(methodName);\n  }\n\n  private unavailableVerifierBackend(methodName: string): never {\n    throw new StagehandInvalidArgumentError(\n      `V3Evaluator.${methodName}() was configured with ${EVALUATOR_BACKEND_ENV}=verifier, but the verifier backend only supports verify() and generateRubric(). Use \"legacy\" for ask()/batchAsk().`,\n    );\n  }\n\n  private getClient(): LLMClient {\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  private getRubricGenClient(): LLMClient {\n    const override = process.env.VERIFIER_RUBRIC_MODEL as\n      | AvailableModel\n      | undefined;\n    if (!override) return this.getClient();\n\n    const provider = new LLMProvider(this.v3.logger);\n    const overrideProvider = override.includes(\"/\")\n      ? override.slice(0, override.indexOf(\"/\"))\n      : undefined;\n    const defaultProvider = this.modelName.includes(\"/\")\n      ? this.modelName.slice(0, this.modelName.indexOf(\"/\"))\n      : undefined;\n    const sameProvider =\n      overrideProvider !== undefined && overrideProvider === defaultProvider;\n\n    return provider.getClient(\n      override,\n      sameProvider ? this.modelClientOptions : undefined,\n    );\n  }\n\n  private async verifyTrajectoryWithLegacyEvaluator(\n    trajectory: Trajectory,\n    taskSpec: TaskSpec,\n  ): Promise<EvaluationResult> {\n    const screenshots = collectLegacyScreenshots(trajectory);\n    const agentReasoning = renderLegacyAgentReasoning(trajectory);\n    const answer = trajectory.finalAnswer;\n\n    if (!screenshots.length && !answer) {\n      return legacyInsufficientEvidenceResult(\n        \"Legacy evaluator compatibility mode had no screenshots or final answer to evaluate.\",\n      );\n    }\n\n    const result = await this.legacyEvaluator.ask({\n      question: taskSpec.instruction,\n      screenshot: screenshots.length ? screenshots : false,\n      answer,\n      agentReasoning,\n    });\n\n    return legacyEvaluationToResult(result, screenshots.length);\n  }\n}\n\nfunction normalizeConstructorOptions(\n  modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions,\n  modelClientOptions?: ClientOptions,\n  options?: V3EvaluatorOptions,\n): NormalizedConstructorOptions {\n  if (\n    modelNameOrOptions &&\n    typeof modelNameOrOptions === \"object\" &&\n    !Array.isArray(modelNameOrOptions)\n  ) {\n    return {\n      modelName: modelNameOrOptions.modelName,\n      modelClientOptions: modelNameOrOptions.modelClientOptions,\n      backend: modelNameOrOptions.backend ?? options?.backend,\n    };\n  }\n\n  return {\n    modelName: modelNameOrOptions as AvailableModel | undefined,\n    modelClientOptions,\n    backend: options?.backend,\n  };\n}\n\nfunction resolveEvaluatorBackend(\n  explicitBackend?: V3EvaluatorBackend,\n): V3EvaluatorBackend {\n  const configuredBackend =\n    explicitBackend ??\n    process.env[EVALUATOR_BACKEND_ENV] ??\n    DEFAULT_EVALUATOR_BACKEND;\n  const normalizedBackend = configuredBackend.trim().toLowerCase();\n\n  if (normalizedBackend === \"legacy\" || normalizedBackend === \"verifier\") {\n    return normalizedBackend;\n  }\n\n  throw new StagehandInvalidArgumentError(\n    `Invalid ${EVALUATOR_BACKEND_ENV}=\"${configuredBackend}\". Expected \"legacy\" or \"verifier\".`,\n  );\n}\n\nfunction assertVerifierInput(trajectory: Trajectory): TaskSpec {\n  if (!trajectory) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory is required for verification\",\n    );\n  }\n  if (!trajectory.task?.id) {\n    throw new StagehandInvalidArgumentError(\n      \"Trajectory.task.id is required for verification\",\n    );\n  }\n  return trajectory.task;\n}\n\nfunction legacyTaskCompletionCriterion(taskSpec: TaskSpec) {\n  return {\n    criterion: \"legacy-task-completion\",\n    description: `Evaluate whether the task was completed successfully: ${taskSpec.instruction}`,\n    maxPoints: 1,\n  };\n}\n\nfunction collectLegacyScreenshots(trajectory: Trajectory): Buffer[] {\n  const screenshots: Buffer[] = [];\n\n  for (const step of trajectory.steps ?? []) {\n    if (Buffer.isBuffer(step.probeEvidence?.screenshot)) {\n      screenshots.push(step.probeEvidence.screenshot);\n      continue;\n    }\n\n    const agentImage = step.agentEvidence?.modalities?.find(\n      (\n        modality,\n      ): modality is Extract<AgentEvidenceModality, { type: \"image\" }> =>\n        modality.type === \"image\" && Buffer.isBuffer(modality.bytes),\n    );\n\n    if (agentImage) {\n      screenshots.push(agentImage.bytes);\n    }\n  }\n\n  if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {\n    screenshots.push(trajectory.finalObservation.screenshot);\n  }\n\n  return screenshots;\n}\n\nfunction renderLegacyAgentReasoning(\n  trajectory: Trajectory,\n): string | undefined {\n  const stepLines = (trajectory.steps ?? []).map((step, i) => {\n    const status = step.toolOutput?.ok === false ? \"Tool status: failed\" : \"\";\n    const output = step.toolOutput?.error\n      ? `Tool error: ${step.toolOutput.error}`\n      : `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;\n    return [\n      `Step ${i}: ${step.actionName}`,\n      step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,\n      status || undefined,\n      output,\n    ]\n      .filter(Boolean)\n      .join(\"\\n\");\n  });\n\n  if (!stepLines.length) {\n    return undefined;\n  }\n\n  return truncateForPrompt(\n    `Agent trajectory:\\n${stepLines.join(\"\\n\\n\")}`,\n    16000,\n  );\n}\n\nfunction stringifyForPrompt(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n\n  try {\n    const serialized = JSON.stringify(value);\n    return serialized ?? String(value);\n  } catch {\n    return String(value);\n  }\n}\n\nfunction truncateForPrompt(value: string, maxLength: number): string {\n  if (value.length <= maxLength) {\n    return value;\n  }\n\n  return `${value.slice(0, maxLength)}... [truncated]`;\n}\n\nfunction legacyEvaluationToResult(\n  result: LegacyEvaluationResult,\n  screenshotCount: number,\n): EvaluationResult {\n  const outcomeSuccess = result.evaluation === \"YES\";\n  const invalid = result.evaluation === \"INVALID\";\n  const findings: VerifierFinding[] = invalid\n    ? [\n        {\n          category: \"verifier_uncertainty\",\n          severity: \"warning\",\n          description: result.reasoning,\n        },\n      ]\n    : [];\n\n  return {\n    outcomeSuccess,\n    explanation: result.reasoning,\n    ...(findings.length ? { findings } : {}),\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: result.evaluation,\n      screenshotCount,\n    },\n  };\n}\n\nfunction legacyInsufficientEvidenceResult(reason: string): EvaluationResult {\n  return {\n    outcomeSuccess: false,\n    explanation: reason,\n    findings: [\n      {\n        category: \"trajectory_capture\",\n        severity: \"blocking\",\n        description: reason,\n      },\n    ],\n    rawSteps: {\n      backend: \"legacy\",\n      legacyEvaluation: \"INVALID\",\n      screenshotCount: 0,\n    },\n  };\n}\n"]}

package/dist/esm/lib/version.d.ts

@@ -2,4 +2,4 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export declare const STAGEHAND_VERSION: "3.5.0";
+export declare const STAGEHAND_VERSION: "3.6.0";

package/dist/esm/lib/version.js

@@ -2,5 +2,5 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export const STAGEHAND_VERSION = "3.5.0";
+export const STAGEHAND_VERSION = "3.6.0";
 //# sourceMappingURL=version.js.map

package/dist/esm/lib/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.5.0\" as const;\n"]}
+{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.6.0\" as const;\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserbasehq/orca",
-  "version": "3.5.1-preview.0",
+  "version": "3.6.0-gemini-cua",
   "description": "An AI web browsing framework focused on simplicity and extensibility.",
   "type": "module",
   "main": "./dist/cjs/index.js",
@@ -35,10 +35,10 @@
     "build-dom-scripts": "tsx scripts/build-dom-scripts.ts",
     "build:cjs": "tsx scripts/build-cjs.ts",
     "build:esm": "tsx scripts/build-esm.ts",
-    "build": "pnpm --filter @browserbasehq/stagehand run --parallel \"/^build:(esm|cjs)$/\"",
-    "prepack": "pnpm -w --dir ../.. exec turbo run build --filter=@browserbasehq/stagehand",
+    "build": "pnpm --filter @browserbasehq/orca run --parallel \"/^build:(esm|cjs)$/\"",
+    "prepack": "pnpm -w --dir ../.. exec turbo run build --filter=@browserbasehq/orca",
     "example": "node --import tsx -e \"const args=process.argv.slice(1).filter(a=>a!=='--'); const [p]=args; const n=(p||'example').replace(/^\\.\\//,'').replace(/\\.ts$/i,''); import('node:path').then(path=>import(new URL(path.resolve('examples', n + '.ts'), 'file:')));\" --",
-    "test": "pnpm -w --dir ../.. exec turbo run test:core test:e2e --filter=@browserbasehq/stagehand --",
+    "test": "pnpm -w --dir ../.. exec turbo run test:core test:e2e --filter=@browserbasehq/orca --",
     "test:core": "tsx scripts/test-core.ts",
     "test:e2e": "tsx scripts/test-e2e.ts",
     "format": "prettier --write .",
@@ -101,7 +101,7 @@
   },
   "optionalDependencies": {
     "@ai-sdk/amazon-bedrock": "^3.0.73",
-    "@ai-sdk/anthropic": "^2.0.34",
+    "@ai-sdk/anthropic": "^2.0.81",
     "@ai-sdk/azure": "^2.0.109",
     "@ai-sdk/cerebras": "^1.0.25",
     "@ai-sdk/deepseek": "^1.0.23",
@@ -123,7 +123,7 @@
     "@types/jsdom": "^27.0.0",
     "@types/node": "^20.11.30",
     "@types/ws": "^8.5.13",
-    "@vitest/coverage-v8": "^4.0.8",
+    "@vitest/coverage-v8": "^4.1.9",
     "adm-zip": "^0.5.16",
     "chalk": "^5.4.1",
     "eslint": "10.0.2",
@@ -134,8 +134,8 @@
     "prettier": "^3.2.5",
     "puppeteer-core": "^24.43.0",
     "tsx": "*",
-    "vite": "^7.3.3",
-    "vitest": "^4.1.6",
+    "vite": "^7.3.5",
+    "vitest": "^4.1.9",
     "zod": "^3.25.76 || ^4.2.0"
   },
   "repository": {