@browserbasehq/orca 3.4.0-preview.0 → 3.5.0-preview.1

package/dist/esm/lib/v3/verifier/types.d.ts

@@ -0,0 +1,281 @@
+/**
+ * Shared verifier types for trajectories, rubrics, evidence, and results.
+ *
+ * The verifier consumes saved trajectories instead of a live browser. DOM and
+ * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve
+ * screenshots sent to the provider plus independent harness probes.
+ */
+/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */
+export interface TrajectoryUsage {
+    input_tokens: number;
+    output_tokens: number;
+    reasoning_tokens?: number;
+    cached_input_tokens?: number;
+    inference_time_ms?: number;
+}
+/** A single criterion in a Stagehand rubric. */
+export interface RubricCriterion {
+    /** Short name of the criterion (e.g., "Add ground beef to cart"). */
+    criterion: string;
+    /** What to evaluate and how to award partial credit. */
+    description: string;
+    /** Maximum points for this criterion. */
+    maxPoints: number;
+    /**
+     * Applicability rule for situational criteria. When this condition is not
+     * met, the criterion is excluded from scoring rather than counted as failed.
+     */
+    condition?: string;
+}
+/** A rubric — list of criteria for a task. */
+export interface Rubric {
+    items: RubricCriterion[];
+}
+/**
+ * Spec for a single task being verified. Carried both at runtime and into the
+ * verifier alongside the trajectory.
+ */
+export interface TaskSpec {
+    /** Stable identifier (e.g., "united_13" for WebTailBench, task_id for Mind2Web). */
+    id: string;
+    /** Task instruction shown to the agent. */
+    instruction: string;
+    /** Starting URL, if any. */
+    initUrl?: string;
+    /** Rubric carried by the dataset or generated by a verifier backend. */
+    precomputedRubric?: Rubric;
+    /** Optional reference answer (set when dataset ships one). */
+    expectedAnswer?: string;
+}
+/**
+ * A single modality unit in tier-1 agent evidence. Mirrors the shape of
+ * ModelMessage content parts so we can reproduce what the LLM ingested.
+ */
+export type AgentEvidenceModality = {
+    type: "text";
+    content: string;
+} | {
+    type: "image";
+    bytes: Buffer;
+    mediaType: string;
+} | {
+    type: "json";
+    content: unknown;
+};
+/**
+ * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the
+ * tool result for this step.
+ *
+ * Modes:
+ *   - CUA: usually a single image modality (the screenshot sent to the provider).
+ *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.
+ *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.
+ */
+export interface AgentEvidence {
+    modalities: AgentEvidenceModality[];
+}
+/**
+ * Tier 2 — independent harness probes around this step.
+ *
+ * If a probe wasn't captured, the field is absent (not null).
+ */
+export interface ProbeEvidence {
+    /** URL after the step's tool execution. */
+    url?: string;
+    /**
+     * Bus screenshot captured after the step. Path on disk is preferred once
+     * persisted; in-memory Buffer is used during a live run.
+     */
+    screenshot?: Buffer;
+    /** Reference to the persisted screenshot file under the trajectory dir. */
+    screenshotPath?: string;
+    /** Accessibility tree snapshot. */
+    ariaTree?: string;
+    /** Verifier-requested probes, keyed by criterion id. */
+    onDemand?: Record<string, unknown>;
+}
+/** Outcome of a single tool execution as seen by the harness. */
+export interface ToolOutput {
+    ok: boolean;
+    /**
+     * The tool's return value. Same payload that flowed into agentEvidence
+     * modalities, but in its native shape (e.g., the extract result, the act
+     * describe-string) rather than serialized for the LLM.
+     */
+    result: unknown;
+    error?: string;
+}
+/** One step in a trajectory: action + reasoning + evidence + outcome. */
+export interface TrajectoryStep {
+    actionName: string;
+    actionArgs: Record<string, unknown>;
+    /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */
+    reasoning: string;
+    agentEvidence: AgentEvidence;
+    probeEvidence: ProbeEvidence;
+    toolOutput: ToolOutput;
+}
+/** Terminal status of the agent run. */
+export type TrajectoryStatus = "complete" | "aborted" | "stalled" | "error";
+/**
+ * Full trajectory for one task run.
+ *
+ * The on-disk layout is one directory per task:
+ *
+ *   .trajectories/<run-id>/<task-id>/
+ *     ├── task_data.json    — TaskSpec + result metadata
+ *     ├── trajectory.json   — this object, with screenshotPath instead of bytes
+ *     ├── screenshots/      — step probe/agent images plus final observation
+ *     ├── scores/
+ *     │   └── result.json       — Result from V3Evaluator.verify()
+ *     └── core.log          — captured action log
+ */
+export interface Trajectory {
+    task: TaskSpec;
+    steps: TrajectoryStep[];
+    finalAnswer?: string;
+    /** Terminal page observation captured after the agent finishes. */
+    finalObservation?: ProbeEvidence;
+    status: TrajectoryStatus;
+    usage: TrajectoryUsage;
+}
+/** Score for a single rubric criterion after evidence analysis + rescoring. */
+export interface CriterionScore {
+    /** Matches RubricCriterion.criterion (the criterion's short name). */
+    criterion: string;
+    /** Maximum possible points for this criterion. */
+    maxPoints: number;
+    /**
+     * Points earned post-evidence-analysis (paper's post_image_earned_points).
+     * Null if the criterion was conditional and its condition wasn't met (excluded
+     * from both numerator and denominator in the process score).
+     */
+    earnedPoints: number | null;
+    /** Verifier's explanation for the score. */
+    explanation: string;
+    /**
+     * True if the criterion is conditional and its condition was determined to
+     * be met. Absent for non-conditional criteria.
+     */
+    conditionMet?: boolean;
+    /**
+     * Set when the verifier had no evidence to ground this criterion in either
+     * tier. Per paper §2, treated as uncontrollable failure → full credit, but
+     * surfaced here so dashboards can flag low-confidence results.
+     */
+    evidenceInsufficient?: boolean;
+}
+/**
+ * First-point-of-failure analysis (paper Step 9a). Identifies the earliest
+ * step where the agent's trajectory went off-track, using a structured error
+ * taxonomy (7 top-level categories, 1.1–7.4 sub-codes).
+ */
+export interface FirstPointOfFailure {
+    stepIndex: number;
+    /** Sub-code from the error taxonomy (e.g., "2.3" for a specific hallucination type). */
+    errorCode: string;
+    /** Top-level category name (Selection, Hallucination, etc.). */
+    category: string;
+    /** Verifier's reasoning for selecting this point. */
+    description?: string;
+}
+/**
+ * Structured observation surfaced by the verifier that another agent or
+ * tooling could act on. Findings are emitted opportunistically by Step 8
+ * (outcome verification) when the verifier notices actionable patterns —
+ * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.
+ *
+ * Not produced for every task: when nothing actionable surfaces, the
+ * `findings` array on the EvaluationResult is empty. Consumers should treat the
+ * field as advisory, not as part of the formal score.
+ */
+export interface VerifierFinding {
+    /**
+     * Category of the observation. Open-ended enum — additional categories may
+     * be added as verifier backends surface new failure modes.
+     */
+    category: "agent_tool_usage" | "agent_strategy" | "rubric_quality" | "trajectory_capture" | "task_specification" | "verifier_uncertainty" | "other";
+    /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */
+    severity: "info" | "warning" | "blocking";
+    /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */
+    description: string;
+    /**
+     * Optional concrete next action another agent could take. Should be
+     * specific enough that it can be acted on without further reasoning —
+     * e.g., "Try double_click instead of triple_click to clear placeholder
+     * text on this form field."
+     */
+    suggestedAction?: string;
+    /** Step indices in the trajectory where this pattern showed up. */
+    relatedSteps?: number[];
+}
+/** Stable debugging summary emitted by verifier backends. */
+export interface VerifierRawSteps {
+    backend?: "legacy" | "verifier";
+    primaryIntent?: string;
+    reasoning?: string;
+    rubricSource?: "precomputed" | "generated" | "none";
+    approach?: "a" | "b";
+    optionalsMode?: "folded" | "separate" | "skip";
+    totalEarned?: number;
+    totalMax?: number;
+    evidenceImages?: number;
+    evidenceTexts?: number;
+    evidenceOriginalScreenshots?: number;
+    legacyEvaluation?: string;
+    screenshotCount?: number;
+}
+/** Task-validity classification (paper Step 10). */
+export interface TaskValidity {
+    /** True if the task is underspecified / has multiple valid interpretations. */
+    isAmbiguous: boolean;
+    /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */
+    isInvalid: boolean;
+    /** Optional sub-codes from the task-classification taxonomy. */
+    ambiguityCodes?: string[];
+    invalidTaskCodes?: string[];
+}
+/**
+ * Evaluator output. Legacy evaluation may only populate outcome fields; richer
+ * verifier backends can also populate process scoring and diagnostics.
+ *
+ * Process and outcome are deliberately independent when both are present:
+ * an agent can follow the right steps but get blocked (high process, low
+ * outcome), or succeed through an unexpected path (variable process, high
+ * outcome).
+ */
+export interface EvaluationResult {
+    /** Did the agent accomplish the task from the user's perspective? */
+    outcomeSuccess: boolean;
+    /** Human-readable explanation for the outcome. */
+    explanation?: string;
+    /** Aggregated earned/max across applicable criteria, in [0, 1]. */
+    processScore?: number;
+    /** Per-criterion breakdown after rescoring. */
+    perCriterion?: CriterionScore[];
+    /** Step 9a — first step where the trajectory went off-track, if any. */
+    firstPointOfFailure?: FirstPointOfFailure;
+    /** Step 10 — task-itself ambiguity / validity. */
+    taskValidity?: TaskValidity;
+    /**
+     * Ids (RubricCriterion.criterion strings) of criteria where neither tier of
+     * evidence resolved the question. Treated as uncontrollable → full credit,
+     * but flagged here so consumers can decide whether to discount the score.
+     */
+    evidenceInsufficient?: string[];
+    /**
+     * Structured observations from the verifier that a downstream tool or
+     * follow-up agent could act on. Opportunistic — empty when the verifier
+     * doesn't notice anything actionable. Not part of the score; advisory.
+     */
+    findings?: VerifierFinding[];
+    /** Debugging summary from the active evaluator backend. */
+    rawSteps?: VerifierRawSteps;
+}
+/**
+ * Verifier interface. Implementations consume a Trajectory and return an
+ * EvaluationResult — they MUST NOT touch a live browser.
+ */
+export interface Verifier {
+    verify(trajectory: Trajectory): Promise<EvaluationResult>;
+}

package/dist/esm/lib/v3/verifier/types.js

@@ -0,0 +1,9 @@
+/**
+ * Shared verifier types for trajectories, rubrics, evidence, and results.
+ *
+ * The verifier consumes saved trajectories instead of a live browser. DOM and
+ * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve
+ * screenshots sent to the provider plus independent harness probes.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/esm/lib/v3/verifier/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG","sourcesContent":["/**\n * Shared verifier types for trajectories, rubrics, evidence, and results.\n *\n * The verifier consumes saved trajectories instead of a live browser. DOM and\n * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve\n * screenshots sent to the provider plus independent harness probes.\n */\n\n/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */\nexport interface TrajectoryUsage {\n  input_tokens: number;\n  output_tokens: number;\n  reasoning_tokens?: number;\n  cached_input_tokens?: number;\n  inference_time_ms?: number;\n}\n\n/** A single criterion in a Stagehand rubric. */\nexport interface RubricCriterion {\n  /** Short name of the criterion (e.g., \"Add ground beef to cart\"). */\n  criterion: string;\n  /** What to evaluate and how to award partial credit. */\n  description: string;\n  /** Maximum points for this criterion. */\n  maxPoints: number;\n  /**\n   * Applicability rule for situational criteria. When this condition is not\n   * met, the criterion is excluded from scoring rather than counted as failed.\n   */\n  condition?: string;\n}\n\n/** A rubric — list of criteria for a task. */\nexport interface Rubric {\n  items: RubricCriterion[];\n}\n\n/**\n * Spec for a single task being verified. Carried both at runtime and into the\n * verifier alongside the trajectory.\n */\nexport interface TaskSpec {\n  /** Stable identifier (e.g., \"united_13\" for WebTailBench, task_id for Mind2Web). */\n  id: string;\n  /** Task instruction shown to the agent. */\n  instruction: string;\n  /** Starting URL, if any. */\n  initUrl?: string;\n  /** Rubric carried by the dataset or generated by a verifier backend. */\n  precomputedRubric?: Rubric;\n  /** Optional reference answer (set when dataset ships one). */\n  expectedAnswer?: string;\n}\n\n/**\n * A single modality unit in tier-1 agent evidence. Mirrors the shape of\n * ModelMessage content parts so we can reproduce what the LLM ingested.\n */\nexport type AgentEvidenceModality =\n  | { type: \"text\"; content: string }\n  | { type: \"image\"; bytes: Buffer; mediaType: string }\n  | { type: \"json\"; content: unknown };\n\n/**\n * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the\n * tool result for this step.\n *\n * Modes:\n *   - CUA: usually a single image modality (the screenshot sent to the provider).\n *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.\n *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.\n */\nexport interface AgentEvidence {\n  modalities: AgentEvidenceModality[];\n}\n\n/**\n * Tier 2 — independent harness probes around this step.\n *\n * If a probe wasn't captured, the field is absent (not null).\n */\nexport interface ProbeEvidence {\n  /** URL after the step's tool execution. */\n  url?: string;\n  /**\n   * Bus screenshot captured after the step. Path on disk is preferred once\n   * persisted; in-memory Buffer is used during a live run.\n   */\n  screenshot?: Buffer;\n  /** Reference to the persisted screenshot file under the trajectory dir. */\n  screenshotPath?: string;\n  /** Accessibility tree snapshot. */\n  ariaTree?: string;\n  /** Verifier-requested probes, keyed by criterion id. */\n  onDemand?: Record<string, unknown>;\n}\n\n/** Outcome of a single tool execution as seen by the harness. */\nexport interface ToolOutput {\n  ok: boolean;\n  /**\n   * The tool's return value. Same payload that flowed into agentEvidence\n   * modalities, but in its native shape (e.g., the extract result, the act\n   * describe-string) rather than serialized for the LLM.\n   */\n  result: unknown;\n  error?: string;\n}\n\n/** One step in a trajectory: action + reasoning + evidence + outcome. */\nexport interface TrajectoryStep {\n  actionName: string;\n  actionArgs: Record<string, unknown>;\n  /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */\n  reasoning: string;\n  agentEvidence: AgentEvidence;\n  probeEvidence: ProbeEvidence;\n  toolOutput: ToolOutput;\n}\n\n/** Terminal status of the agent run. */\nexport type TrajectoryStatus = \"complete\" | \"aborted\" | \"stalled\" | \"error\";\n\n/**\n * Full trajectory for one task run.\n *\n * The on-disk layout is one directory per task:\n *\n *   .trajectories/<run-id>/<task-id>/\n *     ├── task_data.json    — TaskSpec + result metadata\n *     ├── trajectory.json   — this object, with screenshotPath instead of bytes\n *     ├── screenshots/      — step probe/agent images plus final observation\n *     ├── scores/\n *     │   └── result.json       — Result from V3Evaluator.verify()\n *     └── core.log          — captured action log\n */\nexport interface Trajectory {\n  task: TaskSpec;\n  steps: TrajectoryStep[];\n  finalAnswer?: string;\n  /** Terminal page observation captured after the agent finishes. */\n  finalObservation?: ProbeEvidence;\n  status: TrajectoryStatus;\n  usage: TrajectoryUsage;\n}\n\n/** Score for a single rubric criterion after evidence analysis + rescoring. */\nexport interface CriterionScore {\n  /** Matches RubricCriterion.criterion (the criterion's short name). */\n  criterion: string;\n  /** Maximum possible points for this criterion. */\n  maxPoints: number;\n  /**\n   * Points earned post-evidence-analysis (paper's post_image_earned_points).\n   * Null if the criterion was conditional and its condition wasn't met (excluded\n   * from both numerator and denominator in the process score).\n   */\n  earnedPoints: number | null;\n  /** Verifier's explanation for the score. */\n  explanation: string;\n  /**\n   * True if the criterion is conditional and its condition was determined to\n   * be met. Absent for non-conditional criteria.\n   */\n  conditionMet?: boolean;\n  /**\n   * Set when the verifier had no evidence to ground this criterion in either\n   * tier. Per paper §2, treated as uncontrollable failure → full credit, but\n   * surfaced here so dashboards can flag low-confidence results.\n   */\n  evidenceInsufficient?: boolean;\n}\n\n/**\n * First-point-of-failure analysis (paper Step 9a). Identifies the earliest\n * step where the agent's trajectory went off-track, using a structured error\n * taxonomy (7 top-level categories, 1.1–7.4 sub-codes).\n */\nexport interface FirstPointOfFailure {\n  stepIndex: number;\n  /** Sub-code from the error taxonomy (e.g., \"2.3\" for a specific hallucination type). */\n  errorCode: string;\n  /** Top-level category name (Selection, Hallucination, etc.). */\n  category: string;\n  /** Verifier's reasoning for selecting this point. */\n  description?: string;\n}\n\n/**\n * Structured observation surfaced by the verifier that another agent or\n * tooling could act on. Findings are emitted opportunistically by Step 8\n * (outcome verification) when the verifier notices actionable patterns —\n * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.\n *\n * Not produced for every task: when nothing actionable surfaces, the\n * `findings` array on the EvaluationResult is empty. Consumers should treat the\n * field as advisory, not as part of the formal score.\n */\nexport interface VerifierFinding {\n  /**\n   * Category of the observation. Open-ended enum — additional categories may\n   * be added as verifier backends surface new failure modes.\n   */\n  category:\n    | \"agent_tool_usage\" // agent's tool calls had repeated issues (misclicks, wrong args, retries)\n    | \"agent_strategy\" // higher-level planning / decision-making problems\n    | \"rubric_quality\" // criteria were overly strict, ambiguous, or contradictory\n    | \"trajectory_capture\" // gaps in evidence (missing screenshots, empty steps)\n    | \"task_specification\" // task instruction was ambiguous / under- or over-specified\n    | \"verifier_uncertainty\" // verifier itself couldn't confidently decide\n    | \"other\";\n  /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */\n  severity: \"info\" | \"warning\" | \"blocking\";\n  /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */\n  description: string;\n  /**\n   * Optional concrete next action another agent could take. Should be\n   * specific enough that it can be acted on without further reasoning —\n   * e.g., \"Try double_click instead of triple_click to clear placeholder\n   * text on this form field.\"\n   */\n  suggestedAction?: string;\n  /** Step indices in the trajectory where this pattern showed up. */\n  relatedSteps?: number[];\n}\n\n/** Stable debugging summary emitted by verifier backends. */\nexport interface VerifierRawSteps {\n  backend?: \"legacy\" | \"verifier\";\n  primaryIntent?: string;\n  reasoning?: string;\n  rubricSource?: \"precomputed\" | \"generated\" | \"none\";\n  approach?: \"a\" | \"b\";\n  optionalsMode?: \"folded\" | \"separate\" | \"skip\";\n  totalEarned?: number;\n  totalMax?: number;\n  evidenceImages?: number;\n  evidenceTexts?: number;\n  evidenceOriginalScreenshots?: number;\n  legacyEvaluation?: string;\n  screenshotCount?: number;\n}\n\n/** Task-validity classification (paper Step 10). */\nexport interface TaskValidity {\n  /** True if the task is underspecified / has multiple valid interpretations. */\n  isAmbiguous: boolean;\n  /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */\n  isInvalid: boolean;\n  /** Optional sub-codes from the task-classification taxonomy. */\n  ambiguityCodes?: string[];\n  invalidTaskCodes?: string[];\n}\n\n/**\n * Evaluator output. Legacy evaluation may only populate outcome fields; richer\n * verifier backends can also populate process scoring and diagnostics.\n *\n * Process and outcome are deliberately independent when both are present:\n * an agent can follow the right steps but get blocked (high process, low\n * outcome), or succeed through an unexpected path (variable process, high\n * outcome).\n */\nexport interface EvaluationResult {\n  /** Did the agent accomplish the task from the user's perspective? */\n  outcomeSuccess: boolean;\n  /** Human-readable explanation for the outcome. */\n  explanation?: string;\n  /** Aggregated earned/max across applicable criteria, in [0, 1]. */\n  processScore?: number;\n  /** Per-criterion breakdown after rescoring. */\n  perCriterion?: CriterionScore[];\n  /** Step 9a — first step where the trajectory went off-track, if any. */\n  firstPointOfFailure?: FirstPointOfFailure;\n  /** Step 10 — task-itself ambiguity / validity. */\n  taskValidity?: TaskValidity;\n  /**\n   * Ids (RubricCriterion.criterion strings) of criteria where neither tier of\n   * evidence resolved the question. Treated as uncontrollable → full credit,\n   * but flagged here so consumers can decide whether to discount the score.\n   */\n  evidenceInsufficient?: string[];\n  /**\n   * Structured observations from the verifier that a downstream tool or\n   * follow-up agent could act on. Opportunistic — empty when the verifier\n   * doesn't notice anything actionable. Not part of the score; advisory.\n   */\n  findings?: VerifierFinding[];\n  /** Debugging summary from the active evaluator backend. */\n  rawSteps?: VerifierRawSteps;\n}\n\n/**\n * Verifier interface. Implementations consume a Trajectory and return an\n * EvaluationResult — they MUST NOT touch a live browser.\n */\nexport interface Verifier {\n  verify(trajectory: Trajectory): Promise<EvaluationResult>;\n}\n"]}

package/dist/esm/lib/v3Evaluator.d.ts

@@ -1,19 +1,32 @@
-/**
- * V3Evaluator mirrors Evaluator but operates on a V3 instance instead of Stagehand.
- * It uses the V3 page/screenshot APIs and constructs an LLM client to run
- * structured evaluations (YES/NO with reasoning) on screenshots and/or text.
- */
 import type { AvailableModel, ClientOptions } from "./v3/types/public/model.js";
-import type { EvaluateOptions, BatchAskOptions, EvaluationResult } from "./v3/types/private/evaluator.js";
+import type { EvaluateOptions, BatchAskOptions, EvaluationResult as LegacyEvaluationResult } from "./v3/types/private/evaluator.js";
 import { V3 } from "./v3/v3.js";
-export declare class V3Evaluator {
-    private v3;
-    private modelName;
-    private modelClientOptions;
-    private silentLogger;
-    constructor(v3: V3, modelName?: AvailableModel, modelClientOptions?: ClientOptions);
-    private getClient;
-    ask(options: EvaluateOptions): Promise<EvaluationResult>;
-    batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]>;
-    private _evaluateWithMultipleScreenshots;
+import type { Trajectory, TaskSpec, EvaluationResult, Rubric, Verifier } from "./v3/verifier/index.js";
+export type V3EvaluatorBackend = "legacy" | "verifier";
+export type V3EvaluatorOptions = {
+    /**
+     * Selects the evaluator implementation.
+     *
+     * "legacy" preserves the existing screenshot/text YES/NO evaluator.
+     * "verifier" is reserved for the rubric verifier backend.
+     *
+     * @default process.env.STAGEHAND_EVALUATOR_BACKEND || "legacy"
+     */
+    backend?: V3EvaluatorBackend;
+};
+export type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {
+    modelName?: AvailableModel;
+    modelClientOptions?: ClientOptions;
+};
+export declare class V3Evaluator implements Verifier {
+    private readonly backend;
+    private readonly legacyEvaluator;
+    constructor(v3: V3, modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions, modelClientOptions?: ClientOptions, options?: V3EvaluatorOptions);
+    ask(options: EvaluateOptions): Promise<LegacyEvaluationResult>;
+    batchAsk(options: BatchAskOptions): Promise<LegacyEvaluationResult[]>;
+    verify(trajectory: Trajectory): Promise<EvaluationResult>;
+    generateRubric(taskSpec: TaskSpec): Promise<Rubric>;
+    private getLegacyBackend;
+    private unavailableVerifierBackend;
+    private verifyTrajectoryWithLegacyEvaluator;
 }