@browserbasehq/orca 3.5.0-vertex-test → 3.6.0-gemini-cua

package/dist/cjs/lib/v3/verifier/types.d.ts

@@ -0,0 +1,429 @@
+/**
+ * Shared verifier types for trajectories, rubrics, evidence, and results.
+ *
+ * The verifier consumes saved trajectories instead of a live browser. DOM and
+ * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve
+ * screenshots sent to the provider plus independent harness probes.
+ */
+import type { LLMClient } from "../llm/LLMClient.js";
+import type { LogLine } from "../types/public/logs.js";
+/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */
+export interface TrajectoryUsage {
+    input_tokens: number;
+    output_tokens: number;
+    reasoning_tokens?: number;
+    cached_input_tokens?: number;
+    inference_time_ms?: number;
+}
+/** A single criterion in a Stagehand rubric. */
+export interface RubricCriterion {
+    /** Short name of the criterion (e.g., "Add ground beef to cart"). */
+    criterion: string;
+    /** What to evaluate and how to award partial credit. */
+    description: string;
+    /** Maximum points for this criterion. */
+    maxPoints: number;
+    /**
+     * Applicability rule for situational criteria. When this condition is not
+     * met, the criterion is excluded from scoring rather than counted as failed.
+     */
+    condition?: string;
+}
+/** A rubric — list of criteria for a task. */
+export interface Rubric {
+    items: RubricCriterion[];
+}
+/**
+ * Spec for a single task being verified. Carried both at runtime and into the
+ * verifier alongside the trajectory.
+ */
+export interface TaskSpec {
+    /** Stable identifier (e.g., "united_13" for WebTailBench, task_id for Mind2Web). */
+    id: string;
+    /** Task instruction shown to the agent. */
+    instruction: string;
+    /** Starting URL, if any. */
+    initUrl?: string;
+    /** Rubric carried by the dataset or generated by a verifier backend. */
+    precomputedRubric?: Rubric;
+    /** Optional reference answer (set when dataset ships one). */
+    expectedAnswer?: string;
+}
+/**
+ * A single modality unit in tier-1 agent evidence. Mirrors the shape of
+ * ModelMessage content parts so we can reproduce what the LLM ingested.
+ */
+export type AgentEvidenceModality = {
+    type: "text";
+    content: string;
+} | {
+    type: "image";
+    bytes: Buffer;
+    mediaType: string;
+} | {
+    type: "json";
+    content: unknown;
+};
+/**
+ * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the
+ * tool result for this step.
+ *
+ * Modes:
+ *   - CUA: usually a single image modality (the screenshot sent to the provider).
+ *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.
+ *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.
+ */
+export interface AgentEvidence {
+    modalities: AgentEvidenceModality[];
+}
+/**
+ * Tier 2 — independent harness probes around this step.
+ *
+ * If a probe wasn't captured, the field is absent (not null).
+ */
+export interface ProbeEvidence {
+    /** URL after the step's tool execution. */
+    url?: string;
+    /**
+     * Bus screenshot captured after the step. Path on disk is preferred once
+     * persisted; in-memory Buffer is used during a live run.
+     */
+    screenshot?: Buffer;
+    /** Reference to the persisted screenshot file under the trajectory dir. */
+    screenshotPath?: string;
+    /** Accessibility tree snapshot. */
+    ariaTree?: string;
+    /** Verifier-requested probes, keyed by criterion id. */
+    onDemand?: Record<string, unknown>;
+}
+/** Outcome of a single tool execution as seen by the harness. */
+export interface ToolOutput {
+    ok: boolean;
+    /**
+     * The tool's return value. Same payload that flowed into agentEvidence
+     * modalities, but in its native shape (e.g., the extract result, the act
+     * describe-string) rather than serialized for the LLM.
+     */
+    result: unknown;
+    error?: string;
+}
+/** One step in a trajectory: action + reasoning + evidence + outcome. */
+export interface TrajectoryStep {
+    actionName: string;
+    actionArgs: Record<string, unknown>;
+    /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */
+    reasoning: string;
+    agentEvidence: AgentEvidence;
+    probeEvidence: ProbeEvidence;
+    toolOutput: ToolOutput;
+}
+/** Terminal status of the agent run. */
+export type TrajectoryStatus = "complete" | "aborted" | "stalled" | "error";
+/**
+ * Full trajectory for one task run.
+ *
+ * The on-disk layout is one directory per task:
+ *
+ *   .trajectories/<run-id>/<task-id>/
+ *     ├── task_data.json           — TaskSpec + result metadata
+ *     ├── trajectory.json          — this object, with image paths instead of bytes
+ *     ├── screenshots/
+ *     │   ├── probe/<N>.png        — tier-2 probe screenshot per step
+ *     │   ├── probe/final.png      — final terminal observation screenshot
+ *     │   └── agent/<N>.png        — tier-1 image the model received per step
+ *     ├── scores/
+ *     │   └── result.json          — Result from V3Evaluator.verify()
+ *     └── core.log                 — captured action log
+ */
+export interface Trajectory {
+    task: TaskSpec;
+    steps: TrajectoryStep[];
+    finalAnswer?: string;
+    /** Terminal page observation captured after the agent finishes. */
+    finalObservation?: ProbeEvidence;
+    status: TrajectoryStatus;
+    usage: TrajectoryUsage;
+}
+/** A single screenshot kept by Step 1, ready for downstream relevance scoring. */
+export interface CanonicalScreenshot {
+    /** 0-based position in the kept-screenshots array. Stable across the pipeline. */
+    canonicalIndex: number;
+    /**
+     * 0-based position in `Trajectory.steps` this screenshot came from
+     * (steps.length for the final observation). Lets downstream prompts
+     * cross-reference the action history.
+     */
+    originalStepIndex: number;
+    /** The resized PNG/JPEG buffer (or native bytes if sharp unavailable). */
+    bytes: Buffer;
+    /** MIME media type. Always "image/png" after the optional resize. */
+    mediaType: string;
+    /** Reason this frame was kept: "first" / "last" / "diverges". */
+    keptReason: "first" | "last" | "diverges" | "no-dedup";
+}
+/**
+ * A text evidence point sourced from tier-2 probes or tier-1 tool outputs.
+ * These feed the same relevance + scoring path as screenshots, letting DOM
+ * and hybrid agents preserve extract/aria/tool-return evidence without a
+ * separate verifier architecture.
+ */
+export interface CanonicalTextEvidence {
+    /** 0-based position in the combined evidence-point array. */
+    canonicalIndex: number;
+    originalStepIndex: number;
+    /** Where the text came from. */
+    source: "probe-aria" | "agent-text" | "agent-json" | "tool-output";
+    /** The text payload, already truncated. */
+    content: string;
+}
+export type CanonicalEvidence = CanonicalScreenshot | CanonicalTextEvidence;
+/** Result of Step 1 evidence loading. */
+export interface EvidenceLoadResult {
+    /** Kept frames, in chronological order. */
+    screenshots: CanonicalScreenshot[];
+    /**
+     * Maps trajectory step position → canonical index in `screenshots`. Steps
+     * whose screenshots were deduplicated point to the surviving canonical frame
+     * (typically the prior kept frame). Useful for "find me the screenshot for
+     * step K" lookups in downstream prompts.
+     */
+    stepIndexToCanonical: Map<number, number>;
+    /** Number of original frames considered. */
+    originalCount: number;
+    /** Number of frames kept post-dedup (== screenshots.length). */
+    keptCount: number;
+    /** Effective thresholds used (resolved from env). */
+    thresholds: {
+        ssim: number;
+        mse: number;
+        resize: number;
+    };
+}
+/** Options for evidence loading; primarily test seams over the defaults. */
+export interface EvidenceLoadOptions {
+    /** SSIM similarity threshold for dedup (default 0.75). */
+    ssimThreshold?: number;
+    /** MSE similarity threshold for dedup (default 30). */
+    mseThreshold?: number;
+    /** Scale factor applied before relevance scoring (default 0.7). */
+    imageResize?: number;
+}
+/** Score for a single rubric criterion after evidence analysis + rescoring. */
+export interface CriterionScore {
+    /** Matches RubricCriterion.criterion (the criterion's short name). */
+    criterion: string;
+    /** Maximum possible points for this criterion. */
+    maxPoints: number;
+    /**
+     * Points earned after evidence analysis. Null when the criterion is
+     * conditional and its condition was not met — excluded from both numerator
+     * and denominator in the process score.
+     */
+    earnedPoints: number | null;
+    /** Verifier's explanation for the score. */
+    explanation: string;
+    /**
+     * True if the criterion is conditional and its condition was determined to
+     * be met. Absent for non-conditional criteria.
+     */
+    conditionMet?: boolean;
+    /**
+     * Set when the verifier had no evidence to ground this criterion in either
+     * tier. Treated as uncontrollable failure (full credit) but surfaced here
+     * so dashboards can flag low-confidence results.
+     */
+    evidenceInsufficient?: boolean;
+}
+/**
+ * Earliest step where the agent's trajectory went off-track, classified
+ * against the error taxonomy (7 top-level categories, 1.1–7.4 sub-codes).
+ */
+export interface FirstPointOfFailure {
+    stepIndex: number;
+    /** Sub-code from the error taxonomy (e.g., "2.3" for a specific hallucination type). */
+    errorCode: string;
+    /** Top-level category name (Selection, Hallucination, etc.). */
+    category: string;
+    /** Verifier's reasoning for selecting this point. */
+    description?: string;
+}
+/**
+ * Structured observation surfaced by the verifier that another agent or
+ * tooling could act on. Findings are emitted opportunistically by Step 8
+ * (outcome verification) when the verifier notices actionable patterns —
+ * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.
+ *
+ * Not produced for every task: when nothing actionable surfaces, the
+ * `findings` array on the EvaluationResult is empty. Consumers should treat the
+ * field as advisory, not as part of the formal score.
+ */
+export interface VerifierFinding {
+    /**
+     * Category of the observation. Open-ended enum — additional categories may
+     * be added as verifier backends surface new failure modes.
+     */
+    category: "agent_tool_usage" | "agent_strategy" | "rubric_quality" | "trajectory_capture" | "task_specification" | "verifier_uncertainty" | "other";
+    /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */
+    severity: "info" | "warning" | "blocking";
+    /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */
+    description: string;
+    /**
+     * Optional concrete next action another agent could take. Should be
+     * specific enough that it can be acted on without further reasoning —
+     * e.g., "Try double_click instead of triple_click to clear placeholder
+     * text on this form field."
+     */
+    suggestedAction?: string;
+    /** Step indices in the trajectory where this pattern showed up. */
+    relatedSteps?: number[];
+}
+/** Stable debugging summary emitted by verifier backends. */
+export interface VerifierRawSteps {
+    backend?: "legacy" | "verifier";
+    reason?: string;
+    primaryIntent?: string;
+    reasoning?: string;
+    rubricSource?: "precomputed" | "generated" | "none";
+    approach?: "a" | "b" | "outcome-only";
+    optionalsMode?: "folded" | "separate" | "skip";
+    totalEarned?: number;
+    totalMax?: number;
+    evidenceImages?: number;
+    evidenceTexts?: number;
+    evidenceOriginalScreenshots?: number;
+    screenshotsAttached?: number;
+    legacyEvaluation?: string;
+    screenshotCount?: number;
+}
+/** Task-validity classification: whether the task is even answerable. */
+export interface TaskValidity {
+    /** True if the task is underspecified / has multiple valid interpretations. */
+    isAmbiguous: boolean;
+    /** Explanation for why the task is ambiguous, when available. */
+    ambiguityReason?: string;
+    /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */
+    isInvalid: boolean;
+    /** Explanation for why the task is invalid, when available. */
+    invalidReason?: string;
+    /** Optional sub-codes from the task-classification taxonomy. */
+    ambiguityCodes?: string[];
+    invalidTaskCodes?: string[];
+}
+/**
+ * Evaluator output. Legacy evaluation may only populate outcome fields; richer
+ * verifier backends can also populate process scoring and diagnostics.
+ *
+ * Process and outcome are deliberately independent when both are present:
+ * an agent can follow the right steps but get blocked (high process, low
+ * outcome), or succeed through an unexpected path (variable process, high
+ * outcome).
+ */
+export interface EvaluationResult {
+    /** Did the agent accomplish the task from the user's perspective? */
+    outcomeSuccess: boolean;
+    /** Human-readable explanation for the outcome. */
+    explanation?: string;
+    /** Aggregated earned/max across applicable criteria, in [0, 1]. */
+    processScore?: number;
+    /** Per-criterion breakdown after rescoring. */
+    perCriterion?: CriterionScore[];
+    /** Step 9a — first step where the trajectory went off-track, if any. */
+    firstPointOfFailure?: FirstPointOfFailure;
+    /** Step 10 — task-itself ambiguity / validity. */
+    taskValidity?: TaskValidity;
+    /**
+     * Ids (RubricCriterion.criterion strings) of criteria where neither tier of
+     * evidence resolved the question. Treated as uncontrollable → full credit,
+     * but flagged here so consumers can decide whether to discount the score.
+     */
+    evidenceInsufficient?: string[];
+    /**
+     * Structured observations from the verifier that a downstream tool or
+     * follow-up agent could act on. Opportunistic — empty when the verifier
+     * doesn't notice anything actionable. Not part of the score; advisory.
+     */
+    findings?: VerifierFinding[];
+    /** Debugging summary from the active evaluator backend. */
+    rawSteps?: VerifierRawSteps;
+}
+/**
+ * Verifier interface. Implementations consume a Trajectory and return an
+ * EvaluationResult — they MUST NOT touch a live browser.
+ */
+export interface Verifier {
+    verify(trajectory: Trajectory): Promise<EvaluationResult>;
+}
+export interface RubricVerifierOptions {
+    /** Factory that returns a configured LLMClient. Called per pipeline step so callers can supply step-specific clients. */
+    getClient: () => LLMClient;
+    /** Optional factory for rubric generation so callers can route it to a stronger model. */
+    getRubricGenClient?: () => LLMClient;
+    /** Logger; defaults to a no-op so the verifier stays quiet inside V3Evaluator. */
+    logger?: (line: LogLine) => void;
+    /**
+     * Override any verifier knob. Env vars supply the defaults; values here win.
+     * Useful for tests and for cross-verify sweeps that want different budgets
+     * per run.
+     */
+    config?: Partial<VerifierConfig>;
+}
+/**
+ * Resolved verifier knobs. Constructed once from env (and optional overrides)
+ * by RubricVerifier's constructor; subsequent verify() calls can pass a
+ * Partial to shift any field.
+ */
+export interface VerifierConfig {
+    /** Which pipeline path to take: per-criterion (a), fused (b), or skip rubric entirely (outcome-only). */
+    approach: "a" | "b" | "outcome-only";
+    /** Folded (in fused call), separate (own calls), or skip (omit). */
+    optionalSteps: "folded" | "separate" | "skip";
+    /** Top-K evidence points selected per criterion. */
+    topK: number;
+    /** Batch size for the relevance-scoring LLM call. */
+    relevanceBatchSize: number;
+    /** Image cap on the outcome-only path. */
+    outcomeMaxImages: number;
+    /** Concurrent LLM calls across batches / criteria. */
+    maxParallel: number;
+    /** Token budgets for the three evidence channels. */
+    evidenceTokenBudget: number;
+    outcomeEvidenceTokenBudget: number;
+    actionHistoryTokenBudget: number;
+    /** Per-section character limits applied during evidence-text assembly. */
+    truncation: {
+        /** Master switch: when true, all per-section limits go to MAX_SAFE_INTEGER. */
+        disabled: boolean;
+        evidenceTextPreview: number;
+        groupedEvidenceText: number;
+        buildEvidenceText: number;
+        buildEvidenceAria: number;
+        actionHistoryReasoning: number;
+    };
+}
+export interface ErrorTaxonomySubCategory {
+    /** Sub-code (e.g., "2.3"). */
+    code: string;
+    /** Human-readable name (e.g., "Output fabrication"). */
+    name: string;
+    /** Detailed description ported from the .md. Markdown formatting preserved. */
+    description: string;
+}
+export interface ErrorTaxonomyCategory {
+    /** Top-level number (1-8). */
+    number: number;
+    /** Top-level name (e.g., "Hallucination Errors"). */
+    name: string;
+    /** One-sentence summary of the category. */
+    summary: string;
+    /** Sub-categories. The last one is always an "Other" catch-all. */
+    subCategories: ErrorTaxonomySubCategory[];
+}
+export interface ParseFailureStepNumbersOptions {
+    /**
+     * Maximum unique step numbers to expand from ranges. Protects the verifier
+     * from malformed model output such as "0-2147483647".
+     */
+    maxExpandedSteps?: number;
+    /** Optional inclusive upper bound for accepted step numbers. */
+    maxStep?: number;
+}

package/dist/cjs/lib/v3/verifier/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/cjs/lib/v3/verifier/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/types.ts"],"names":[],"mappings":"","sourcesContent":["/**\n * Shared verifier types for trajectories, rubrics, evidence, and results.\n *\n * The verifier consumes saved trajectories instead of a live browser. DOM and\n * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve\n * screenshots sent to the provider plus independent harness probes.\n */\nimport type { LLMClient } from \"../llm/LLMClient.js\";\nimport type { LogLine } from \"../types/public/logs.js\";\n\n/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */\nexport interface TrajectoryUsage {\n  input_tokens: number;\n  output_tokens: number;\n  reasoning_tokens?: number;\n  cached_input_tokens?: number;\n  inference_time_ms?: number;\n}\n\n/** A single criterion in a Stagehand rubric. */\nexport interface RubricCriterion {\n  /** Short name of the criterion (e.g., \"Add ground beef to cart\"). */\n  criterion: string;\n  /** What to evaluate and how to award partial credit. */\n  description: string;\n  /** Maximum points for this criterion. */\n  maxPoints: number;\n  /**\n   * Applicability rule for situational criteria. When this condition is not\n   * met, the criterion is excluded from scoring rather than counted as failed.\n   */\n  condition?: string;\n}\n\n/** A rubric — list of criteria for a task. */\nexport interface Rubric {\n  items: RubricCriterion[];\n}\n\n/**\n * Spec for a single task being verified. Carried both at runtime and into the\n * verifier alongside the trajectory.\n */\nexport interface TaskSpec {\n  /** Stable identifier (e.g., \"united_13\" for WebTailBench, task_id for Mind2Web). */\n  id: string;\n  /** Task instruction shown to the agent. */\n  instruction: string;\n  /** Starting URL, if any. */\n  initUrl?: string;\n  /** Rubric carried by the dataset or generated by a verifier backend. */\n  precomputedRubric?: Rubric;\n  /** Optional reference answer (set when dataset ships one). */\n  expectedAnswer?: string;\n}\n\n/**\n * A single modality unit in tier-1 agent evidence. Mirrors the shape of\n * ModelMessage content parts so we can reproduce what the LLM ingested.\n */\nexport type AgentEvidenceModality =\n  | { type: \"text\"; content: string }\n  | { type: \"image\"; bytes: Buffer; mediaType: string }\n  | { type: \"json\"; content: unknown };\n\n/**\n * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the\n * tool result for this step.\n *\n * Modes:\n *   - CUA: usually a single image modality (the screenshot sent to the provider).\n *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.\n *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.\n */\nexport interface AgentEvidence {\n  modalities: AgentEvidenceModality[];\n}\n\n/**\n * Tier 2 — independent harness probes around this step.\n *\n * If a probe wasn't captured, the field is absent (not null).\n */\nexport interface ProbeEvidence {\n  /** URL after the step's tool execution. */\n  url?: string;\n  /**\n   * Bus screenshot captured after the step. Path on disk is preferred once\n   * persisted; in-memory Buffer is used during a live run.\n   */\n  screenshot?: Buffer;\n  /** Reference to the persisted screenshot file under the trajectory dir. */\n  screenshotPath?: string;\n  /** Accessibility tree snapshot. */\n  ariaTree?: string;\n  /** Verifier-requested probes, keyed by criterion id. */\n  onDemand?: Record<string, unknown>;\n}\n\n/** Outcome of a single tool execution as seen by the harness. */\nexport interface ToolOutput {\n  ok: boolean;\n  /**\n   * The tool's return value. Same payload that flowed into agentEvidence\n   * modalities, but in its native shape (e.g., the extract result, the act\n   * describe-string) rather than serialized for the LLM.\n   */\n  result: unknown;\n  error?: string;\n}\n\n/** One step in a trajectory: action + reasoning + evidence + outcome. */\nexport interface TrajectoryStep {\n  actionName: string;\n  actionArgs: Record<string, unknown>;\n  /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */\n  reasoning: string;\n  agentEvidence: AgentEvidence;\n  probeEvidence: ProbeEvidence;\n  toolOutput: ToolOutput;\n}\n\n/** Terminal status of the agent run. */\nexport type TrajectoryStatus = \"complete\" | \"aborted\" | \"stalled\" | \"error\";\n\n/**\n * Full trajectory for one task run.\n *\n * The on-disk layout is one directory per task:\n *\n *   .trajectories/<run-id>/<task-id>/\n *     ├── task_data.json           — TaskSpec + result metadata\n *     ├── trajectory.json          — this object, with image paths instead of bytes\n *     ├── screenshots/\n *     │   ├── probe/<N>.png        — tier-2 probe screenshot per step\n *     │   ├── probe/final.png      — final terminal observation screenshot\n *     │   └── agent/<N>.png        — tier-1 image the model received per step\n *     ├── scores/\n *     │   └── result.json          — Result from V3Evaluator.verify()\n *     └── core.log                 — captured action log\n */\nexport interface Trajectory {\n  task: TaskSpec;\n  steps: TrajectoryStep[];\n  finalAnswer?: string;\n  /** Terminal page observation captured after the agent finishes. */\n  finalObservation?: ProbeEvidence;\n  status: TrajectoryStatus;\n  usage: TrajectoryUsage;\n}\n\n/** A single screenshot kept by Step 1, ready for downstream relevance scoring. */\nexport interface CanonicalScreenshot {\n  /** 0-based position in the kept-screenshots array. Stable across the pipeline. */\n  canonicalIndex: number;\n  /**\n   * 0-based position in `Trajectory.steps` this screenshot came from\n   * (steps.length for the final observation). Lets downstream prompts\n   * cross-reference the action history.\n   */\n  originalStepIndex: number;\n  /** The resized PNG/JPEG buffer (or native bytes if sharp unavailable). */\n  bytes: Buffer;\n  /** MIME media type. Always \"image/png\" after the optional resize. */\n  mediaType: string;\n  /** Reason this frame was kept: \"first\" / \"last\" / \"diverges\". */\n  keptReason: \"first\" | \"last\" | \"diverges\" | \"no-dedup\";\n}\n\n/**\n * A text evidence point sourced from tier-2 probes or tier-1 tool outputs.\n * These feed the same relevance + scoring path as screenshots, letting DOM\n * and hybrid agents preserve extract/aria/tool-return evidence without a\n * separate verifier architecture.\n */\nexport interface CanonicalTextEvidence {\n  /** 0-based position in the combined evidence-point array. */\n  canonicalIndex: number;\n  originalStepIndex: number;\n  /** Where the text came from. */\n  source: \"probe-aria\" | \"agent-text\" | \"agent-json\" | \"tool-output\";\n  /** The text payload, already truncated. */\n  content: string;\n}\n\nexport type CanonicalEvidence = CanonicalScreenshot | CanonicalTextEvidence;\n\n/** Result of Step 1 evidence loading. */\nexport interface EvidenceLoadResult {\n  /** Kept frames, in chronological order. */\n  screenshots: CanonicalScreenshot[];\n  /**\n   * Maps trajectory step position → canonical index in `screenshots`. Steps\n   * whose screenshots were deduplicated point to the surviving canonical frame\n   * (typically the prior kept frame). Useful for \"find me the screenshot for\n   * step K\" lookups in downstream prompts.\n   */\n  stepIndexToCanonical: Map<number, number>;\n  /** Number of original frames considered. */\n  originalCount: number;\n  /** Number of frames kept post-dedup (== screenshots.length). */\n  keptCount: number;\n  /** Effective thresholds used (resolved from env). */\n  thresholds: {\n    ssim: number;\n    mse: number;\n    resize: number;\n  };\n}\n\n/** Options for evidence loading; primarily test seams over the defaults. */\nexport interface EvidenceLoadOptions {\n  /** SSIM similarity threshold for dedup (default 0.75). */\n  ssimThreshold?: number;\n  /** MSE similarity threshold for dedup (default 30). */\n  mseThreshold?: number;\n  /** Scale factor applied before relevance scoring (default 0.7). */\n  imageResize?: number;\n}\n\n/** Score for a single rubric criterion after evidence analysis + rescoring. */\nexport interface CriterionScore {\n  /** Matches RubricCriterion.criterion (the criterion's short name). */\n  criterion: string;\n  /** Maximum possible points for this criterion. */\n  maxPoints: number;\n  /**\n   * Points earned after evidence analysis. Null when the criterion is\n   * conditional and its condition was not met — excluded from both numerator\n   * and denominator in the process score.\n   */\n  earnedPoints: number | null;\n  /** Verifier's explanation for the score. */\n  explanation: string;\n  /**\n   * True if the criterion is conditional and its condition was determined to\n   * be met. Absent for non-conditional criteria.\n   */\n  conditionMet?: boolean;\n  /**\n   * Set when the verifier had no evidence to ground this criterion in either\n   * tier. Treated as uncontrollable failure (full credit) but surfaced here\n   * so dashboards can flag low-confidence results.\n   */\n  evidenceInsufficient?: boolean;\n}\n\n/**\n * Earliest step where the agent's trajectory went off-track, classified\n * against the error taxonomy (7 top-level categories, 1.1–7.4 sub-codes).\n */\nexport interface FirstPointOfFailure {\n  stepIndex: number;\n  /** Sub-code from the error taxonomy (e.g., \"2.3\" for a specific hallucination type). */\n  errorCode: string;\n  /** Top-level category name (Selection, Hallucination, etc.). */\n  category: string;\n  /** Verifier's reasoning for selecting this point. */\n  description?: string;\n}\n\n/**\n * Structured observation surfaced by the verifier that another agent or\n * tooling could act on. Findings are emitted opportunistically by Step 8\n * (outcome verification) when the verifier notices actionable patterns —\n * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.\n *\n * Not produced for every task: when nothing actionable surfaces, the\n * `findings` array on the EvaluationResult is empty. Consumers should treat the\n * field as advisory, not as part of the formal score.\n */\nexport interface VerifierFinding {\n  /**\n   * Category of the observation. Open-ended enum — additional categories may\n   * be added as verifier backends surface new failure modes.\n   */\n  category:\n    | \"agent_tool_usage\" // agent's tool calls had repeated issues (misclicks, wrong args, retries)\n    | \"agent_strategy\" // higher-level planning / decision-making problems\n    | \"rubric_quality\" // criteria were overly strict, ambiguous, or contradictory\n    | \"trajectory_capture\" // gaps in evidence (missing screenshots, empty steps)\n    | \"task_specification\" // task instruction was ambiguous / under- or over-specified\n    | \"verifier_uncertainty\" // verifier itself couldn't confidently decide\n    | \"other\";\n  /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */\n  severity: \"info\" | \"warning\" | \"blocking\";\n  /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */\n  description: string;\n  /**\n   * Optional concrete next action another agent could take. Should be\n   * specific enough that it can be acted on without further reasoning —\n   * e.g., \"Try double_click instead of triple_click to clear placeholder\n   * text on this form field.\"\n   */\n  suggestedAction?: string;\n  /** Step indices in the trajectory where this pattern showed up. */\n  relatedSteps?: number[];\n}\n\n/** Stable debugging summary emitted by verifier backends. */\nexport interface VerifierRawSteps {\n  backend?: \"legacy\" | \"verifier\";\n  reason?: string;\n  primaryIntent?: string;\n  reasoning?: string;\n  rubricSource?: \"precomputed\" | \"generated\" | \"none\";\n  approach?: \"a\" | \"b\" | \"outcome-only\";\n  optionalsMode?: \"folded\" | \"separate\" | \"skip\";\n  totalEarned?: number;\n  totalMax?: number;\n  evidenceImages?: number;\n  evidenceTexts?: number;\n  evidenceOriginalScreenshots?: number;\n  screenshotsAttached?: number;\n  legacyEvaluation?: string;\n  screenshotCount?: number;\n}\n\n/** Task-validity classification: whether the task is even answerable. */\nexport interface TaskValidity {\n  /** True if the task is underspecified / has multiple valid interpretations. */\n  isAmbiguous: boolean;\n  /** Explanation for why the task is ambiguous, when available. */\n  ambiguityReason?: string;\n  /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */\n  isInvalid: boolean;\n  /** Explanation for why the task is invalid, when available. */\n  invalidReason?: string;\n  /** Optional sub-codes from the task-classification taxonomy. */\n  ambiguityCodes?: string[];\n  invalidTaskCodes?: string[];\n}\n\n/**\n * Evaluator output. Legacy evaluation may only populate outcome fields; richer\n * verifier backends can also populate process scoring and diagnostics.\n *\n * Process and outcome are deliberately independent when both are present:\n * an agent can follow the right steps but get blocked (high process, low\n * outcome), or succeed through an unexpected path (variable process, high\n * outcome).\n */\nexport interface EvaluationResult {\n  /** Did the agent accomplish the task from the user's perspective? */\n  outcomeSuccess: boolean;\n  /** Human-readable explanation for the outcome. */\n  explanation?: string;\n  /** Aggregated earned/max across applicable criteria, in [0, 1]. */\n  processScore?: number;\n  /** Per-criterion breakdown after rescoring. */\n  perCriterion?: CriterionScore[];\n  /** Step 9a — first step where the trajectory went off-track, if any. */\n  firstPointOfFailure?: FirstPointOfFailure;\n  /** Step 10 — task-itself ambiguity / validity. */\n  taskValidity?: TaskValidity;\n  /**\n   * Ids (RubricCriterion.criterion strings) of criteria where neither tier of\n   * evidence resolved the question. Treated as uncontrollable → full credit,\n   * but flagged here so consumers can decide whether to discount the score.\n   */\n  evidenceInsufficient?: string[];\n  /**\n   * Structured observations from the verifier that a downstream tool or\n   * follow-up agent could act on. Opportunistic — empty when the verifier\n   * doesn't notice anything actionable. Not part of the score; advisory.\n   */\n  findings?: VerifierFinding[];\n  /** Debugging summary from the active evaluator backend. */\n  rawSteps?: VerifierRawSteps;\n}\n\n/**\n * Verifier interface. Implementations consume a Trajectory and return an\n * EvaluationResult — they MUST NOT touch a live browser.\n */\nexport interface Verifier {\n  verify(trajectory: Trajectory): Promise<EvaluationResult>;\n}\n\nexport interface RubricVerifierOptions {\n  /** Factory that returns a configured LLMClient. Called per pipeline step so callers can supply step-specific clients. */\n  getClient: () => LLMClient;\n  /** Optional factory for rubric generation so callers can route it to a stronger model. */\n  getRubricGenClient?: () => LLMClient;\n  /** Logger; defaults to a no-op so the verifier stays quiet inside V3Evaluator. */\n  logger?: (line: LogLine) => void;\n  /**\n   * Override any verifier knob. Env vars supply the defaults; values here win.\n   * Useful for tests and for cross-verify sweeps that want different budgets\n   * per run.\n   */\n  config?: Partial<VerifierConfig>;\n}\n\n/**\n * Resolved verifier knobs. Constructed once from env (and optional overrides)\n * by RubricVerifier's constructor; subsequent verify() calls can pass a\n * Partial to shift any field.\n */\nexport interface VerifierConfig {\n  /** Which pipeline path to take: per-criterion (a), fused (b), or skip rubric entirely (outcome-only). */\n  approach: \"a\" | \"b\" | \"outcome-only\";\n  /** Folded (in fused call), separate (own calls), or skip (omit). */\n  optionalSteps: \"folded\" | \"separate\" | \"skip\";\n  /** Top-K evidence points selected per criterion. */\n  topK: number;\n  /** Batch size for the relevance-scoring LLM call. */\n  relevanceBatchSize: number;\n  /** Image cap on the outcome-only path. */\n  outcomeMaxImages: number;\n  /** Concurrent LLM calls across batches / criteria. */\n  maxParallel: number;\n  /** Token budgets for the three evidence channels. */\n  evidenceTokenBudget: number;\n  outcomeEvidenceTokenBudget: number;\n  actionHistoryTokenBudget: number;\n  /** Per-section character limits applied during evidence-text assembly. */\n  truncation: {\n    /** Master switch: when true, all per-section limits go to MAX_SAFE_INTEGER. */\n    disabled: boolean;\n    evidenceTextPreview: number;\n    groupedEvidenceText: number;\n    buildEvidenceText: number;\n    buildEvidenceAria: number;\n    actionHistoryReasoning: number;\n  };\n}\n\nexport interface ErrorTaxonomySubCategory {\n  /** Sub-code (e.g., \"2.3\"). */\n  code: string;\n  /** Human-readable name (e.g., \"Output fabrication\"). */\n  name: string;\n  /** Detailed description ported from the .md. Markdown formatting preserved. */\n  description: string;\n}\n\nexport interface ErrorTaxonomyCategory {\n  /** Top-level number (1-8). */\n  number: number;\n  /** Top-level name (e.g., \"Hallucination Errors\"). */\n  name: string;\n  /** One-sentence summary of the category. */\n  summary: string;\n  /** Sub-categories. The last one is always an \"Other\" catch-all. */\n  subCategories: ErrorTaxonomySubCategory[];\n}\n\nexport interface ParseFailureStepNumbersOptions {\n  /**\n   * Maximum unique step numbers to expand from ranges. Protects the verifier\n   * from malformed model output such as \"0-2147483647\".\n   */\n  maxExpandedSteps?: number;\n  /** Optional inclusive upper bound for accepted step numbers. */\n  maxStep?: number;\n}\n"]}

package/dist/cjs/lib/v3Evaluator.d.ts

@@ -1,6 +1,7 @@
 import type { AvailableModel, ClientOptions } from "./v3/types/public/model.js";
-import type { EvaluateOptions, BatchAskOptions, EvaluationResult } from "./v3/types/private/evaluator.js";
+import type { EvaluateOptions, BatchAskOptions, EvaluationResult as LegacyEvaluationResult } from "./v3/types/private/evaluator.js";
 import { V3 } from "./v3/v3.js";
+import type { Trajectory, TaskSpec, EvaluationResult, Rubric, Verifier } from "./v3/verifier/index.js";
 export type V3EvaluatorBackend = "legacy" | "verifier";
 export type V3EvaluatorOptions = {
     /**
@@ -17,11 +18,20 @@ export type V3EvaluatorConstructorOptions = V3EvaluatorOptions & {
     modelName?: AvailableModel;
     modelClientOptions?: ClientOptions;
 };
-export declare class V3Evaluator {
+export declare class V3Evaluator implements Verifier {
+    private readonly v3;
     private readonly backend;
+    private readonly modelName;
+    private readonly modelClientOptions;
     private readonly legacyEvaluator;
     constructor(v3: V3, modelNameOrOptions?: AvailableModel | V3EvaluatorConstructorOptions, modelClientOptions?: ClientOptions, options?: V3EvaluatorOptions);
-    ask(options: EvaluateOptions): Promise<EvaluationResult>;
-    batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]>;
+    ask(options: EvaluateOptions): Promise<LegacyEvaluationResult>;
+    batchAsk(options: BatchAskOptions): Promise<LegacyEvaluationResult[]>;
+    verify(trajectory: Trajectory): Promise<EvaluationResult>;
+    generateRubric(taskSpec: TaskSpec): Promise<Rubric>;
     private getLegacyBackend;
+    private unavailableVerifierBackend;
+    private getClient;
+    private getRubricGenClient;
+    private verifyTrajectoryWithLegacyEvaluator;
 }