@browserbasehq/orca 3.7.0-preview.1 → 3.7.2-preview.0

package/dist/cjs/lib/v3/verifier/types.d.ts

@@ -122,9 +122,10 @@ export type TrajectoryStatus = "complete" | "aborted" | "stalled" | "error";
 /**
  * Full trajectory for one task run.
  *
- * The on-disk layout is one directory per task:
+ * The on-disk layout is one directory per task run, under the run-scoped group
+ * dir (`<experiment>__<model>__<runToken>`, or "default" for ad-hoc runs):
  *
- *   .trajectories/<run-id>/<task-id>/
+ *   .trajectories/<group>/<task-id>/<run-id>/
  *     ├── task_data.json           — TaskSpec + result metadata
  *     ├── trajectory.json          — this object, with image paths instead of bytes
  *     ├── screenshots/

package/dist/cjs/lib/v3/verifier/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/types.ts"],"names":[],"mappings":"","sourcesContent":["/**\n * Shared verifier types for trajectories, rubrics, evidence, and results.\n *\n * The verifier consumes saved trajectories instead of a live browser. DOM and\n * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve\n * screenshots sent to the provider plus independent harness probes.\n */\nimport type { LLMClient } from \"../llm/LLMClient.js\";\nimport type { LogLine } from \"../types/public/logs.js\";\n\n/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */\nexport interface TrajectoryUsage {\n  input_tokens: number;\n  output_tokens: number;\n  reasoning_tokens?: number;\n  cached_input_tokens?: number;\n  inference_time_ms?: number;\n}\n\n/** A single criterion in a Stagehand rubric. */\nexport interface RubricCriterion {\n  /** Short name of the criterion (e.g., \"Add ground beef to cart\"). */\n  criterion: string;\n  /** What to evaluate and how to award partial credit. */\n  description: string;\n  /** Maximum points for this criterion. */\n  maxPoints: number;\n  /**\n   * Applicability rule for situational criteria. When this condition is not\n   * met, the criterion is excluded from scoring rather than counted as failed.\n   */\n  condition?: string;\n}\n\n/** A rubric — list of criteria for a task. */\nexport interface Rubric {\n  items: RubricCriterion[];\n}\n\n/**\n * Spec for a single task being verified. Carried both at runtime and into the\n * verifier alongside the trajectory.\n */\nexport interface TaskSpec {\n  /** Stable identifier (e.g., \"united_13\" for WebTailBench, task_id for Mind2Web). */\n  id: string;\n  /** Task instruction shown to the agent. */\n  instruction: string;\n  /** Starting URL, if any. */\n  initUrl?: string;\n  /** Rubric carried by the dataset or generated by a verifier backend. */\n  precomputedRubric?: Rubric;\n  /** Optional reference answer (set when dataset ships one). */\n  expectedAnswer?: string;\n}\n\n/**\n * A single modality unit in tier-1 agent evidence. Mirrors the shape of\n * ModelMessage content parts so we can reproduce what the LLM ingested.\n */\nexport type AgentEvidenceModality =\n  | { type: \"text\"; content: string }\n  | { type: \"image\"; bytes: Buffer; mediaType: string }\n  | { type: \"json\"; content: unknown };\n\n/**\n * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the\n * tool result for this step.\n *\n * Modes:\n *   - CUA: usually a single image modality (the screenshot sent to the provider).\n *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.\n *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.\n */\nexport interface AgentEvidence {\n  modalities: AgentEvidenceModality[];\n}\n\n/**\n * Tier 2 — independent harness probes around this step.\n *\n * If a probe wasn't captured, the field is absent (not null).\n */\nexport interface ProbeEvidence {\n  /** URL after the step's tool execution. */\n  url?: string;\n  /**\n   * Bus screenshot captured after the step. Path on disk is preferred once\n   * persisted; in-memory Buffer is used during a live run.\n   */\n  screenshot?: Buffer;\n  /** Reference to the persisted screenshot file under the trajectory dir. */\n  screenshotPath?: string;\n  /** Accessibility tree snapshot. */\n  ariaTree?: string;\n  /** Verifier-requested probes, keyed by criterion id. */\n  onDemand?: Record<string, unknown>;\n}\n\n/** Outcome of a single tool execution as seen by the harness. */\nexport interface ToolOutput {\n  ok: boolean;\n  /**\n   * The tool's return value. Same payload that flowed into agentEvidence\n   * modalities, but in its native shape (e.g., the extract result, the act\n   * describe-string) rather than serialized for the LLM.\n   */\n  result: unknown;\n  error?: string;\n}\n\n/** One step in a trajectory: action + reasoning + evidence + outcome. */\nexport interface TrajectoryStep {\n  actionName: string;\n  actionArgs: Record<string, unknown>;\n  /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */\n  reasoning: string;\n  agentEvidence: AgentEvidence;\n  probeEvidence: ProbeEvidence;\n  toolOutput: ToolOutput;\n}\n\n/** Terminal status of the agent run. */\nexport type TrajectoryStatus = \"complete\" | \"aborted\" | \"stalled\" | \"error\";\n\n/**\n * Full trajectory for one task run.\n *\n * The on-disk layout is one directory per task:\n *\n *   .trajectories/<run-id>/<task-id>/\n *     ├── task_data.json           — TaskSpec + result metadata\n *     ├── trajectory.json          — this object, with image paths instead of bytes\n *     ├── screenshots/\n *     │   ├── probe/<N>.png        — tier-2 probe screenshot per step\n *     │   ├── probe/final.png      — final terminal observation screenshot\n *     │   └── agent/<N>.png        — tier-1 image the model received per step\n *     ├── scores/\n *     │   └── result.json          — Result from V3Evaluator.verify()\n *     └── core.log                 — captured action log\n */\nexport interface Trajectory {\n  task: TaskSpec;\n  steps: TrajectoryStep[];\n  finalAnswer?: string;\n  /** Terminal page observation captured after the agent finishes. */\n  finalObservation?: ProbeEvidence;\n  status: TrajectoryStatus;\n  usage: TrajectoryUsage;\n}\n\n/** A single screenshot kept by Step 1, ready for downstream relevance scoring. */\nexport interface CanonicalScreenshot {\n  /** 0-based position in the kept-screenshots array. Stable across the pipeline. */\n  canonicalIndex: number;\n  /**\n   * 0-based position in `Trajectory.steps` this screenshot came from\n   * (steps.length for the final observation). Lets downstream prompts\n   * cross-reference the action history.\n   */\n  originalStepIndex: number;\n  /** The resized PNG/JPEG buffer (or native bytes if sharp unavailable). */\n  bytes: Buffer;\n  /** MIME media type. Always \"image/png\" after the optional resize. */\n  mediaType: string;\n  /** Reason this frame was kept: \"first\" / \"last\" / \"diverges\". */\n  keptReason: \"first\" | \"last\" | \"diverges\" | \"no-dedup\";\n}\n\n/**\n * A text evidence point sourced from tier-2 probes or tier-1 tool outputs.\n * These feed the same relevance + scoring path as screenshots, letting DOM\n * and hybrid agents preserve extract/aria/tool-return evidence without a\n * separate verifier architecture.\n */\nexport interface CanonicalTextEvidence {\n  /** 0-based position in the combined evidence-point array. */\n  canonicalIndex: number;\n  originalStepIndex: number;\n  /** Where the text came from. */\n  source: \"probe-aria\" | \"agent-text\" | \"agent-json\" | \"tool-output\";\n  /** The text payload, already truncated. */\n  content: string;\n}\n\nexport type CanonicalEvidence = CanonicalScreenshot | CanonicalTextEvidence;\n\n/** Result of Step 1 evidence loading. */\nexport interface EvidenceLoadResult {\n  /** Kept frames, in chronological order. */\n  screenshots: CanonicalScreenshot[];\n  /**\n   * Maps trajectory step position → canonical index in `screenshots`. Steps\n   * whose screenshots were deduplicated point to the surviving canonical frame\n   * (typically the prior kept frame). Useful for \"find me the screenshot for\n   * step K\" lookups in downstream prompts.\n   */\n  stepIndexToCanonical: Map<number, number>;\n  /** Number of original frames considered. */\n  originalCount: number;\n  /** Number of frames kept post-dedup (== screenshots.length). */\n  keptCount: number;\n  /** Effective thresholds used (resolved from env). */\n  thresholds: {\n    ssim: number;\n    mse: number;\n    resize: number;\n  };\n}\n\n/** Options for evidence loading; primarily test seams over the defaults. */\nexport interface EvidenceLoadOptions {\n  /** SSIM similarity threshold for dedup (default 0.75). */\n  ssimThreshold?: number;\n  /** MSE similarity threshold for dedup (default 30). */\n  mseThreshold?: number;\n  /** Scale factor applied before relevance scoring (default 0.7). */\n  imageResize?: number;\n}\n\n/** Score for a single rubric criterion after evidence analysis + rescoring. */\nexport interface CriterionScore {\n  /** Matches RubricCriterion.criterion (the criterion's short name). */\n  criterion: string;\n  /** Maximum possible points for this criterion. */\n  maxPoints: number;\n  /**\n   * Points earned after evidence analysis. Null when the criterion is\n   * conditional and its condition was not met — excluded from both numerator\n   * and denominator in the process score.\n   */\n  earnedPoints: number | null;\n  /** Verifier's explanation for the score. */\n  explanation: string;\n  /**\n   * True if the criterion is conditional and its condition was determined to\n   * be met. Absent for non-conditional criteria.\n   */\n  conditionMet?: boolean;\n  /**\n   * Set when the verifier had no evidence to ground this criterion in either\n   * tier. Treated as uncontrollable failure (full credit) but surfaced here\n   * so dashboards can flag low-confidence results.\n   */\n  evidenceInsufficient?: boolean;\n}\n\n/**\n * Earliest step where the agent's trajectory went off-track, classified\n * against the error taxonomy (7 top-level categories, 1.1–7.4 sub-codes).\n */\nexport interface FirstPointOfFailure {\n  stepIndex: number;\n  /** Sub-code from the error taxonomy (e.g., \"2.3\" for a specific hallucination type). */\n  errorCode: string;\n  /** Top-level category name (Selection, Hallucination, etc.). */\n  category: string;\n  /** Verifier's reasoning for selecting this point. */\n  description?: string;\n}\n\n/**\n * Structured observation surfaced by the verifier that another agent or\n * tooling could act on. Findings are emitted opportunistically by Step 8\n * (outcome verification) when the verifier notices actionable patterns —\n * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.\n *\n * Not produced for every task: when nothing actionable surfaces, the\n * `findings` array on the EvaluationResult is empty. Consumers should treat the\n * field as advisory, not as part of the formal score.\n */\nexport interface VerifierFinding {\n  /**\n   * Category of the observation. Open-ended enum — additional categories may\n   * be added as verifier backends surface new failure modes.\n   */\n  category:\n    | \"agent_tool_usage\" // agent's tool calls had repeated issues (misclicks, wrong args, retries)\n    | \"agent_strategy\" // higher-level planning / decision-making problems\n    | \"rubric_quality\" // criteria were overly strict, ambiguous, or contradictory\n    | \"trajectory_capture\" // gaps in evidence (missing screenshots, empty steps)\n    | \"task_specification\" // task instruction was ambiguous / under- or over-specified\n    | \"verifier_uncertainty\" // verifier itself couldn't confidently decide\n    | \"other\";\n  /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */\n  severity: \"info\" | \"warning\" | \"blocking\";\n  /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */\n  description: string;\n  /**\n   * Optional concrete next action another agent could take. Should be\n   * specific enough that it can be acted on without further reasoning —\n   * e.g., \"Try double_click instead of triple_click to clear placeholder\n   * text on this form field.\"\n   */\n  suggestedAction?: string;\n  /** Step indices in the trajectory where this pattern showed up. */\n  relatedSteps?: number[];\n}\n\n/** Stable debugging summary emitted by verifier backends. */\nexport interface VerifierRawSteps {\n  backend?: \"legacy\" | \"verifier\";\n  reason?: string;\n  primaryIntent?: string;\n  reasoning?: string;\n  rubricSource?: \"precomputed\" | \"generated\" | \"none\";\n  approach?: \"a\" | \"b\" | \"outcome-only\";\n  optionalsMode?: \"folded\" | \"separate\" | \"skip\";\n  totalEarned?: number;\n  totalMax?: number;\n  evidenceImages?: number;\n  evidenceTexts?: number;\n  evidenceOriginalScreenshots?: number;\n  screenshotsAttached?: number;\n  legacyEvaluation?: string;\n  screenshotCount?: number;\n}\n\n/** Task-validity classification: whether the task is even answerable. */\nexport interface TaskValidity {\n  /** True if the task is underspecified / has multiple valid interpretations. */\n  isAmbiguous: boolean;\n  /** Explanation for why the task is ambiguous, when available. */\n  ambiguityReason?: string;\n  /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */\n  isInvalid: boolean;\n  /** Explanation for why the task is invalid, when available. */\n  invalidReason?: string;\n  /** Optional sub-codes from the task-classification taxonomy. */\n  ambiguityCodes?: string[];\n  invalidTaskCodes?: string[];\n}\n\n/**\n * Evaluator output. Legacy evaluation may only populate outcome fields; richer\n * verifier backends can also populate process scoring and diagnostics.\n *\n * Process and outcome are deliberately independent when both are present:\n * an agent can follow the right steps but get blocked (high process, low\n * outcome), or succeed through an unexpected path (variable process, high\n * outcome).\n */\nexport interface EvaluationResult {\n  /** Did the agent accomplish the task from the user's perspective? */\n  outcomeSuccess: boolean;\n  /** Human-readable explanation for the outcome. */\n  explanation?: string;\n  /** Aggregated earned/max across applicable criteria, in [0, 1]. */\n  processScore?: number;\n  /** Per-criterion breakdown after rescoring. */\n  perCriterion?: CriterionScore[];\n  /** Step 9a — first step where the trajectory went off-track, if any. */\n  firstPointOfFailure?: FirstPointOfFailure;\n  /** Step 10 — task-itself ambiguity / validity. */\n  taskValidity?: TaskValidity;\n  /**\n   * Ids (RubricCriterion.criterion strings) of criteria where neither tier of\n   * evidence resolved the question. Treated as uncontrollable → full credit,\n   * but flagged here so consumers can decide whether to discount the score.\n   */\n  evidenceInsufficient?: string[];\n  /**\n   * Structured observations from the verifier that a downstream tool or\n   * follow-up agent could act on. Opportunistic — empty when the verifier\n   * doesn't notice anything actionable. Not part of the score; advisory.\n   */\n  findings?: VerifierFinding[];\n  /** Debugging summary from the active evaluator backend. */\n  rawSteps?: VerifierRawSteps;\n}\n\n/**\n * Verifier interface. Implementations consume a Trajectory and return an\n * EvaluationResult — they MUST NOT touch a live browser.\n */\nexport interface Verifier {\n  verify(trajectory: Trajectory): Promise<EvaluationResult>;\n}\n\nexport interface RubricVerifierOptions {\n  /** Factory that returns a configured LLMClient. Called per pipeline step so callers can supply step-specific clients. */\n  getClient: () => LLMClient;\n  /** Optional factory for rubric generation so callers can route it to a stronger model. */\n  getRubricGenClient?: () => LLMClient;\n  /** Logger; defaults to a no-op so the verifier stays quiet inside V3Evaluator. */\n  logger?: (line: LogLine) => void;\n  /**\n   * Override any verifier knob. Env vars supply the defaults; values here win.\n   * Useful for tests and for cross-verify sweeps that want different budgets\n   * per run.\n   */\n  config?: Partial<VerifierConfig>;\n}\n\n/**\n * Resolved verifier knobs. Constructed once from env (and optional overrides)\n * by RubricVerifier's constructor; subsequent verify() calls can pass a\n * Partial to shift any field.\n */\nexport interface VerifierConfig {\n  /** Which pipeline path to take: per-criterion (a), fused (b), or skip rubric entirely (outcome-only). */\n  approach: \"a\" | \"b\" | \"outcome-only\";\n  /** Folded (in fused call), separate (own calls), or skip (omit). */\n  optionalSteps: \"folded\" | \"separate\" | \"skip\";\n  /** Top-K evidence points selected per criterion. */\n  topK: number;\n  /** Batch size for the relevance-scoring LLM call. */\n  relevanceBatchSize: number;\n  /** Image cap on the outcome-only path. */\n  outcomeMaxImages: number;\n  /** Concurrent LLM calls across batches / criteria. */\n  maxParallel: number;\n  /** Token budgets for the three evidence channels. */\n  evidenceTokenBudget: number;\n  outcomeEvidenceTokenBudget: number;\n  actionHistoryTokenBudget: number;\n  /** Per-section character limits applied during evidence-text assembly. */\n  truncation: {\n    /** Master switch: when true, all per-section limits go to MAX_SAFE_INTEGER. */\n    disabled: boolean;\n    evidenceTextPreview: number;\n    groupedEvidenceText: number;\n    buildEvidenceText: number;\n    buildEvidenceAria: number;\n    actionHistoryReasoning: number;\n  };\n}\n\nexport interface ErrorTaxonomySubCategory {\n  /** Sub-code (e.g., \"2.3\"). */\n  code: string;\n  /** Human-readable name (e.g., \"Output fabrication\"). */\n  name: string;\n  /** Detailed description ported from the .md. Markdown formatting preserved. */\n  description: string;\n}\n\nexport interface ErrorTaxonomyCategory {\n  /** Top-level number (1-8). */\n  number: number;\n  /** Top-level name (e.g., \"Hallucination Errors\"). */\n  name: string;\n  /** One-sentence summary of the category. */\n  summary: string;\n  /** Sub-categories. The last one is always an \"Other\" catch-all. */\n  subCategories: ErrorTaxonomySubCategory[];\n}\n\nexport interface ParseFailureStepNumbersOptions {\n  /**\n   * Maximum unique step numbers to expand from ranges. Protects the verifier\n   * from malformed model output such as \"0-2147483647\".\n   */\n  maxExpandedSteps?: number;\n  /** Optional inclusive upper bound for accepted step numbers. */\n  maxStep?: number;\n}\n"]}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../lib/v3/verifier/types.ts"],"names":[],"mappings":"","sourcesContent":["/**\n * Shared verifier types for trajectories, rubrics, evidence, and results.\n *\n * The verifier consumes saved trajectories instead of a live browser. DOM and\n * Hybrid runs preserve tool-return text/JSON evidence, while CUA runs preserve\n * screenshots sent to the provider plus independent harness probes.\n */\nimport type { LLMClient } from \"../llm/LLMClient.js\";\nimport type { LogLine } from \"../types/public/logs.js\";\n\n/** Token usage for one or more LLM calls. Matches AgentResult.usage shape. */\nexport interface TrajectoryUsage {\n  input_tokens: number;\n  output_tokens: number;\n  reasoning_tokens?: number;\n  cached_input_tokens?: number;\n  inference_time_ms?: number;\n}\n\n/** A single criterion in a Stagehand rubric. */\nexport interface RubricCriterion {\n  /** Short name of the criterion (e.g., \"Add ground beef to cart\"). */\n  criterion: string;\n  /** What to evaluate and how to award partial credit. */\n  description: string;\n  /** Maximum points for this criterion. */\n  maxPoints: number;\n  /**\n   * Applicability rule for situational criteria. When this condition is not\n   * met, the criterion is excluded from scoring rather than counted as failed.\n   */\n  condition?: string;\n}\n\n/** A rubric — list of criteria for a task. */\nexport interface Rubric {\n  items: RubricCriterion[];\n}\n\n/**\n * Spec for a single task being verified. Carried both at runtime and into the\n * verifier alongside the trajectory.\n */\nexport interface TaskSpec {\n  /** Stable identifier (e.g., \"united_13\" for WebTailBench, task_id for Mind2Web). */\n  id: string;\n  /** Task instruction shown to the agent. */\n  instruction: string;\n  /** Starting URL, if any. */\n  initUrl?: string;\n  /** Rubric carried by the dataset or generated by a verifier backend. */\n  precomputedRubric?: Rubric;\n  /** Optional reference answer (set when dataset ships one). */\n  expectedAnswer?: string;\n}\n\n/**\n * A single modality unit in tier-1 agent evidence. Mirrors the shape of\n * ModelMessage content parts so we can reproduce what the LLM ingested.\n */\nexport type AgentEvidenceModality =\n  | { type: \"text\"; content: string }\n  | { type: \"image\"; bytes: Buffer; mediaType: string }\n  | { type: \"json\"; content: unknown };\n\n/**\n * Tier 1 — exactly the bytes/strings/objects the agent's LLM ingested as the\n * tool result for this step.\n *\n * Modes:\n *   - CUA: usually a single image modality (the screenshot sent to the provider).\n *   - Hybrid: tool result with optional screenshotBase64 → one image + one text.\n *   - DOM: tool returns (extract JSON, ariaTree text, etc.) → text/json modalities.\n */\nexport interface AgentEvidence {\n  modalities: AgentEvidenceModality[];\n}\n\n/**\n * Tier 2 — independent harness probes around this step.\n *\n * If a probe wasn't captured, the field is absent (not null).\n */\nexport interface ProbeEvidence {\n  /** URL after the step's tool execution. */\n  url?: string;\n  /**\n   * Bus screenshot captured after the step. Path on disk is preferred once\n   * persisted; in-memory Buffer is used during a live run.\n   */\n  screenshot?: Buffer;\n  /** Reference to the persisted screenshot file under the trajectory dir. */\n  screenshotPath?: string;\n  /** Accessibility tree snapshot. */\n  ariaTree?: string;\n  /** Verifier-requested probes, keyed by criterion id. */\n  onDemand?: Record<string, unknown>;\n}\n\n/** Outcome of a single tool execution as seen by the harness. */\nexport interface ToolOutput {\n  ok: boolean;\n  /**\n   * The tool's return value. Same payload that flowed into agentEvidence\n   * modalities, but in its native shape (e.g., the extract result, the act\n   * describe-string) rather than serialized for the LLM.\n   */\n  result: unknown;\n  error?: string;\n}\n\n/** One step in a trajectory: action + reasoning + evidence + outcome. */\nexport interface TrajectoryStep {\n  actionName: string;\n  actionArgs: Record<string, unknown>;\n  /** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */\n  reasoning: string;\n  agentEvidence: AgentEvidence;\n  probeEvidence: ProbeEvidence;\n  toolOutput: ToolOutput;\n}\n\n/** Terminal status of the agent run. */\nexport type TrajectoryStatus = \"complete\" | \"aborted\" | \"stalled\" | \"error\";\n\n/**\n * Full trajectory for one task run.\n *\n * The on-disk layout is one directory per task run, under the run-scoped group\n * dir (`<experiment>__<model>__<runToken>`, or \"default\" for ad-hoc runs):\n *\n *   .trajectories/<group>/<task-id>/<run-id>/\n *     ├── task_data.json           — TaskSpec + result metadata\n *     ├── trajectory.json          — this object, with image paths instead of bytes\n *     ├── screenshots/\n *     │   ├── probe/<N>.png        — tier-2 probe screenshot per step\n *     │   ├── probe/final.png      — final terminal observation screenshot\n *     │   └── agent/<N>.png        — tier-1 image the model received per step\n *     ├── scores/\n *     │   └── result.json          — Result from V3Evaluator.verify()\n *     └── core.log                 — captured action log\n */\nexport interface Trajectory {\n  task: TaskSpec;\n  steps: TrajectoryStep[];\n  finalAnswer?: string;\n  /** Terminal page observation captured after the agent finishes. */\n  finalObservation?: ProbeEvidence;\n  status: TrajectoryStatus;\n  usage: TrajectoryUsage;\n}\n\n/** A single screenshot kept by Step 1, ready for downstream relevance scoring. */\nexport interface CanonicalScreenshot {\n  /** 0-based position in the kept-screenshots array. Stable across the pipeline. */\n  canonicalIndex: number;\n  /**\n   * 0-based position in `Trajectory.steps` this screenshot came from\n   * (steps.length for the final observation). Lets downstream prompts\n   * cross-reference the action history.\n   */\n  originalStepIndex: number;\n  /** The resized PNG/JPEG buffer (or native bytes if sharp unavailable). */\n  bytes: Buffer;\n  /** MIME media type. Always \"image/png\" after the optional resize. */\n  mediaType: string;\n  /** Reason this frame was kept: \"first\" / \"last\" / \"diverges\". */\n  keptReason: \"first\" | \"last\" | \"diverges\" | \"no-dedup\";\n}\n\n/**\n * A text evidence point sourced from tier-2 probes or tier-1 tool outputs.\n * These feed the same relevance + scoring path as screenshots, letting DOM\n * and hybrid agents preserve extract/aria/tool-return evidence without a\n * separate verifier architecture.\n */\nexport interface CanonicalTextEvidence {\n  /** 0-based position in the combined evidence-point array. */\n  canonicalIndex: number;\n  originalStepIndex: number;\n  /** Where the text came from. */\n  source: \"probe-aria\" | \"agent-text\" | \"agent-json\" | \"tool-output\";\n  /** The text payload, already truncated. */\n  content: string;\n}\n\nexport type CanonicalEvidence = CanonicalScreenshot | CanonicalTextEvidence;\n\n/** Result of Step 1 evidence loading. */\nexport interface EvidenceLoadResult {\n  /** Kept frames, in chronological order. */\n  screenshots: CanonicalScreenshot[];\n  /**\n   * Maps trajectory step position → canonical index in `screenshots`. Steps\n   * whose screenshots were deduplicated point to the surviving canonical frame\n   * (typically the prior kept frame). Useful for \"find me the screenshot for\n   * step K\" lookups in downstream prompts.\n   */\n  stepIndexToCanonical: Map<number, number>;\n  /** Number of original frames considered. */\n  originalCount: number;\n  /** Number of frames kept post-dedup (== screenshots.length). */\n  keptCount: number;\n  /** Effective thresholds used (resolved from env). */\n  thresholds: {\n    ssim: number;\n    mse: number;\n    resize: number;\n  };\n}\n\n/** Options for evidence loading; primarily test seams over the defaults. */\nexport interface EvidenceLoadOptions {\n  /** SSIM similarity threshold for dedup (default 0.75). */\n  ssimThreshold?: number;\n  /** MSE similarity threshold for dedup (default 30). */\n  mseThreshold?: number;\n  /** Scale factor applied before relevance scoring (default 0.7). */\n  imageResize?: number;\n}\n\n/** Score for a single rubric criterion after evidence analysis + rescoring. */\nexport interface CriterionScore {\n  /** Matches RubricCriterion.criterion (the criterion's short name). */\n  criterion: string;\n  /** Maximum possible points for this criterion. */\n  maxPoints: number;\n  /**\n   * Points earned after evidence analysis. Null when the criterion is\n   * conditional and its condition was not met — excluded from both numerator\n   * and denominator in the process score.\n   */\n  earnedPoints: number | null;\n  /** Verifier's explanation for the score. */\n  explanation: string;\n  /**\n   * True if the criterion is conditional and its condition was determined to\n   * be met. Absent for non-conditional criteria.\n   */\n  conditionMet?: boolean;\n  /**\n   * Set when the verifier had no evidence to ground this criterion in either\n   * tier. Treated as uncontrollable failure (full credit) but surfaced here\n   * so dashboards can flag low-confidence results.\n   */\n  evidenceInsufficient?: boolean;\n}\n\n/**\n * Earliest step where the agent's trajectory went off-track, classified\n * against the error taxonomy (7 top-level categories, 1.1–7.4 sub-codes).\n */\nexport interface FirstPointOfFailure {\n  stepIndex: number;\n  /** Sub-code from the error taxonomy (e.g., \"2.3\" for a specific hallucination type). */\n  errorCode: string;\n  /** Top-level category name (Selection, Hallucination, etc.). */\n  category: string;\n  /** Verifier's reasoning for selecting this point. */\n  description?: string;\n}\n\n/**\n * Structured observation surfaced by the verifier that another agent or\n * tooling could act on. Findings are emitted opportunistically by Step 8\n * (outcome verification) when the verifier notices actionable patterns —\n * repeated tool-call failures, ambiguous task specs, evidence gaps, etc.\n *\n * Not produced for every task: when nothing actionable surfaces, the\n * `findings` array on the EvaluationResult is empty. Consumers should treat the\n * field as advisory, not as part of the formal score.\n */\nexport interface VerifierFinding {\n  /**\n   * Category of the observation. Open-ended enum — additional categories may\n   * be added as verifier backends surface new failure modes.\n   */\n  category:\n    | \"agent_tool_usage\" // agent's tool calls had repeated issues (misclicks, wrong args, retries)\n    | \"agent_strategy\" // higher-level planning / decision-making problems\n    | \"rubric_quality\" // criteria were overly strict, ambiguous, or contradictory\n    | \"trajectory_capture\" // gaps in evidence (missing screenshots, empty steps)\n    | \"task_specification\" // task instruction was ambiguous / under- or over-specified\n    | \"verifier_uncertainty\" // verifier itself couldn't confidently decide\n    | \"other\";\n  /** Impact: info (FYI), warning (worth investigating), blocking (broke the task). */\n  severity: \"info\" | \"warning\" | \"blocking\";\n  /** What the verifier noticed. Plain prose, grounded in evidence from the trajectory. */\n  description: string;\n  /**\n   * Optional concrete next action another agent could take. Should be\n   * specific enough that it can be acted on without further reasoning —\n   * e.g., \"Try double_click instead of triple_click to clear placeholder\n   * text on this form field.\"\n   */\n  suggestedAction?: string;\n  /** Step indices in the trajectory where this pattern showed up. */\n  relatedSteps?: number[];\n}\n\n/** Stable debugging summary emitted by verifier backends. */\nexport interface VerifierRawSteps {\n  backend?: \"legacy\" | \"verifier\";\n  reason?: string;\n  primaryIntent?: string;\n  reasoning?: string;\n  rubricSource?: \"precomputed\" | \"generated\" | \"none\";\n  approach?: \"a\" | \"b\" | \"outcome-only\";\n  optionalsMode?: \"folded\" | \"separate\" | \"skip\";\n  totalEarned?: number;\n  totalMax?: number;\n  evidenceImages?: number;\n  evidenceTexts?: number;\n  evidenceOriginalScreenshots?: number;\n  screenshotsAttached?: number;\n  legacyEvaluation?: string;\n  screenshotCount?: number;\n}\n\n/** Task-validity classification: whether the task is even answerable. */\nexport interface TaskValidity {\n  /** True if the task is underspecified / has multiple valid interpretations. */\n  isAmbiguous: boolean;\n  /** Explanation for why the task is ambiguous, when available. */\n  ambiguityReason?: string;\n  /** True if the task is impossible / illegal / NSFW / otherwise infeasible. */\n  isInvalid: boolean;\n  /** Explanation for why the task is invalid, when available. */\n  invalidReason?: string;\n  /** Optional sub-codes from the task-classification taxonomy. */\n  ambiguityCodes?: string[];\n  invalidTaskCodes?: string[];\n}\n\n/**\n * Evaluator output. Legacy evaluation may only populate outcome fields; richer\n * verifier backends can also populate process scoring and diagnostics.\n *\n * Process and outcome are deliberately independent when both are present:\n * an agent can follow the right steps but get blocked (high process, low\n * outcome), or succeed through an unexpected path (variable process, high\n * outcome).\n */\nexport interface EvaluationResult {\n  /** Did the agent accomplish the task from the user's perspective? */\n  outcomeSuccess: boolean;\n  /** Human-readable explanation for the outcome. */\n  explanation?: string;\n  /** Aggregated earned/max across applicable criteria, in [0, 1]. */\n  processScore?: number;\n  /** Per-criterion breakdown after rescoring. */\n  perCriterion?: CriterionScore[];\n  /** Step 9a — first step where the trajectory went off-track, if any. */\n  firstPointOfFailure?: FirstPointOfFailure;\n  /** Step 10 — task-itself ambiguity / validity. */\n  taskValidity?: TaskValidity;\n  /**\n   * Ids (RubricCriterion.criterion strings) of criteria where neither tier of\n   * evidence resolved the question. Treated as uncontrollable → full credit,\n   * but flagged here so consumers can decide whether to discount the score.\n   */\n  evidenceInsufficient?: string[];\n  /**\n   * Structured observations from the verifier that a downstream tool or\n   * follow-up agent could act on. Opportunistic — empty when the verifier\n   * doesn't notice anything actionable. Not part of the score; advisory.\n   */\n  findings?: VerifierFinding[];\n  /** Debugging summary from the active evaluator backend. */\n  rawSteps?: VerifierRawSteps;\n}\n\n/**\n * Verifier interface. Implementations consume a Trajectory and return an\n * EvaluationResult — they MUST NOT touch a live browser.\n */\nexport interface Verifier {\n  verify(trajectory: Trajectory): Promise<EvaluationResult>;\n}\n\nexport interface RubricVerifierOptions {\n  /** Factory that returns a configured LLMClient. Called per pipeline step so callers can supply step-specific clients. */\n  getClient: () => LLMClient;\n  /** Optional factory for rubric generation so callers can route it to a stronger model. */\n  getRubricGenClient?: () => LLMClient;\n  /** Logger; defaults to a no-op so the verifier stays quiet inside V3Evaluator. */\n  logger?: (line: LogLine) => void;\n  /**\n   * Override any verifier knob. Env vars supply the defaults; values here win.\n   * Useful for tests and for cross-verify sweeps that want different budgets\n   * per run.\n   */\n  config?: Partial<VerifierConfig>;\n}\n\n/**\n * Resolved verifier knobs. Constructed once from env (and optional overrides)\n * by RubricVerifier's constructor; subsequent verify() calls can pass a\n * Partial to shift any field.\n */\nexport interface VerifierConfig {\n  /** Which pipeline path to take: per-criterion (a), fused (b), or skip rubric entirely (outcome-only). */\n  approach: \"a\" | \"b\" | \"outcome-only\";\n  /** Folded (in fused call), separate (own calls), or skip (omit). */\n  optionalSteps: \"folded\" | \"separate\" | \"skip\";\n  /** Top-K evidence points selected per criterion. */\n  topK: number;\n  /** Batch size for the relevance-scoring LLM call. */\n  relevanceBatchSize: number;\n  /** Image cap on the outcome-only path. */\n  outcomeMaxImages: number;\n  /** Concurrent LLM calls across batches / criteria. */\n  maxParallel: number;\n  /** Token budgets for the three evidence channels. */\n  evidenceTokenBudget: number;\n  outcomeEvidenceTokenBudget: number;\n  actionHistoryTokenBudget: number;\n  /** Per-section character limits applied during evidence-text assembly. */\n  truncation: {\n    /** Master switch: when true, all per-section limits go to MAX_SAFE_INTEGER. */\n    disabled: boolean;\n    evidenceTextPreview: number;\n    groupedEvidenceText: number;\n    buildEvidenceText: number;\n    buildEvidenceAria: number;\n    actionHistoryReasoning: number;\n  };\n}\n\nexport interface ErrorTaxonomySubCategory {\n  /** Sub-code (e.g., \"2.3\"). */\n  code: string;\n  /** Human-readable name (e.g., \"Output fabrication\"). */\n  name: string;\n  /** Detailed description ported from the .md. Markdown formatting preserved. */\n  description: string;\n}\n\nexport interface ErrorTaxonomyCategory {\n  /** Top-level number (1-8). */\n  number: number;\n  /** Top-level name (e.g., \"Hallucination Errors\"). */\n  name: string;\n  /** One-sentence summary of the category. */\n  summary: string;\n  /** Sub-categories. The last one is always an \"Other\" catch-all. */\n  subCategories: ErrorTaxonomySubCategory[];\n}\n\nexport interface ParseFailureStepNumbersOptions {\n  /**\n   * Maximum unique step numbers to expand from ranges. Protects the verifier\n   * from malformed model output such as \"0-2147483647\".\n   */\n  maxExpandedSteps?: number;\n  /** Optional inclusive upper bound for accepted step numbers. */\n  maxStep?: number;\n}\n"]}

package/dist/cjs/lib/version.d.ts

@@ -2,4 +2,4 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export declare const STAGEHAND_VERSION: "3.6.0";
+export declare const STAGEHAND_VERSION: "3.7.1";

package/dist/cjs/lib/version.js

@@ -5,5 +5,5 @@ exports.STAGEHAND_VERSION = void 0;
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-exports.STAGEHAND_VERSION = "3.6.0";
+exports.STAGEHAND_VERSION = "3.7.1";
 //# sourceMappingURL=version.js.map

package/dist/cjs/lib/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":";;;AAAA;;;GAGG;AACU,QAAA,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.6.0\" as const;\n"]}
+{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":";;;AAAA;;;GAGG;AACU,QAAA,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.7.1\" as const;\n"]}

package/dist/esm/lib/modelUtils.d.ts

@@ -1,4 +1,14 @@
 import { ClientOptions, ModelConfiguration } from "./v3/types/public/model.js";
+/**
+ * Sentinel model name that delegates model selection to the Stagehand API.
+ * Only valid when running through the API (env: "BROWSERBASE" with
+ * disableAPI: false); no local LLM client or provider API key is involved.
+ */
+export declare const AUTO_MODEL_NAME = "auto";
+export declare function isAutoModel(model?: string | {
+    modelName: string;
+    [key: string]: unknown;
+}): boolean;
 export declare function extractModelName(model?: string | {
     modelName: string;
     [key: string]: unknown;

package/dist/esm/lib/modelUtils.js

@@ -1,4 +1,13 @@
 import { AVAILABLE_CUA_MODELS, } from "./v3/types/public/agent.js";
+/**
+ * Sentinel model name that delegates model selection to the Stagehand API.
+ * Only valid when running through the API (env: "BROWSERBASE" with
+ * disableAPI: false); no local LLM client or provider API key is involved.
+ */
+export const AUTO_MODEL_NAME = "auto";
+export function isAutoModel(model) {
+    return extractModelName(model) === AUTO_MODEL_NAME;
+}
 //useful when resolving a model from string or object formats we accept
 export function extractModelName(model) {
     if (!model)

package/dist/esm/lib/modelUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelUtils.js","sourceRoot":"","sources":["../../../lib/modelUtils.ts"],"names":[],"mappings":"AACA,OAAO,EACL,oBAAoB,GAErB,MAAM,4BAA4B,CAAC;AAEpC,uEAAuE;AACvE,MAAM,UAAU,gBAAgB,CAC9B,KAA8D;IAE9D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAI1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkC;IAM7D,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAE,CAAC;IAC7C,MAAM,aAAa,GACjB,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,CAAC,GAAG,EAAE;YACJ,6DAA6D;YAC7D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IAEX,uDAAuD;IACvD,MAAM,mBAAmB,GAAG,aAAa,CAAC,QAAQ,KAAK,SAAS,CAAC;IAEjE,oFAAoF;IACpF,IAAI,QAAgB,CAAC;IACrB,IAAI,eAAuB,CAAC;IAE5B,IAAI,mBAAmB,EAAE,CAAC;QACxB,QAAQ,GAAG,aAAa,CAAC,QAAkB,CAAC;QAC5C,eAAe,GAAG,WAAW,CAAC,CAAC,2BAA2B;IAC5D,CAAC;SAAM,CAAC;QACN,kCAAkC;QAClC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC1C,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC1B,eAAe,GAAG,KAAK,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,4BAA4B;IAC5B,MAAM,KAAK,GACT,mBAAmB;QACnB,oBAAoB,CAAC,QAAQ,CAAC,WAAgC,CAAC,CAAC;IAElE,OAAO;QACL,QAAQ;QACR,SAAS,EAAE,eAAe;QAC1B,aAAa;QACb,KAAK;KACN,CAAC;AACJ,CAAC","sourcesContent":["import { ClientOptions, ModelConfiguration } from \"./v3/types/public/model.js\";\nimport {\n  AVAILABLE_CUA_MODELS,\n  AvailableCuaModel,\n} from \"./v3/types/public/agent.js\";\n\n//useful when resolving a model from string or object formats we accept\nexport function extractModelName(\n  model?: string | { modelName: string; [key: string]: unknown },\n): string | undefined {\n  if (!model) return undefined;\n  return typeof model === \"string\" ? model : model.modelName;\n}\n\nexport function splitModelName(model: string): {\n  provider: string;\n  modelName: string;\n} {\n  const firstSlashIndex = model.indexOf(\"/\");\n  const provider = model.substring(0, firstSlashIndex);\n  const modelName = model.substring(firstSlashIndex + 1);\n  return { provider, modelName };\n}\n\nexport function resolveModel(model: string | ModelConfiguration): {\n  provider: string;\n  modelName: string;\n  clientOptions: ClientOptions;\n  isCua: boolean;\n} {\n  const modelString = extractModelName(model)!;\n  const clientOptions =\n    typeof model === \"string\"\n      ? {}\n      : (() => {\n          // eslint-disable-next-line @typescript-eslint/no-unused-vars\n          const { modelName: _, ...rest } = model;\n          return rest;\n        })();\n\n  // Check if provider is explicitly set in clientOptions\n  const hasExplicitProvider = clientOptions.provider !== undefined;\n\n  // If provider is explicitly set, don't split the model name - pass it through as-is\n  let provider: string;\n  let parsedModelName: string;\n\n  if (hasExplicitProvider) {\n    provider = clientOptions.provider as string;\n    parsedModelName = modelString; // Keep the full model name\n  } else {\n    // Parse the model string normally\n    const split = splitModelName(modelString);\n    provider = split.provider;\n    parsedModelName = split.modelName;\n  }\n\n  // Check if it's a CUA model\n  const isCua =\n    hasExplicitProvider ||\n    AVAILABLE_CUA_MODELS.includes(modelString as AvailableCuaModel);\n\n  return {\n    provider,\n    modelName: parsedModelName,\n    clientOptions,\n    isCua,\n  };\n}\n"]}
+{"version":3,"file":"modelUtils.js","sourceRoot":"","sources":["../../../lib/modelUtils.ts"],"names":[],"mappings":"AACA,OAAO,EACL,oBAAoB,GAErB,MAAM,4BAA4B,CAAC;AAEpC;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAC;AAEtC,MAAM,UAAU,WAAW,CACzB,KAA8D;IAE9D,OAAO,gBAAgB,CAAC,KAAK,CAAC,KAAK,eAAe,CAAC;AACrD,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,gBAAgB,CAC9B,KAA8D;IAE9D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAI1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkC;IAM7D,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAE,CAAC;IAC7C,MAAM,aAAa,GACjB,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,CAAC,GAAG,EAAE;YACJ,6DAA6D;YAC7D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IAEX,uDAAuD;IACvD,MAAM,mBAAmB,GAAG,aAAa,CAAC,QAAQ,KAAK,SAAS,CAAC;IAEjE,oFAAoF;IACpF,IAAI,QAAgB,CAAC;IACrB,IAAI,eAAuB,CAAC;IAE5B,IAAI,mBAAmB,EAAE,CAAC;QACxB,QAAQ,GAAG,aAAa,CAAC,QAAkB,CAAC;QAC5C,eAAe,GAAG,WAAW,CAAC,CAAC,2BAA2B;IAC5D,CAAC;SAAM,CAAC;QACN,kCAAkC;QAClC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC1C,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC1B,eAAe,GAAG,KAAK,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,4BAA4B;IAC5B,MAAM,KAAK,GACT,mBAAmB;QACnB,oBAAoB,CAAC,QAAQ,CAAC,WAAgC,CAAC,CAAC;IAElE,OAAO;QACL,QAAQ;QACR,SAAS,EAAE,eAAe;QAC1B,aAAa;QACb,KAAK;KACN,CAAC;AACJ,CAAC","sourcesContent":["import { ClientOptions, ModelConfiguration } from \"./v3/types/public/model.js\";\nimport {\n  AVAILABLE_CUA_MODELS,\n  AvailableCuaModel,\n} from \"./v3/types/public/agent.js\";\n\n/**\n * Sentinel model name that delegates model selection to the Stagehand API.\n * Only valid when running through the API (env: \"BROWSERBASE\" with\n * disableAPI: false); no local LLM client or provider API key is involved.\n */\nexport const AUTO_MODEL_NAME = \"auto\";\n\nexport function isAutoModel(\n  model?: string | { modelName: string; [key: string]: unknown },\n): boolean {\n  return extractModelName(model) === AUTO_MODEL_NAME;\n}\n\n//useful when resolving a model from string or object formats we accept\nexport function extractModelName(\n  model?: string | { modelName: string; [key: string]: unknown },\n): string | undefined {\n  if (!model) return undefined;\n  return typeof model === \"string\" ? model : model.modelName;\n}\n\nexport function splitModelName(model: string): {\n  provider: string;\n  modelName: string;\n} {\n  const firstSlashIndex = model.indexOf(\"/\");\n  const provider = model.substring(0, firstSlashIndex);\n  const modelName = model.substring(firstSlashIndex + 1);\n  return { provider, modelName };\n}\n\nexport function resolveModel(model: string | ModelConfiguration): {\n  provider: string;\n  modelName: string;\n  clientOptions: ClientOptions;\n  isCua: boolean;\n} {\n  const modelString = extractModelName(model)!;\n  const clientOptions =\n    typeof model === \"string\"\n      ? {}\n      : (() => {\n          // eslint-disable-next-line @typescript-eslint/no-unused-vars\n          const { modelName: _, ...rest } = model;\n          return rest;\n        })();\n\n  // Check if provider is explicitly set in clientOptions\n  const hasExplicitProvider = clientOptions.provider !== undefined;\n\n  // If provider is explicitly set, don't split the model name - pass it through as-is\n  let provider: string;\n  let parsedModelName: string;\n\n  if (hasExplicitProvider) {\n    provider = clientOptions.provider as string;\n    parsedModelName = modelString; // Keep the full model name\n  } else {\n    // Parse the model string normally\n    const split = splitModelName(modelString);\n    provider = split.provider;\n    parsedModelName = split.modelName;\n  }\n\n  // Check if it's a CUA model\n  const isCua =\n    hasExplicitProvider ||\n    AVAILABLE_CUA_MODELS.includes(modelString as AvailableCuaModel);\n\n  return {\n    provider,\n    modelName: parsedModelName,\n    clientOptions,\n    isCua,\n  };\n}\n"]}

package/dist/esm/lib/v3/agent/AgentProvider.js

@@ -9,6 +9,9 @@ export const modelToAgentProviderMap = {
     "gpt-5.4": "openai",
     "gpt-5.4-mini": "openai",
     "gpt-5.5": "openai",
+    "gpt-5.6-terra": "openai",
+    "gpt-5.6-luna": "openai",
+    "gpt-5.6-sol": "openai",
     "computer-use-preview": "openai",
     "computer-use-preview-2025-03-11": "openai",
     "claude-sonnet-4-20250514": "anthropic",

package/dist/esm/lib/v3/agent/AgentProvider.js.map

@@ -1 +1 @@
-{"version":3,"file":"AgentProvider.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/AgentProvider.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EACL,qBAAqB,EACrB,6BAA6B,GAC9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,0CAA0C;AAC1C,MAAM,CAAC,MAAM,uBAAuB,GAAsC;IACxE,SAAS,EAAE,QAAQ;IACnB,cAAc,EAAE,QAAQ;IACxB,SAAS,EAAE,QAAQ;IACnB,sBAAsB,EAAE,QAAQ;IAChC,iCAAiC,EAAE,QAAQ;IAC3C,0BAA0B,EAAE,WAAW;IACvC,4BAA4B,EAAE,WAAW;IACzC,0BAA0B,EAAE,WAAW;IACvC,iBAAiB,EAAE,WAAW;IAC9B,iBAAiB,EAAE,WAAW;IAC9B,mBAAmB,EAAE,WAAW;IAChC,kBAAkB,EAAE,WAAW;IAC/B,2BAA2B,EAAE,WAAW;IACxC,gBAAgB,EAAE,WAAW;IAC7B,yCAAyC,EAAE,QAAQ;IACnD,wBAAwB,EAAE,QAAQ;IAClC,kBAAkB,EAAE,QAAQ;IAC5B,sBAAsB,EAAE,QAAQ;IAChC,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF;;;;GAIG;AACH,MAAM,OAAO,aAAa;IAChB,MAAM,CAA6B;IAE3C;;OAEG;IACH,YAAY,MAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CACP,SAAiB,EACjB,aAA6B,EAC7B,wBAAiC,EACjC,KAAe;QAEf,uDAAuD;QACvD,MAAM,gBAAgB,GAAG,aAAa,EAAE,QAE3B,CAAC;QACd,MAAM,IAAI,GAAG,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,kCAAkC,IAAI,YAAY,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE;YACvH,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,QAAQ;oBACX,OAAO,IAAI,eAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,kBAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,QAAQ;oBACX,OAAO,IAAI,eAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,kBAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,CACd,CAAC;gBACJ;oBACE,MAAM,IAAI,6BAA6B,CACrC,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAC9C,oBAAoB,CACrB,CAAC;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC,YAAY,EAAE;gBACvD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,SAAiB;QACvC,MAAM,UAAU,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAEjD,IAAI,UAAU,IAAI,uBAAuB,EAAE,CAAC;YAC1C,OAAO,uBAAuB,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,qBAAqB,CAC7B,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,EACpC,oBAAoB,CACrB,CAAC;IACJ,CAAC;CACF","sourcesContent":["import { ToolSet } from \"ai\";\nimport { stripModelProvider } from \"../../utils.js\";\nimport { AgentProviderType } from \"../types/public/agent.js\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport {\n  UnsupportedModelError,\n  UnsupportedModelProviderError,\n} from \"../types/public/sdkErrors.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AnthropicCUAClient } from \"./AnthropicCUAClient.js\";\nimport { OpenAICUAClient } from \"./OpenAICUAClient.js\";\nimport { GoogleCUAClient } from \"./GoogleCUAClient.js\";\nimport { MicrosoftCUAClient } from \"./MicrosoftCUAClient.js\";\n\n// Map model names to their provider types\nexport const modelToAgentProviderMap: Record<string, AgentProviderType> = {\n  \"gpt-5.4\": \"openai\",\n  \"gpt-5.4-mini\": \"openai\",\n  \"gpt-5.5\": \"openai\",\n  \"computer-use-preview\": \"openai\",\n  \"computer-use-preview-2025-03-11\": \"openai\",\n  \"claude-sonnet-4-20250514\": \"anthropic\",\n  \"claude-sonnet-4-5-20250929\": \"anthropic\",\n  \"claude-opus-4-5-20251101\": \"anthropic\",\n  \"claude-opus-4-6\": \"anthropic\",\n  \"claude-opus-4-8\": \"anthropic\",\n  \"claude-sonnet-4-6\": \"anthropic\",\n  \"claude-haiku-4-5\": \"anthropic\",\n  \"claude-haiku-4-5-20251001\": \"anthropic\",\n  \"claude-fable-5\": \"anthropic\",\n  \"gemini-2.5-computer-use-preview-10-2025\": \"google\",\n  \"gemini-3-flash-preview\": \"google\",\n  \"gemini-3.5-flash\": \"google\",\n  \"gemini-3-pro-preview\": \"google\",\n  \"fara-7b\": \"microsoft\",\n};\n\n/**\n * Provider for agent clients\n * This class is responsible for creating the appropriate agent client\n * based on the provider type\n */\nexport class AgentProvider {\n  private logger: (message: LogLine) => void;\n\n  /**\n   * Create a new agent provider\n   */\n  constructor(logger: (message: LogLine) => void) {\n    this.logger = logger;\n  }\n\n  getClient(\n    modelName: string,\n    clientOptions?: ClientOptions,\n    userProvidedInstructions?: string,\n    tools?: ToolSet,\n  ): AgentClient {\n    // Check if provider is explicitly set in clientOptions\n    const explicitProvider = clientOptions?.provider as\n      | AgentProviderType\n      | undefined;\n    const type = explicitProvider || AgentProvider.getAgentProvider(modelName);\n\n    this.logger({\n      category: \"agent\",\n      message: `Getting agent client for type: ${type}, model: ${modelName}${explicitProvider ? \" (explicit provider)\" : \"\"}`,\n      level: 2,\n    });\n\n    try {\n      switch (type) {\n        case \"openai\":\n          return new OpenAICUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"anthropic\":\n          return new AnthropicCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"google\":\n          return new GoogleCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"microsoft\":\n          return new MicrosoftCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n          );\n        default:\n          throw new UnsupportedModelProviderError(\n            [\"openai\", \"anthropic\", \"google\", \"microsoft\"],\n            \"Computer Use Agent\",\n          );\n      }\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      this.logger({\n        category: \"agent\",\n        message: `Error creating agent client: ${errorMessage}`,\n        level: 0,\n      });\n      throw error;\n    }\n  }\n\n  static getAgentProvider(modelName: string): AgentProviderType {\n    const normalized = stripModelProvider(modelName);\n\n    if (normalized in modelToAgentProviderMap) {\n      return modelToAgentProviderMap[normalized];\n    }\n\n    throw new UnsupportedModelError(\n      Object.keys(modelToAgentProviderMap),\n      \"Computer Use Agent\",\n    );\n  }\n}\n"]}
+{"version":3,"file":"AgentProvider.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/AgentProvider.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EACL,qBAAqB,EACrB,6BAA6B,GAC9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,0CAA0C;AAC1C,MAAM,CAAC,MAAM,uBAAuB,GAAsC;IACxE,SAAS,EAAE,QAAQ;IACnB,cAAc,EAAE,QAAQ;IACxB,SAAS,EAAE,QAAQ;IACnB,eAAe,EAAE,QAAQ;IACzB,cAAc,EAAE,QAAQ;IACxB,aAAa,EAAE,QAAQ;IACvB,sBAAsB,EAAE,QAAQ;IAChC,iCAAiC,EAAE,QAAQ;IAC3C,0BAA0B,EAAE,WAAW;IACvC,4BAA4B,EAAE,WAAW;IACzC,0BAA0B,EAAE,WAAW;IACvC,iBAAiB,EAAE,WAAW;IAC9B,iBAAiB,EAAE,WAAW;IAC9B,mBAAmB,EAAE,WAAW;IAChC,kBAAkB,EAAE,WAAW;IAC/B,2BAA2B,EAAE,WAAW;IACxC,gBAAgB,EAAE,WAAW;IAC7B,yCAAyC,EAAE,QAAQ;IACnD,wBAAwB,EAAE,QAAQ;IAClC,kBAAkB,EAAE,QAAQ;IAC5B,sBAAsB,EAAE,QAAQ;IAChC,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF;;;;GAIG;AACH,MAAM,OAAO,aAAa;IAChB,MAAM,CAA6B;IAE3C;;OAEG;IACH,YAAY,MAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CACP,SAAiB,EACjB,aAA6B,EAC7B,wBAAiC,EACjC,KAAe;QAEf,uDAAuD;QACvD,MAAM,gBAAgB,GAAG,aAAa,EAAE,QAE3B,CAAC;QACd,MAAM,IAAI,GAAG,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,kCAAkC,IAAI,YAAY,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE;YACvH,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,QAAQ;oBACX,OAAO,IAAI,eAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,kBAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,QAAQ;oBACX,OAAO,IAAI,eAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,kBAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,CACd,CAAC;gBACJ;oBACE,MAAM,IAAI,6BAA6B,CACrC,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAC9C,oBAAoB,CACrB,CAAC;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC,YAAY,EAAE;gBACvD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,SAAiB;QACvC,MAAM,UAAU,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAEjD,IAAI,UAAU,IAAI,uBAAuB,EAAE,CAAC;YAC1C,OAAO,uBAAuB,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,qBAAqB,CAC7B,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,EACpC,oBAAoB,CACrB,CAAC;IACJ,CAAC;CACF","sourcesContent":["import { ToolSet } from \"ai\";\nimport { stripModelProvider } from \"../../utils.js\";\nimport { AgentProviderType } from \"../types/public/agent.js\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport {\n  UnsupportedModelError,\n  UnsupportedModelProviderError,\n} from \"../types/public/sdkErrors.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AnthropicCUAClient } from \"./AnthropicCUAClient.js\";\nimport { OpenAICUAClient } from \"./OpenAICUAClient.js\";\nimport { GoogleCUAClient } from \"./GoogleCUAClient.js\";\nimport { MicrosoftCUAClient } from \"./MicrosoftCUAClient.js\";\n\n// Map model names to their provider types\nexport const modelToAgentProviderMap: Record<string, AgentProviderType> = {\n  \"gpt-5.4\": \"openai\",\n  \"gpt-5.4-mini\": \"openai\",\n  \"gpt-5.5\": \"openai\",\n  \"gpt-5.6-terra\": \"openai\",\n  \"gpt-5.6-luna\": \"openai\",\n  \"gpt-5.6-sol\": \"openai\",\n  \"computer-use-preview\": \"openai\",\n  \"computer-use-preview-2025-03-11\": \"openai\",\n  \"claude-sonnet-4-20250514\": \"anthropic\",\n  \"claude-sonnet-4-5-20250929\": \"anthropic\",\n  \"claude-opus-4-5-20251101\": \"anthropic\",\n  \"claude-opus-4-6\": \"anthropic\",\n  \"claude-opus-4-8\": \"anthropic\",\n  \"claude-sonnet-4-6\": \"anthropic\",\n  \"claude-haiku-4-5\": \"anthropic\",\n  \"claude-haiku-4-5-20251001\": \"anthropic\",\n  \"claude-fable-5\": \"anthropic\",\n  \"gemini-2.5-computer-use-preview-10-2025\": \"google\",\n  \"gemini-3-flash-preview\": \"google\",\n  \"gemini-3.5-flash\": \"google\",\n  \"gemini-3-pro-preview\": \"google\",\n  \"fara-7b\": \"microsoft\",\n};\n\n/**\n * Provider for agent clients\n * This class is responsible for creating the appropriate agent client\n * based on the provider type\n */\nexport class AgentProvider {\n  private logger: (message: LogLine) => void;\n\n  /**\n   * Create a new agent provider\n   */\n  constructor(logger: (message: LogLine) => void) {\n    this.logger = logger;\n  }\n\n  getClient(\n    modelName: string,\n    clientOptions?: ClientOptions,\n    userProvidedInstructions?: string,\n    tools?: ToolSet,\n  ): AgentClient {\n    // Check if provider is explicitly set in clientOptions\n    const explicitProvider = clientOptions?.provider as\n      | AgentProviderType\n      | undefined;\n    const type = explicitProvider || AgentProvider.getAgentProvider(modelName);\n\n    this.logger({\n      category: \"agent\",\n      message: `Getting agent client for type: ${type}, model: ${modelName}${explicitProvider ? \" (explicit provider)\" : \"\"}`,\n      level: 2,\n    });\n\n    try {\n      switch (type) {\n        case \"openai\":\n          return new OpenAICUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"anthropic\":\n          return new AnthropicCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"google\":\n          return new GoogleCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"microsoft\":\n          return new MicrosoftCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n          );\n        default:\n          throw new UnsupportedModelProviderError(\n            [\"openai\", \"anthropic\", \"google\", \"microsoft\"],\n            \"Computer Use Agent\",\n          );\n      }\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      this.logger({\n        category: \"agent\",\n        message: `Error creating agent client: ${errorMessage}`,\n        level: 0,\n      });\n      throw error;\n    }\n  }\n\n  static getAgentProvider(modelName: string): AgentProviderType {\n    const normalized = stripModelProvider(modelName);\n\n    if (normalized in modelToAgentProviderMap) {\n      return modelToAgentProviderMap[normalized];\n    }\n\n    throw new UnsupportedModelError(\n      Object.keys(modelToAgentProviderMap),\n      \"Computer Use Agent\",\n    );\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/tools/click.js

@@ -30,9 +30,18 @@ export const clickTool = (v3, provider) => tool({
             });
             // Only request XPath when caching is enabled to avoid unnecessary computation
             const shouldCollectXpath = v3.isAgentReplayActive();
-            const xpath = await page.click(processed.x, processed.y, {
-                returnXpath: shouldCollectXpath,
-            });
+            // On a touch session, actuate as a trusted tap rather than a mouse click:
+            // mobile layouts commonly gate their handlers on touch/pointer events, where
+            // a synthesized mouse click never registers (a size selector highlights but
+            // still reports "please choose a size"). Mirrors the CUA handler's routing.
+            const useTouch = v3.usesTouch;
+            const xpath = useTouch
+                ? await page.tap(processed.x, processed.y, {
+                    returnXpath: shouldCollectXpath,
+                })
+                : await page.click(processed.x, processed.y, {
+                    returnXpath: shouldCollectXpath,
+                });
             const screenshotBase64 = await waitAndCaptureScreenshot(page);
             // Record as an "act" step with proper Action for deterministic replay (only when caching)
             if (shouldCollectXpath) {
@@ -41,7 +50,9 @@ export const clickTool = (v3, provider) => tool({
                     const action = {
                         selector: normalizedXpath,
                         description: describe,
-                        method: "click",
+                        // Replay the same input kind that was recorded, so a cached mobile
+                        // run does not silently fall back to a mouse click.
+                        method: useTouch ? "tap" : "click",
                         arguments: [],
                     };
                     v3.recordAgentReplayStep({

package/dist/esm/lib/v3/agent/tools/click.js.map

@@ -1 +1 @@
-{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,0EAA0E;YAC1E,6EAA6E;YAC7E,4EAA4E;YAC5E,4EAA4E;YAC5E,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;YAC9B,MAAM,KAAK,GAAG,QAAQ;gBACpB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvC,WAAW,EAAE,kBAAkB;iBAChC,CAAC;gBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACzC,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;YAEP,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,mEAAmE;wBACnE,oDAAoD;wBACpD,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;wBAClC,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // On a touch session, actuate as a trusted tap rather than a mouse click:\n        // mobile layouts commonly gate their handlers on touch/pointer events, where\n        // a synthesized mouse click never registers (a size selector highlights but\n        // still reports \"please choose a size\"). Mirrors the CUA handler's routing.\n        const useTouch = v3.usesTouch;\n        const xpath = useTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              // Replay the same input kind that was recorded, so a cached mobile\n              // run does not silently fall back to a mouse click.\n              method: useTouch ? \"tap\" : \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/fillFormVision.js

@@ -74,10 +74,18 @@ MANDATORY USE CASES (always use fillFormVision for these):
                 const shouldCollectXpath = v3.isAgentReplayActive();
                 const actions = [];
                 for (const field of safeFields) {
-                    // Click the field, only requesting XPath when caching is enabled
-                    const xpath = await page.click(field.coordinates.x, field.coordinates.y, {
-                        returnXpath: shouldCollectXpath,
-                    });
+                    // Focus the field, only requesting XPath when caching is enabled. On a
+                    // touch session this must be a trusted tap — a synthesized mouse click
+                    // never registers on touch-gated mobile layouts, so the field would stay
+                    // unfocused and the value go nowhere. The recorded step stays a "type",
+                    // whose replay focuses via the selector rather than a pointer event.
+                    const xpath = v3.usesTouch
+                        ? await page.tap(field.coordinates.x, field.coordinates.y, {
+                            returnXpath: shouldCollectXpath,
+                        })
+                        : await page.click(field.coordinates.x, field.coordinates.y, {
+                            returnXpath: shouldCollectXpath,
+                        });
                     // Substitute variables only at the moment of typing so the resolved
                     // value never leaves this scope.
                     await page.type(substituteVariables(field.value, variables));

package/dist/esm/lib/v3/agent/tools/fillFormVision.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates per field. Keep the original `value` (with any\n        // `%variableName%` tokens) so substituted secrets never leak through\n        // the tool result, replay cache, returned AgentResult.actions, or\n        // anything that logs the action. Variables are substituted only at\n        // typing time below.\n        const safeFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            action: field.action,\n            value: field.value,\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of safeFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          // Substitute variables only at the moment of typing so the resolved\n          // value never leaves this scope.\n          await page.type(substituteVariables(field.value, variables));\n\n          // Build Action with XPath for deterministic replay (only when caching).\n          // Store the placeholder value so cache entries don't contain secrets.\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: safeFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,uEAAuE;oBACvE,uEAAuE;oBACvE,yEAAyE;oBACzE,wEAAwE;oBACxE,qEAAqE;oBACrE,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;wBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACvD,WAAW,EAAE,kBAAkB;yBAChC,CAAC;wBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACzD,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACP,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates per field. Keep the original `value` (with any\n        // `%variableName%` tokens) so substituted secrets never leak through\n        // the tool result, replay cache, returned AgentResult.actions, or\n        // anything that logs the action. Variables are substituted only at\n        // typing time below.\n        const safeFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            action: field.action,\n            value: field.value,\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of safeFields) {\n          // Focus the field, only requesting XPath when caching is enabled. On a\n          // touch session this must be a trusted tap — a synthesized mouse click\n          // never registers on touch-gated mobile layouts, so the field would stay\n          // unfocused and the value go nowhere. The recorded step stays a \"type\",\n          // whose replay focuses via the selector rather than a pointer event.\n          const xpath = v3.usesTouch\n            ? await page.tap(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              })\n            : await page.click(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              });\n          // Substitute variables only at the moment of typing so the resolved\n          // value never leaves this scope.\n          await page.type(substituteVariables(field.value, variables));\n\n          // Build Action with XPath for deterministic replay (only when caching).\n          // Store the placeholder value so cache entries don't contain secrets.\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: safeFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/type.js

@@ -39,9 +39,18 @@ export const typeTool = (v3, provider, variables) => {
                 });
                 // Only request XPath when caching is enabled to avoid unnecessary computation
                 const shouldCollectXpath = v3.isAgentReplayActive();
-                const xpath = await page.click(processed.x, processed.y, {
-                    returnXpath: shouldCollectXpath,
-                });
+                // Focus the field the way the session's input actually works: on a touch
+                // session a synthesized mouse click never registers on touch-gated mobile
+                // layouts, so the field would stay unfocused and the text go nowhere.
+                // Only the actuation changes — the recorded step is a "type", whose replay
+                // focuses via the selector rather than a pointer event.
+                const xpath = v3.usesTouch
+                    ? await page.tap(processed.x, processed.y, {
+                        returnXpath: shouldCollectXpath,
+                    })
+                    : await page.click(processed.x, processed.y, {
+                        returnXpath: shouldCollectXpath,
+                    });
                 await page.type(actualText);
                 const screenshotBase64 = await waitAndCaptureScreenshot(page);
                 // Record as an "act" step with proper Action for deterministic replay (only when caching)

package/dist/esm/lib/v3/agent/tools/type.js.map

@@ -1 +1 @@
-{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}
+{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,yEAAyE;gBACzE,0EAA0E;gBAC1E,sEAAsE;gBACtE,2EAA2E;gBAC3E,wDAAwD;gBACxD,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;oBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACvC,WAAW,EAAE,kBAAkB;qBAChC,CAAC;oBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACzC,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;gBAEP,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // Focus the field the way the session's input actually works: on a touch\n        // session a synthesized mouse click never registers on touch-gated mobile\n        // layouts, so the field would stay unfocused and the text go nowhere.\n        // Only the actuation changes — the recorded step is a \"type\", whose replay\n        // focuses via the selector rather than a pointer event.\n        const xpath = v3.usesTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js

@@ -3,6 +3,7 @@ import { z } from "zod";
 import { tool } from "ai";
 import { anthropicFallbacksOptions, rejectsForcedToolUse, } from "../../llm/anthropicOptions.js";
 import { getZFactory } from "../../../utils.js";
+import { openAIStoreProviderOptions } from "../../llm/providerOptions.js";
 /**
  * Deep-remove `undefined` values from the run history before it is re-submitted
  * to the forced "done" call.
@@ -105,6 +106,7 @@ Call the "done" tool with:
             : "Provide your final assessment.",
     };
     const modelId = typeof model === "string" ? model : model.modelId;
+    const modelProvider = typeof model === "string" ? undefined : model.provider;
     const fallbacks = anthropicFallbacksOptions(modelId);
     // Models whose always-on thinking rejects forced tool use go straight to
     // "auto" — the prompt already instructs calling "done", and the
@@ -119,7 +121,7 @@ Call the "done" tool with:
             : { type: "tool", toolName: "done" },
         providerOptions: {
             google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
-            openai: { store: false },
+            ...openAIStoreProviderOptions(modelProvider),
             ...(fallbacks ? { anthropic: fallbacks } : {}),
         },
     });

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js.map

@@ -1 +1 @@
-{"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAwC,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAE1B,OAAO,EACL,yBAAyB,EACzB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAUhD;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,kBAAkB,CAAI,KAAQ;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAiB,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjD,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,KAAK,SAAS;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,GAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,QAAwB;IAExB,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,WAAW,CAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAI,CAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;IAClE,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAErD,yEAAyE;IACzE,gEAAgE;IAChE,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,+BAA+B,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC;QACzE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,oBAAoB,CAAC,OAAO,CAAC;YACvC,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QACtC,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;YACxB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport {\n  anthropicFallbacksOptions,\n  rejectsForcedToolUse,\n} from \"../../llm/anthropicOptions.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\ninterface DoneResult {\n  reasoning: string;\n  taskComplete: boolean;\n  messages: ModelMessage[];\n  output?: Record<string, unknown>;\n}\n\n/**\n * Deep-remove `undefined` values from the run history before it is re-submitted\n * to the forced \"done\" call.\n *\n * The AI SDK validates re-submitted messages against its `ModelMessage` schema,\n * whose JSON-value schema rejects `undefined` (only null/string/number/boolean/\n * object/array are allowed). A custom tool that returns an object with an\n * optional field left `undefined` (e.g. `{ matchedExpected: undefined }`) lands\n * that `undefined` inside a tool-result `output.value`, and reasoning models can\n * leave `undefined` in part fields too. Either makes the forced \"done\" call\n * throw `Invalid prompt: messages must be a ModelMessage[]` (STG-2335) — a red\n * error that fires after the run has already completed. Stripping `undefined`\n * keeps the history valid without dropping any real content.\n *\n * Only plain objects and arrays are traversed; class instances (URL, typed\n * arrays for binary image data, Date, …) are passed through untouched.\n */\nfunction stripUndefinedDeep<T>(value: T): T {\n  if (Array.isArray(value)) {\n    return value.map((v) => stripUndefinedDeep(v)) as unknown as T;\n  }\n  if (value !== null && typeof value === \"object\") {\n    const proto = Object.getPrototypeOf(value);\n    if (proto === Object.prototype || proto === null) {\n      const out: Record<string, unknown> = {};\n      for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n        if (v !== undefined) out[k] = stripUndefinedDeep(v);\n      }\n      return out as T;\n    }\n  }\n  return value;\n}\n\nexport function sanitizeMessagesForResubmission(\n  messages: ModelMessage[],\n): ModelMessage[] {\n  return stripUndefinedDeep(messages);\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n  return factory.object({\n    reasoning: factory\n      .string()\n      .describe(\"Brief summary of what actions were taken and the outcome\"),\n    taskComplete: factory\n      .boolean()\n      .describe(\"true if the task was fully completed, false otherwise\"),\n  });\n}\n\n/**\n * Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n  model: LanguageModel;\n  inputMessages: ModelMessage[];\n  instruction: string;\n  outputSchema?: StagehandZodObject;\n  logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n  const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n  logger({\n    category: \"agent\",\n    message: \"Agent calling tool: done\",\n    level: 1,\n  });\n  // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n  const factory = outputSchema\n    ? getZFactory(outputSchema as StagehandZodSchema)\n    : z;\n  const baseDoneSchema = buildBaseDoneSchema(factory);\n\n  // Merge base done schema with user-provided output schema if present\n  const doneToolSchema = outputSchema\n    ? baseDoneSchema.extend({\n        output: outputSchema.describe(\n          \"The specific data the user requested from this task\",\n        ),\n      })\n    : baseDoneSchema;\n\n  const outputInstructions = outputSchema\n    ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n        Object.fromEntries(\n          Object.entries(outputSchema.shape).map(\n            ([key, value]: [string, StagehandZodSchema]) => [\n              key,\n              value.description || \"no description\",\n            ],\n          ),\n        ),\n        null,\n        2,\n      )}`\n    : \"\";\n\n  const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n  const doneTool = tool({\n    description: outputSchema\n      ? \"Complete the task with your assessment and the requested output data.\"\n      : \"Complete the task with your final assessment.\",\n    inputSchema: doneToolSchema,\n    execute: async (params) => {\n      return { success: true, ...params };\n    },\n  });\n\n  const userPrompt: ModelMessage = {\n    role: \"user\",\n    content: outputSchema\n      ? \"Provide your final assessment and the requested output data.\"\n      : \"Provide your final assessment.\",\n  };\n\n  const modelId = typeof model === \"string\" ? model : model.modelId;\n  const fallbacks = anthropicFallbacksOptions(modelId);\n\n  // Models whose always-on thinking rejects forced tool use go straight to\n  // \"auto\" — the prompt already instructs calling \"done\", and the\n  // no-tool-call case below handles a plain-text answer.\n  const result = await generateText({\n    model,\n    system: systemPrompt,\n    messages: [...sanitizeMessagesForResubmission(inputMessages), userPrompt],\n    tools: { done: doneTool } as ToolSet,\n    toolChoice: rejectsForcedToolUse(modelId)\n      ? \"auto\"\n      : { type: \"tool\", toolName: \"done\" },\n    providerOptions: {\n      google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n      openai: { store: false },\n      ...(fallbacks ? { anthropic: fallbacks } : {}),\n    },\n  });\n\n  const doneToolCall = result.toolCalls?.find((tc) => tc.toolName === \"done\");\n  const outputMessages: ModelMessage[] = [\n    userPrompt,\n    ...(result.response?.messages || []),\n  ];\n\n  if (!doneToolCall) {\n    return {\n      reasoning: result.text || \"Task execution completed\",\n      taskComplete: false,\n      messages: outputMessages,\n    };\n  }\n\n  const input = doneToolCall.input as {\n    reasoning: string;\n    taskComplete: boolean;\n    output?: Record<string, unknown>;\n  };\n  logger({\n    category: \"agent\",\n    message: `Task completed`,\n    level: 1,\n  });\n\n  return {\n    reasoning: input.reasoning,\n    taskComplete: input.taskComplete,\n    messages: outputMessages,\n    output: input.output,\n  };\n}\n"]}
+{"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAwC,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAE1B,OAAO,EACL,yBAAyB,EACzB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAS1E;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,kBAAkB,CAAI,KAAQ;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAiB,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjD,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,KAAK,SAAS;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,GAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,QAAwB;IAExB,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,WAAW,CAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAI,CAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;IAClE,MAAM,aAAa,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC7E,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAErD,yEAAyE;IACzE,gEAAgE;IAChE,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,+BAA+B,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC;QACzE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,oBAAoB,CAAC,OAAO,CAAC;YACvC,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QACtC,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,GAAG,0BAA0B,CAAC,aAAa,CAAC;YAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport {\n  anthropicFallbacksOptions,\n  rejectsForcedToolUse,\n} from \"../../llm/anthropicOptions.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\nimport { openAIStoreProviderOptions } from \"../../llm/providerOptions.js\";\n\ninterface DoneResult {\n  reasoning: string;\n  taskComplete: boolean;\n  messages: ModelMessage[];\n  output?: Record<string, unknown>;\n}\n\n/**\n * Deep-remove `undefined` values from the run history before it is re-submitted\n * to the forced \"done\" call.\n *\n * The AI SDK validates re-submitted messages against its `ModelMessage` schema,\n * whose JSON-value schema rejects `undefined` (only null/string/number/boolean/\n * object/array are allowed). A custom tool that returns an object with an\n * optional field left `undefined` (e.g. `{ matchedExpected: undefined }`) lands\n * that `undefined` inside a tool-result `output.value`, and reasoning models can\n * leave `undefined` in part fields too. Either makes the forced \"done\" call\n * throw `Invalid prompt: messages must be a ModelMessage[]` (STG-2335) — a red\n * error that fires after the run has already completed. Stripping `undefined`\n * keeps the history valid without dropping any real content.\n *\n * Only plain objects and arrays are traversed; class instances (URL, typed\n * arrays for binary image data, Date, …) are passed through untouched.\n */\nfunction stripUndefinedDeep<T>(value: T): T {\n  if (Array.isArray(value)) {\n    return value.map((v) => stripUndefinedDeep(v)) as unknown as T;\n  }\n  if (value !== null && typeof value === \"object\") {\n    const proto = Object.getPrototypeOf(value);\n    if (proto === Object.prototype || proto === null) {\n      const out: Record<string, unknown> = {};\n      for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n        if (v !== undefined) out[k] = stripUndefinedDeep(v);\n      }\n      return out as T;\n    }\n  }\n  return value;\n}\n\nexport function sanitizeMessagesForResubmission(\n  messages: ModelMessage[],\n): ModelMessage[] {\n  return stripUndefinedDeep(messages);\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n  return factory.object({\n    reasoning: factory\n      .string()\n      .describe(\"Brief summary of what actions were taken and the outcome\"),\n    taskComplete: factory\n      .boolean()\n      .describe(\"true if the task was fully completed, false otherwise\"),\n  });\n}\n\n/**\n * Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n  model: LanguageModel;\n  inputMessages: ModelMessage[];\n  instruction: string;\n  outputSchema?: StagehandZodObject;\n  logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n  const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n  logger({\n    category: \"agent\",\n    message: \"Agent calling tool: done\",\n    level: 1,\n  });\n  // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n  const factory = outputSchema\n    ? getZFactory(outputSchema as StagehandZodSchema)\n    : z;\n  const baseDoneSchema = buildBaseDoneSchema(factory);\n\n  // Merge base done schema with user-provided output schema if present\n  const doneToolSchema = outputSchema\n    ? baseDoneSchema.extend({\n        output: outputSchema.describe(\n          \"The specific data the user requested from this task\",\n        ),\n      })\n    : baseDoneSchema;\n\n  const outputInstructions = outputSchema\n    ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n        Object.fromEntries(\n          Object.entries(outputSchema.shape).map(\n            ([key, value]: [string, StagehandZodSchema]) => [\n              key,\n              value.description || \"no description\",\n            ],\n          ),\n        ),\n        null,\n        2,\n      )}`\n    : \"\";\n\n  const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n  const doneTool = tool({\n    description: outputSchema\n      ? \"Complete the task with your assessment and the requested output data.\"\n      : \"Complete the task with your final assessment.\",\n    inputSchema: doneToolSchema,\n    execute: async (params) => {\n      return { success: true, ...params };\n    },\n  });\n\n  const userPrompt: ModelMessage = {\n    role: \"user\",\n    content: outputSchema\n      ? \"Provide your final assessment and the requested output data.\"\n      : \"Provide your final assessment.\",\n  };\n\n  const modelId = typeof model === \"string\" ? model : model.modelId;\n  const modelProvider = typeof model === \"string\" ? undefined : model.provider;\n  const fallbacks = anthropicFallbacksOptions(modelId);\n\n  // Models whose always-on thinking rejects forced tool use go straight to\n  // \"auto\" — the prompt already instructs calling \"done\", and the\n  // no-tool-call case below handles a plain-text answer.\n  const result = await generateText({\n    model,\n    system: systemPrompt,\n    messages: [...sanitizeMessagesForResubmission(inputMessages), userPrompt],\n    tools: { done: doneTool } as ToolSet,\n    toolChoice: rejectsForcedToolUse(modelId)\n      ? \"auto\"\n      : { type: \"tool\", toolName: \"done\" },\n    providerOptions: {\n      google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n      ...openAIStoreProviderOptions(modelProvider),\n      ...(fallbacks ? { anthropic: fallbacks } : {}),\n    },\n  });\n\n  const doneToolCall = result.toolCalls?.find((tc) => tc.toolName === \"done\");\n  const outputMessages: ModelMessage[] = [\n    userPrompt,\n    ...(result.response?.messages || []),\n  ];\n\n  if (!doneToolCall) {\n    return {\n      reasoning: result.text || \"Task execution completed\",\n      taskComplete: false,\n      messages: outputMessages,\n    };\n  }\n\n  const input = doneToolCall.input as {\n    reasoning: string;\n    taskComplete: boolean;\n    output?: Record<string, unknown>;\n  };\n  logger({\n    category: \"agent\",\n    message: `Task completed`,\n    level: 1,\n  });\n\n  return {\n    reasoning: input.reasoning,\n    taskComplete: input.taskComplete,\n    messages: outputMessages,\n    output: input.output,\n  };\n}\n"]}

package/dist/esm/lib/v3/api.d.ts

@@ -91,7 +91,7 @@ export declare class StagehandAPIClient {
     private latestAgentCacheEntry;
     private warnedStagehandBaseUrl;
     constructor({ apiKey, projectId, logger, serverCache, }: StagehandAPIConstructorParams);
-    init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, browserbaseSessionCreateParams, browserbaseSessionID, }: ClientSessionStartParams): Promise<Api.SessionStartResult>;
+    init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, useTouch, browserbaseSessionCreateParams, browserbaseSessionID, }: ClientSessionStartParams): Promise<Api.SessionStartResult>;
     act({ input, options, frameId, }: ClientActParameters): Promise<ActResult>;
     extract<T extends StagehandZodSchema>({ instruction, schema: zodSchema, options, frameId, }: ClientExtractParameters): Promise<ExtractResult<T>>;
     observe({ instruction, options, frameId, }: ClientObserveParameters): Promise<ObserveResult>;

package/dist/esm/lib/v3/api.js

@@ -1,5 +1,6 @@
 import makeFetchCookie from "fetch-cookie";
 import { getInheritableModelOptions, hasModelProviderAuth, loadApiKeyFromEnv, } from "../utils.js";
+import { isAutoModel } from "../modelUtils.js";
 import { STAGEHAND_VERSION } from "../version.js";
 import { StagehandAPIError, StagehandAPIUnauthorizedError, StagehandHttpError, StagehandResponseBodyError, StagehandResponseParseError, StagehandServerError, ExperimentalNotConfiguredError, } from "./types/public/index.js";
 import { toJsonSchema } from "./zodCompat.js";
@@ -51,7 +52,7 @@ export class StagehandAPIClient {
         // Create a single cookie jar instance that will persist across all requests
         this.fetchWithCookies = makeFetchCookie(fetch);
     }
-    async init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, browserbaseSessionCreateParams, browserbaseSessionID,
+    async init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, useTouch, browserbaseSessionCreateParams, browserbaseSessionID,
     // browser,  TODO for local browsers
      }) {
         this.modelApiKey = modelApiKey;
@@ -76,6 +77,7 @@ export class StagehandAPIClient {
             verbose,
             systemPrompt,
             selfHeal,
+            useTouch,
             browserbaseSessionCreateParams,
             browserbaseSessionID,
             // browser, TODO: only send when connected to local fastify
@@ -412,6 +414,14 @@ export class StagehandAPIClient {
      * model provider differs from the one used to init the session.
      */
     prepareModelConfig(model) {
+        // "auto" delegates model selection to the API, so never inherit the
+        // default model's provider config or API key — the server picks the
+        // provider. Explicit per-call options are still passed through.
+        if (isAutoModel(model)) {
+            return typeof model === "string"
+                ? { modelName: model }
+                : { ...model };
+        }
         if (typeof model === "string") {
             // Extract provider from model string (e.g., "openai/gpt-5-nano" -> "openai")
             const provider = this.getModelProvider(model);