@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/tools/ask-question/ask-question-tool.ts

@@ -7,8 +7,10 @@ import type {
 import {
   QuestionPrompter,
   type QuestionPromptOutcome,
+  type QuestionPromptParamsEntry,
 } from "../../permissions/question-prompter.js";
 import { RiskLevel } from "../../permissions/types.js";
+import { DESKTOP_HELP_GUIDANCE } from "../../util/browser-human-verification.js";
 import {
   invalidToolInputResult,
   toToolInputSchema,
@@ -74,17 +76,43 @@ const MAX_QUESTIONS_PER_BATCH = 5;
 // Callers pass a (possibly single-element) batch of questions. `execute()`
 // forwards them straight to the prompter. Loose so injected fields (e.g.
 // `activity`) never fail validation.
-export const askQuestionInputSchema = z.looseObject({
-  questions: z
-    .array(SingleQuestionSchema)
-    .min(1)
-    .max(MAX_QUESTIONS_PER_BATCH, {
-      message: `At most ${MAX_QUESTIONS_PER_BATCH} questions per batch; split into multiple turns if you need more.`,
-    })
-    .describe(
-      `1–${MAX_QUESTIONS_PER_BATCH} clarifying questions to ask in a single turn. Use a batch when several independent ambiguities block progress; ask one at a time when they're sequentially dependent. Past ${MAX_QUESTIONS_PER_BATCH} questions you should be implementing, not asking.`,
-    ),
-});
+export const askQuestionInputSchema = z
+  .looseObject({
+    desktopHelp: z
+      .object({
+        message: z
+          .string()
+          .min(1)
+          .describe(
+            "Use one short sentence to explain the needed human action, in the user's language.",
+          ),
+        doneLabel: z
+          .string()
+          .min(1)
+          .describe("The label for Done in the user's language."),
+        skipLabel: z
+          .string()
+          .min(1)
+          .describe("The label for Skip in the user's language."),
+      })
+      .describe(
+        "Immediately request human interaction for any CAPTCHA or bot-detection challenge, including sliders and press-and-hold checks. Also use for native dialogs requiring user interaction. For logins, use saved credentials or securely prompt for missing credentials first. Explain what the user should do. Shows a live preview with Step In, Done and Skip. Pass this instead of questions.",
+      )
+      .optional(),
+    questions: z
+      .array(SingleQuestionSchema)
+      .min(1)
+      .max(MAX_QUESTIONS_PER_BATCH, {
+        message: `At most ${MAX_QUESTIONS_PER_BATCH} questions per batch; split into multiple turns if you need more.`,
+      })
+      .describe(
+        `1–${MAX_QUESTIONS_PER_BATCH} clarifying questions to ask in a single turn. Use a batch when several independent ambiguities block progress; ask one at a time when they're sequentially dependent. Past ${MAX_QUESTIONS_PER_BATCH} questions you should be implementing, not asking.`,
+      )
+      .optional(),
+  })
+  .refine((input) => Boolean(input.questions) !== Boolean(input.desktopHelp), {
+    message: "Provide either questions or desktopHelp, not both.",
+  });
 
 export type SingleQuestion = z.infer<typeof SingleQuestionSchema>;
 export type AskQuestionInput = z.infer<typeof askQuestionInputSchema>;
@@ -92,6 +120,12 @@ export type AskQuestionInput = z.infer<typeof askQuestionInputSchema>;
 // ── Tool description ────────────────────────────────────────────────
 
 const DESCRIPTION = [
+  DESKTOP_HELP_GUIDANCE,
+  "For logins, use saved credentials first. Securely collect missing credentials",
+  "with assistant credentials prompt, then fill the login form yourself.",
+  "",
+  "Every call passes exactly one of `questions` or `desktopHelp`.",
+  "",
   "Use this tool whenever a request is ambiguous and can be resolved",
   "by 2–4 plausible interpretations or discrete choices. Prefer it over",
   "plain-text clarification — structured options are faster to answer and",
@@ -223,6 +257,9 @@ export const askQuestionTool = {
   category: "interaction",
   executionTarget: "sandbox",
   defaultRiskLevel: RiskLevel.Low,
+  // Anthropic rejects `oneOf` / `anyOf` / `allOf` at the root of a tool
+  // schema, so the either/or rule between `questions` and `desktopHelp` lives
+  // in the description and the Zod refine, not in the wire schema.
   input_schema: toToolInputSchema(askQuestionInputSchema),
 
   async execute(
@@ -234,7 +271,19 @@ export const askQuestionTool = {
       return invalidToolInputResult("ask_question", parsed.error);
     }
 
-    const questions: SingleQuestion[] = parsed.data.questions;
+    const { desktopHelp } = parsed.data;
+    const questions: QuestionPromptParamsEntry[] = desktopHelp
+      ? [
+          {
+            question: desktopHelp.message,
+            options: [
+              { id: "done", label: desktopHelp.doneLabel },
+              { id: "skip", label: desktopHelp.skipLabel },
+            ],
+            presentation: "virtual_desktop",
+          },
+        ]
+      : parsed.data.questions!;
 
     // No interactive user is present to answer (scheduled/headless/background
     // turn). Don't park the turn on a prompt no one can resolve — proceed with
@@ -243,12 +292,21 @@ export const askQuestionTool = {
     // model asks anyway, so it doesn't wait out the full response timeout.
     if (context.isInteractive === false) {
       return {
-        content:
-          "No interactive user is present to answer; proceeding with reasonable defaults.",
+        content: desktopHelp
+          ? "No interactive user is present to help in the virtual desktop. The obstacle remains unresolved."
+          : "No interactive user is present to answer; proceeding with reasonable defaults.",
         isError: false,
       };
     }
 
+    if (desktopHelp && context.supportsDynamicUi === false) {
+      return {
+        content:
+          "This channel cannot show the virtual desktop preview or Step In. Ask the user to continue in the Vellum app to complete this step. The obstacle remains unresolved.",
+        isError: true,
+      };
+    }
+
     // Channel turns (no dynamic UI) park only when the question can reach the
     // user as a guardian-request card with tappable options: a single-question
     // batch, asked by the guardian, on a channel whose notification adapter
@@ -277,13 +335,35 @@ export const askQuestionTool = {
       };
     }
 
+    let finishDesktopHelp: ((resume: boolean) => Promise<void>) | undefined;
+    if (desktopHelp) {
+      const { prepareDesktopHelp } =
+        await import("../../desktop/desktop-help.js");
+      const prepared = await prepareDesktopHelp(context);
+      if (typeof prepared !== "function") {
+        return prepared;
+      }
+      finishDesktopHelp = prepared;
+    }
+
     const prompter = new QuestionPrompter();
-    const result = await prompter.prompt({
-      conversationId: context.conversationId,
-      questions,
-      toolUseId: context.toolUseId,
-      signal: context.signal,
-    });
+    let result: QuestionPromptOutcome;
+    let resumeDesktop = false;
+    try {
+      result = await prompter.prompt({
+        conversationId: context.conversationId,
+        questions,
+        toolUseId: context.toolUseId,
+        signal: context.signal,
+      });
+      const answer = result.entries[0];
+      resumeDesktop =
+        result.overall === "completed" &&
+        (answer?.decision === "free_text" ||
+          (answer?.decision === "option" && answer.optionId === "done"));
+    } finally {
+      await finishDesktopHelp?.(resumeDesktop);
+    }
 
     // Format the aggregated transcript. Each line is keyed by the original
     // question text (not the daemon-assigned id) — the LLM never sees those
@@ -308,8 +388,26 @@ export const askQuestionTool = {
     const answeredQuestion = toAnsweredQuestion(result);
 
     switch (result.overall) {
-      case "completed":
-        return { content: lines.join("\n"), isError: false, answeredQuestion };
+      case "completed": {
+        let content = lines.join("\n");
+        if (desktopHelp) {
+          const entry = result.entries[0];
+          if (entry?.decision === "free_text") {
+            content +=
+              "\nTake a fresh browser snapshot before continuing; do not assume the obstacle was resolved.";
+          } else if (
+            entry?.decision === "option" &&
+            entry.optionId === "done"
+          ) {
+            content =
+              "The user finished interacting with the virtual desktop. Take a fresh browser snapshot and verify the result before continuing.";
+          } else {
+            content =
+              "The user skipped helping in the virtual desktop. The obstacle may still be present. Use another approach or explain what remains blocked.";
+          }
+        }
+        return { content, isError: false, answeredQuestion };
+      }
       case "closed": {
         const summary =
           "User closed the question card without answering. All questions skipped.";

package/src/tools/browser/browser-execution.ts

@@ -4,6 +4,10 @@ import { getConfig } from "../../config/loader.js";
 import { HostBrowserProxy } from "../../daemon/host-browser-proxy.js";
 import type { ImageContent } from "../../providers/types.js";
 import { wrapUntrustedContent } from "../../security/untrusted-content.js";
+import {
+  DESKTOP_HELP_GUIDANCE,
+  HUMAN_VERIFICATION_GUIDANCE,
+} from "../../util/browser-human-verification.js";
 import { getLogger } from "../../util/logger.js";
 import { truncate } from "../../util/truncate.js";
 import { safeStringSlice } from "../../util/unicode.js";
@@ -56,7 +60,6 @@ import {
   captureScreenshotJpeg,
   dispatchClickAt,
   dispatchHoverAt,
-  dispatchInsertText,
   dispatchKeyPress,
   dispatchWheelScroll,
   evaluateExpression,
@@ -64,6 +67,7 @@ import {
   getCenterPoint,
   getCurrentUrl,
   getPageTitle,
+  insertTextIntoElement,
   navigateAndWait,
   querySelectorBackendNodeId,
   scrollIntoViewIfNeeded,
@@ -1319,7 +1323,9 @@ export async function executeBrowserNavigate(
               "⚠️ CAPTCHA/Cloudflare verification detected on this page.",
             );
             lines.push(
-              "This challenge requires human verification. Surface this clearly: the page cannot be accessed until the verification is solved manually.",
+              context.cdpClient
+                ? DESKTOP_HELP_GUIDANCE
+                : HUMAN_VERIFICATION_GUIDANCE,
             );
             if (cdp.kind === "local") {
               lines.push("");
@@ -1755,60 +1761,6 @@ export async function executeBrowserClick(
   }
 }
 
-// ── Shared input helpers ─────────────────────────────────────────────
-
-/**
- * Focus an element, clear its existing value (handling both
- * `<input>`/`<textarea>` and `contentEditable` targets), re-focus
- * (sites sometimes blur on a programmatic value reset), and insert
- * the requested text via `Input.insertText`.
- *
- * Used by both `executeBrowserType` and `executeBrowserFillCredential`
- * so credential fills cannot append to autofilled / pre-populated
- * fields — appending would leak the existing value into the broker
- * payload and corrupt the resulting password.
- */
-async function clearAndInsertText(
-  cdp: CdpClient,
-  backendNodeId: number,
-  value: string,
-  signal?: AbortSignal,
-): Promise<void> {
-  await focusElement(cdp, backendNodeId, signal);
-
-  // Resolve the node to a Runtime.RemoteObject so we can invoke a
-  // function on the element itself via Runtime.callFunctionOn. This
-  // is more reliable than a keyboard select-all + delete sequence
-  // across input, textarea, and contenteditable targets.
-  const { object } = await cdp.send<{ object: { objectId: string } }>(
-    "DOM.resolveNode",
-    { backendNodeId },
-    signal,
-  );
-  await cdp.send(
-    "Runtime.callFunctionOn",
-    {
-      objectId: object.objectId,
-      functionDeclaration: `function() {
-        if (typeof this.value === "string") {
-          this.value = "";
-        } else if (this.isContentEditable) {
-          this.textContent = "";
-        }
-        this.dispatchEvent(new Event("input", { bubbles: true }));
-      }`,
-      arguments: [],
-    },
-    signal,
-  );
-
-  // Re-focus after clearing — some sites move focus when the value
-  // property is reassigned programmatically.
-  await focusElement(cdp, backendNodeId, signal);
-
-  await dispatchInsertText(cdp, value, signal);
-}
-
 // ── browser_type ─────────────────────────────────────────────────────
 
 export async function executeBrowserType(
@@ -1854,10 +1806,21 @@ export async function executeBrowserType(
     }
 
     if (clearFirst) {
-      await clearAndInsertText(cdp, backendNodeId, text, context.signal);
+      await insertTextIntoElement(
+        cdp,
+        backendNodeId,
+        text,
+        { clearFirst: true },
+        context.signal,
+      );
     } else {
-      await focusElement(cdp, backendNodeId, context.signal);
-      await dispatchInsertText(cdp, text, context.signal);
+      await insertTextIntoElement(
+        cdp,
+        backendNodeId,
+        text,
+        { clearFirst: false },
+        context.signal,
+      );
     }
 
     if (pressEnter) {
@@ -2496,7 +2459,13 @@ export async function executeBrowserFillCredential(
         // would append the credential to the existing value,
         // producing a corrupted password and leaking partial state
         // back into the page.
-        await clearAndInsertText(cdp, backendNodeId, value, context.signal);
+        await insertTextIntoElement(
+          cdp,
+          backendNodeId,
+          value,
+          { clearFirst: true, verify: false },
+          context.signal,
+        );
       },
     });