@vellumai/assistant 0.8.0 → 0.8.2

package/src/tools/ask-question/ask-question-tool.ts

@@ -0,0 +1,304 @@
+import { z } from "zod";
+
+import { QuestionPrompter } from "../../permissions/question-prompter.js";
+import { RiskLevel } from "../../permissions/types.js";
+import type { ToolDefinition } from "../../providers/types.js";
+import { broadcastMessage } from "../../runtime/assistant-event-hub.js";
+import type { Tool, ToolContext, ToolExecutionResult } from "../types.js";
+
+// ── Input schema ────────────────────────────────────────────────────
+// Runtime validation lives in Zod; the wire-level definition surfaced
+// to the LLM is the hand-written JSON Schema in getDefinition() below.
+// (The codebase does not currently use zod-to-json-schema for tool defs,
+// so the two are kept in sync manually.)
+
+const OptionSchema = z.object({
+  id: z.string().min(1),
+  label: z.string().min(1),
+  description: z.string().optional(),
+});
+
+// One question in a (possibly single-element) batch. Intentionally has no
+// `id` field — per-question ids are daemon-assigned (`q1`, `q2`, ...) inside
+// the prompter, never supplied by the LLM. This keeps the LLM-facing schema
+// smaller and removes a validation surface (no duplicate-id check, no
+// length cap on ids).
+const SingleQuestionSchema = z.object({
+  question: z.string().min(1),
+  description: z.string().optional(),
+  // 2–4 LLM-supplied options. The client renders a fixed 5th "Type
+  // something else" slot for free-text, so the model must keep the
+  // structured set to 4 or fewer.
+  options: z.array(OptionSchema).min(2).max(4),
+  freeTextPlaceholder: z.string().optional(),
+});
+
+// Cap at 5 questions per batch. Past that it starts to feel like a form,
+// not a clarification — the model should be implementing, not asking. Any
+// input with ≥6 entries is rejected with a clear Zod error.
+const MAX_QUESTIONS_PER_BATCH = 5;
+
+// Both the new batched shape (`questions[]`) and the legacy flat shape are
+// accepted. `execute()` normalizes legacy callers into a one-element
+// `questions` array before forwarding to the prompter.
+const InputSchema = z
+  .object({
+    questions: z
+      .array(SingleQuestionSchema)
+      .min(1)
+      .max(MAX_QUESTIONS_PER_BATCH, {
+        message: `At most ${MAX_QUESTIONS_PER_BATCH} questions per batch; split into multiple turns if you need more.`,
+      })
+      .optional(),
+    // Legacy flat fields. Optional so batched callers can omit them; when
+    // present and `questions` is absent, they are normalized into a
+    // one-element batch in `execute()`.
+    question: z.string().min(1).optional(),
+    description: z.string().optional(),
+    options: z.array(OptionSchema).min(2).max(4).optional(),
+    freeTextPlaceholder: z.string().optional(),
+  })
+  .refine(
+    (v) =>
+      v.questions !== undefined ||
+      (v.question !== undefined && v.options !== undefined),
+    {
+      message:
+        "Provide `questions` (preferred) or the legacy flat fields (`question` + `options`).",
+    },
+  );
+
+export type SingleQuestion = z.infer<typeof SingleQuestionSchema>;
+export type AskQuestionInput = z.infer<typeof InputSchema>;
+
+// ── Tool description ────────────────────────────────────────────────
+
+const DESCRIPTION = [
+  "Use this tool whenever a request is ambiguous and can be resolved",
+  "by 2–4 plausible interpretations or discrete choices. Prefer it over",
+  "plain-text clarification — structured options are faster to answer and",
+  "remove guessing.",
+  "",
+  "When in doubt between (a) asking inline and (b) calling ask_question with",
+  "structured options: call ask_question. The structured choices are better UX.",
+  "",
+  'Example: given a request like "schedule lunch with Alice next week" where there',
+  "are two plausible Alice contacts, ask which Alice with options like",
+  '`{id: "alice_work", label: "Alice (work)"}` and',
+  '`{id: "alice_personal", label: "Alice (personal)"}`.',
+  "",
+  "Batch related clarifications into one call by passing multiple entries in",
+  "`questions` (up to 5). Each question gets its own page with a Skip button.",
+  "",
+  "When NOT to use this tool:",
+  "- The answer is obvious from context or recent conversation.",
+  "- The question is genuinely open-ended (more than ~4 plausible answers) —",
+  "  fall back to plain text.",
+  "- You're about to take a low-stakes reversible action and can adjust based",
+  "  on feedback.",
+  "",
+  "If a question is skipped, proceed with reasonable defaults for that",
+  "question; if every question in the batch is skipped, stop interrupting",
+  "and use defaults across the board.",
+  "",
+  "Provide 2–4 options. A free-text fallback is always added by the UI — do not",
+  "include a 'something else' option yourself.",
+  "",
+  "Each option needs a stable `id` (the value the response carries back) and a",
+  "short human-readable `label`. Optional `description` adds one line of",
+  "context shown beneath the label.",
+].join("\n");
+
+// ── Tool ────────────────────────────────────────────────────────────
+
+export class AskQuestionTool implements Tool {
+  name = "ask_question";
+  description = DESCRIPTION;
+  category = "interaction";
+  defaultRiskLevel = RiskLevel.Low;
+
+  // Override hook for tests: lets a test replace the prompter factory
+  // without monkey-patching the module. Default factory wires the real
+  // broadcastMessage so the question reaches every connected client.
+  private prompterFactory: () => Pick<QuestionPrompter, "prompt">;
+
+  constructor(
+    prompterFactory: () => Pick<QuestionPrompter, "prompt"> = () =>
+      new QuestionPrompter({ broadcastMessage }),
+  ) {
+    this.prompterFactory = prompterFactory;
+  }
+
+  getDefinition(): ToolDefinition {
+    // Shared option-schema fragment used by both the batched `questions[]`
+    // shape and the legacy flat `options` field.
+    const optionItemsSchema = {
+      type: "object",
+      properties: {
+        id: {
+          type: "string",
+          description:
+            "Stable identifier for this option (returned verbatim in the response).",
+        },
+        label: {
+          type: "string",
+          description: "Short human-readable label.",
+        },
+        description: {
+          type: "string",
+          description: "Optional one-line context shown beneath the label.",
+        },
+      },
+      required: ["id", "label"],
+    } as const;
+
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: "object",
+        properties: {
+          // ── Recommended shape ─────────────────────────────────────
+          questions: {
+            type: "array",
+            minItems: 1,
+            maxItems: MAX_QUESTIONS_PER_BATCH,
+            description: `Recommended shape. 1–${MAX_QUESTIONS_PER_BATCH} clarifying questions to ask in a single turn. Use a batch when several independent ambiguities block progress; ask one at a time when they're sequentially dependent. Past ${MAX_QUESTIONS_PER_BATCH} questions you should be implementing, not asking.`,
+            items: {
+              type: "object",
+              properties: {
+                question: {
+                  type: "string",
+                  description: "The clarifying question to display.",
+                },
+                description: {
+                  type: "string",
+                  description:
+                    "Optional one-line context shown beneath the question.",
+                },
+                options: {
+                  type: "array",
+                  minItems: 2,
+                  maxItems: 4,
+                  description:
+                    "2–4 structured options. The UI always appends a free-text fallback slot, so do not include a 'something else' option here.",
+                  items: optionItemsSchema,
+                },
+                freeTextPlaceholder: {
+                  type: "string",
+                  description:
+                    "Optional placeholder text shown inside the free-text fallback input.",
+                },
+              },
+              required: ["question", "options"],
+            },
+          },
+          // ── Legacy single-question fields ─────────────────────────
+          // Kept optional so existing prompt caches and any single-question
+          // callers continue to work. New callers should use `questions`.
+          question: {
+            type: "string",
+            description:
+              "Legacy: the single clarifying question. Prefer `questions[]` for new code.",
+          },
+          description: {
+            type: "string",
+            description:
+              "Legacy: optional one-line context shown beneath the question. Prefer `questions[].description`.",
+          },
+          options: {
+            type: "array",
+            minItems: 2,
+            maxItems: 4,
+            description:
+              "Legacy: 2–4 structured options. Prefer `questions[].options`. The UI always appends a free-text fallback slot, so do not include a 'something else' option here.",
+            items: optionItemsSchema,
+          },
+          freeTextPlaceholder: {
+            type: "string",
+            description:
+              "Legacy: optional placeholder text for the free-text fallback input. Prefer `questions[].freeTextPlaceholder`.",
+          },
+        },
+        // No top-level `required` — caller must supply either `questions`
+        // or the legacy flat trio (`question` + `options`). Enforced in Zod.
+      },
+    };
+  }
+
+  async execute(
+    input: Record<string, unknown>,
+    context: ToolContext,
+  ): Promise<ToolExecutionResult> {
+    const parsed = InputSchema.safeParse(input);
+    if (!parsed.success) {
+      return {
+        content: `Invalid input: ${parsed.error.message}`,
+        isError: true,
+      };
+    }
+
+    // Normalize legacy flat input into a one-element `questions` batch so
+    // downstream code only has to deal with the batched shape. The refine
+    // above guarantees `question` and `options` are present whenever
+    // `questions` is absent.
+    const questions: SingleQuestion[] = parsed.data.questions ?? [
+      {
+        question: parsed.data.question!,
+        description: parsed.data.description,
+        options: parsed.data.options!,
+        freeTextPlaceholder: parsed.data.freeTextPlaceholder,
+      },
+    ];
+
+    const prompter = this.prompterFactory();
+    const result = await prompter.prompt({
+      conversationId: context.conversationId,
+      questions,
+      toolUseId: context.toolUseId,
+      signal: context.signal,
+    });
+
+    // Format the aggregated transcript. Each line is keyed by the original
+    // question text (not the daemon-assigned id) — the LLM never sees those
+    // ids, and human-readable labels read better in the result content.
+    const lines = result.entries.map((entry, i) => {
+      const q = questions[i]!;
+      const prefix = `Question "${q.question}" →`;
+      if (entry.decision === "option") {
+        const chosen = q.options.find((o) => o.id === entry.optionId);
+        const label = chosen?.label ?? "(unknown)";
+        return `${prefix} Option: ${entry.optionId} (${label})`;
+      }
+      if (entry.decision === "free_text") {
+        return `${prefix} Free text: ${entry.text ?? ""}`;
+      }
+      return `${prefix} Skipped`;
+    });
+
+    switch (result.overall) {
+      case "completed":
+        return { content: lines.join("\n"), isError: false };
+      case "closed": {
+        const summary =
+          "User closed the question card without answering. All questions skipped.";
+        return {
+          content: [summary, ...lines].join("\n"),
+          isError: false,
+        };
+      }
+      case "timed_out":
+        return {
+          content: "User did not respond within timeout",
+          isError: true,
+        };
+      case "aborted":
+        return {
+          content: "Question aborted",
+          isError: true,
+        };
+    }
+  }
+}
+
+export const askQuestionTool = new AskQuestionTool();

package/src/tools/browser/__tests__/browser-execution-acquire.test.ts

@@ -0,0 +1,206 @@
+/**
+ * Tests for the target_client_id sticky-mode override in
+ * acquireCdpClientWithMode (browser-execution.ts).
+ *
+ * Fix 1: when target_client_id is provided, the sticky backend kind
+ * remembered from prior turns in the conversation must NOT be applied.
+ * The factory must receive mode:"extension" so the request reaches the
+ * host-browser proxy regardless of any prior local/cdp-inspect preference.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { ToolContext } from "../../types.js";
+import type { BrowserMode, CdpClientKind } from "../cdp-client/types.js";
+
+// ---------------------------------------------------------------------------
+// Captured call state
+// ---------------------------------------------------------------------------
+
+interface CdpClientCallOpts {
+  mode?: BrowserMode;
+  targetClientId?: string;
+}
+
+const getCdpClientCalls: CdpClientCallOpts[] = [];
+
+function makeFakeScopedClient(kind: CdpClientKind, conversationId: string) {
+  return {
+    kind,
+    conversationId,
+    send: mock(async () => ({})),
+    dispose: mock(() => {}),
+  };
+}
+
+const getCdpClientMock = mock(
+  (ctx: ToolContext, opts?: CdpClientCallOpts) => {
+    getCdpClientCalls.push({
+      mode: opts?.mode,
+      targetClientId: opts?.targetClientId,
+    });
+    return makeFakeScopedClient("extension", ctx.conversationId);
+  },
+);
+
+// ---------------------------------------------------------------------------
+// Mutable sticky-kind control
+// ---------------------------------------------------------------------------
+
+let stickyKind: CdpClientKind | null = null;
+
+const setPreferredBackendKindMock = mock(
+  (_conversationId: string, _kind: CdpClientKind) => {},
+);
+const clearPreferredBackendKindMock = mock((_conversationId: string) => {});
+
+// ---------------------------------------------------------------------------
+// Module mocks (must be declared before dynamic import)
+// ---------------------------------------------------------------------------
+
+mock.module("../cdp-client/factory.js", () => ({
+  getCdpClient: getCdpClientMock,
+  buildCandidateList: mock(() => []),
+  isDesktopAutoCooldownActive: () => false,
+}));
+
+mock.module("../browser-manager.js", () => ({
+  browserManager: {
+    getPreferredBackendKind: (_conversationId: string) => stickyKind,
+    setPreferredBackendKind: setPreferredBackendKindMock,
+    clearPreferredBackendKind: clearPreferredBackendKindMock,
+    storeSnapshotBackendNodeMap: () => {},
+    clearSnapshotBackendNodeMap: () => {},
+    resolveSnapshotBackendNodeId: () => undefined,
+    isInteractive: () => false,
+    supportsRouteInterception: false,
+  },
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({
+    hostBrowser: {
+      cdpInspect: {
+        enabled: false,
+        host: "localhost",
+        port: 9222,
+        probeTimeoutMs: 500,
+        desktopAuto: { enabled: false, cooldownMs: 30_000 },
+      },
+    },
+  }),
+}));
+
+mock.module("../../../daemon/host-browser-proxy.js", () => ({
+  HostBrowserProxy: {
+    get instance() {
+      return {
+        isAvailable: () => false,
+        hasExtensionClient: () => false,
+        request: () => Promise.reject(new Error("no extension")),
+      };
+    },
+  },
+}));
+
+mock.module("../runtime-check.js", () => ({
+  checkBrowserRuntime: async () => ({
+    playwrightAvailable: true,
+    chromiumInstalled: true,
+    chromiumPath: "/tmp/chromium",
+    error: null,
+  }),
+}));
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () => ({
+    debug: () => {},
+    warn: () => {},
+    info: () => {},
+    error: () => {},
+  }),
+}));
+
+// Import under test after all mock.module calls.
+const { executeBrowserAttach } = await import("../browser-execution.js");
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeContext(conversationId: string): ToolContext {
+  return {
+    conversationId,
+    workingDir: "/tmp",
+    trustClass: "guardian",
+    signal: new AbortController().signal,
+  } as unknown as ToolContext;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("acquireCdpClientWithMode: target_client_id overrides sticky backend mode", () => {
+  beforeEach(() => {
+    getCdpClientCalls.length = 0;
+    getCdpClientMock.mockClear();
+    setPreferredBackendKindMock.mockClear();
+    clearPreferredBackendKindMock.mockClear();
+    stickyKind = null;
+  });
+
+  test("sticky local + target_client_id → getCdpClient receives mode:extension, not local", async () => {
+    // Simulate a prior turn that pinned the conversation to the local backend.
+    stickyKind = "local";
+
+    await executeBrowserAttach(
+      { target_client_id: "host-client-abc" },
+      makeContext("sticky-local-override"),
+    );
+
+    expect(getCdpClientMock).toHaveBeenCalledTimes(1);
+    // Fix 1: sticky "local" must be bypassed when target_client_id is present.
+    expect(getCdpClientCalls[0].mode).toBe("extension");
+    expect(getCdpClientCalls[0].targetClientId).toBe("host-client-abc");
+  });
+
+  test("sticky cdp-inspect + target_client_id → getCdpClient receives mode:extension, not cdp-inspect", async () => {
+    stickyKind = "cdp-inspect";
+
+    await executeBrowserAttach(
+      { target_client_id: "host-client-xyz" },
+      makeContext("sticky-inspect-override"),
+    );
+
+    expect(getCdpClientMock).toHaveBeenCalledTimes(1);
+    expect(getCdpClientCalls[0].mode).toBe("extension");
+    expect(getCdpClientCalls[0].targetClientId).toBe("host-client-xyz");
+  });
+
+  test("sticky local + no target_client_id → getCdpClient receives mode:local (sticky honored)", async () => {
+    // Without target_client_id, the sticky preference must still apply.
+    stickyKind = "local";
+
+    await executeBrowserAttach(
+      {}, // no target_client_id
+      makeContext("sticky-honored"),
+    );
+
+    expect(getCdpClientMock).toHaveBeenCalledTimes(1);
+    expect(getCdpClientCalls[0].mode).toBe("local");
+    expect(getCdpClientCalls[0].targetClientId).toBeUndefined();
+  });
+
+  test("no sticky + target_client_id → getCdpClient receives mode:extension", async () => {
+    stickyKind = null; // No prior sticky preference
+
+    await executeBrowserAttach(
+      { target_client_id: "host-client-fresh" },
+      makeContext("no-sticky-targeted"),
+    );
+
+    expect(getCdpClientMock).toHaveBeenCalledTimes(1);
+    expect(getCdpClientCalls[0].mode).toBe("extension");
+    expect(getCdpClientCalls[0].targetClientId).toBe("host-client-fresh");
+  });
+});

package/src/tools/browser/browser-execution.ts

@@ -90,7 +90,7 @@ const MODE_TRADEOFFS: Record<StatusCheckMode, string[]> = {
   [BROWSER_STATUS_MODE.EXTENSION]: [
     "This is the preferred approach for all things browser-use.",
     "On macOS, the host browser proxy is provisioned automatically via the desktop client's SSE bridge — no extension install required.",
-    "When the Chrome extension is also installed, it takes priority for direct WebSocket routing to the user's Chrome session.",
+    "When the Chrome extension is also installed, it takes priority for direct WebSocket routing to the active Chrome session.",
     "More secure than relying on Chrome's native remote debugging functionality.",
   ],
   [BROWSER_STATUS_MODE.CDP_INSPECT]: [
@@ -101,8 +101,8 @@ const MODE_TRADEOFFS: Record<StatusCheckMode, string[]> = {
   ],
   [BROWSER_STATUS_MODE.LOCAL]: [
     "The least-preferred approach for all things browser-use.",
-    "Considered a last-resort fallback if the user has not installed the Chrome Extension or enabled remote debugging in Chrome, and has indicated that they do not want to.",
-    "Does not use the user's existing browser profile, so sessions/cookies may differ.",
+    "Considered a last-resort fallback when the Chrome Extension is not installed, remote debugging in Chrome is not enabled, and neither will be enabled.",
+    "Does not use the existing browser profile, so sessions/cookies may differ.",
     "Requires that Playwright and Chromium are installed on the host machine,",
   ],
 };
@@ -378,16 +378,25 @@ function acquireCdpClientWithMode(
   }
   const browserMode = modeResult.mode;
 
+  const targetClientId =
+    typeof input.target_client_id === "string" && input.target_client_id !== ""
+      ? input.target_client_id
+      : undefined;
+
   const rememberedKind = browserManager.getPreferredBackendKind(
     context.conversationId,
   );
+  // target_client_id requires the extension proxy path — bypass any sticky
+  // backend remembered from prior turns so the explicit target always wins.
   const effectiveMode: BrowserMode =
-    browserMode === "auto" && rememberedKind !== null
-      ? rememberedKind
-      : browserMode;
+    targetClientId != null
+      ? "extension"
+      : browserMode === "auto" && rememberedKind !== null
+        ? rememberedKind
+        : browserMode;
 
   try {
-    const raw = getCdpClient(context, { mode: effectiveMode });
+    const raw = getCdpClient(context, { mode: effectiveMode, targetClientId });
     const cdp = wrapWithKindMemo(raw, context.conversationId);
     return { cdp, browserMode };
   } catch (err) {
@@ -395,10 +404,16 @@ function acquireCdpClientWithMode(
     // a remembered backend kind that has since become unavailable. Drop
     // the stale memo and retry with fresh auto selection so a dead
     // sticky preference doesn't surface as a hard failure.
-    if (browserMode === "auto" && effectiveMode !== "auto") {
+    // Do not apply this fallback when target_client_id is set — a targeting
+    // failure must surface as an error, not silently route elsewhere.
+    if (
+      browserMode === "auto" &&
+      effectiveMode !== "auto" &&
+      targetClientId == null
+    ) {
       browserManager.clearPreferredBackendKind(context.conversationId);
       try {
-        const raw = getCdpClient(context, { mode: "auto" });
+        const raw = getCdpClient(context, { mode: "auto", targetClientId });
         const cdp = wrapWithKindMemo(raw, context.conversationId);
         return { cdp, browserMode };
       } catch (retryErr) {
@@ -941,10 +956,10 @@ export async function executeBrowserNavigate(
                 "2. Use credential fill to enter email/password from credential_store",
               );
               lines.push(
-                "3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
+                "3. For email verification codes, use ui_show with a form to request the code mid-turn",
               );
               lines.push(
-                "4. Do NOT give up or tell the user to sign in manually - handle the login flow yourself",
+                "4. Do NOT give up or suggest manual sign-in - handle the login flow yourself",
               );
             }
           } else {
@@ -953,7 +968,7 @@ export async function executeBrowserNavigate(
               "⚠️ CAPTCHA/Cloudflare verification detected on this page.",
             );
             lines.push(
-              "The user needs to solve this challenge manually. Please inform the user that the page requires human verification before the content can be accessed.",
+              "This challenge requires human verification. Surface this clearly: the page cannot be accessed until the verification is solved manually.",
             );
           }
         } else {
@@ -968,10 +983,10 @@ export async function executeBrowserNavigate(
             "2. Use credential fill to enter email/password from credential_store",
           );
           lines.push(
-            "3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
+            "3. For email verification codes, use ui_show with a form to request the code mid-turn",
           );
           lines.push(
-            "4. Do NOT give up or tell the user to sign in manually - handle the login flow yourself",
+            "4. Do NOT give up or suggest manual sign-in - handle the login flow yourself",
           );
         }
       }