@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/tools/browser/cdp-client/cdp-dom-helpers.ts

@@ -154,6 +154,173 @@ export async function dispatchInsertText(
   await cdp.send("Input.insertText", { text }, signal);
 }
 
+interface TextInsertionOptions {
+  clearFirst: boolean;
+  verify?: boolean;
+}
+
+/**
+ * Select editable content, insert text through CDP, and confirm that the live
+ * target contains the requested text. Controls without text-selection support
+ * are cleared through their native value setter and an input event.
+ */
+export async function insertTextIntoElement(
+  cdp: CdpClient,
+  backendNodeId: number,
+  text: string,
+  options: TextInsertionOptions,
+  signal?: AbortSignal,
+): Promise<void> {
+  await focusElement(cdp, backendNodeId, signal);
+
+  const { object } = await cdp.send<{ object: { objectId: string } }>(
+    "DOM.resolveNode",
+    { backendNodeId },
+    signal,
+  );
+  const selection = await cdp.send<{
+    result?: {
+      value?: { initialLength?: number; needsRefocus?: boolean };
+    };
+    exceptionDetails?: { text?: string; exception?: { description?: string } };
+  }>(
+    "Runtime.callFunctionOn",
+    {
+      objectId: object.objectId,
+      functionDeclaration: `function(clearFirst) {
+        if (typeof this.value === "string") {
+          const value = this.value;
+          if (clearFirst && typeof this.select === "function") {
+            try {
+              this.select();
+              if (this.selectionStart === 0 && this.selectionEnd === value.length) {
+                return { needsRefocus: false };
+              }
+            } catch {}
+
+            const view = this.ownerDocument?.defaultView;
+            const constructor = this.tagName === "INPUT"
+              ? view?.HTMLInputElement
+              : this.tagName === "TEXTAREA"
+                ? view?.HTMLTextAreaElement
+                : undefined;
+            const valueSetter = constructor
+              ? Object.getOwnPropertyDescriptor(constructor.prototype, "value")?.set
+              : undefined;
+            if (!valueSetter || !view?.InputEvent) {
+              throw new Error("Element does not support replacing its text");
+            }
+            valueSetter.call(this, "");
+            this.dispatchEvent(new view.InputEvent("input", {
+              bubbles: true,
+              inputType: "deleteContentBackward",
+              data: null,
+            }));
+            return { needsRefocus: true };
+          }
+          if (typeof this.setSelectionRange === "function") {
+            const end = this.value.length;
+            try {
+              this.setSelectionRange(clearFirst ? 0 : end, end);
+            } catch (error) {
+              if (clearFirst) {
+                throw error;
+              }
+            }
+            return clearFirst
+              ? { needsRefocus: false }
+              : { initialLength: value.length, needsRefocus: false };
+          }
+        }
+        if (this.isContentEditable) {
+          const value = this.textContent ?? "";
+          const selection = this.ownerDocument.getSelection();
+          const range = this.ownerDocument.createRange();
+          range.selectNodeContents(this);
+          if (!clearFirst) {
+            range.collapse(false);
+          }
+          selection.removeAllRanges();
+          selection.addRange(range);
+          return clearFirst
+            ? { needsRefocus: false }
+            : { initialLength: value.length, needsRefocus: false };
+        }
+        throw new Error("Element is not an editable text target");
+      }`,
+      arguments: [{ value: options.clearFirst }],
+      returnByValue: true,
+    },
+    signal,
+  );
+  if (selection.exceptionDetails) {
+    const message =
+      selection.exceptionDetails.exception?.description ??
+      selection.exceptionDetails.text ??
+      "Unable to select editable text";
+    throw new CdpError("cdp_error", message, {
+      cdpMethod: "Runtime.callFunctionOn",
+      cdpParams: { backendNodeId },
+    });
+  }
+
+  if (selection.result?.value?.needsRefocus === true) {
+    await focusElement(cdp, backendNodeId, signal);
+  }
+  await dispatchInsertText(cdp, text, signal);
+  if (options.verify === false) {
+    return;
+  }
+
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const verification = await cdp.send<{
+      result?: { value?: { connected?: boolean; matches?: boolean } };
+    }>(
+      "Runtime.callFunctionOn",
+      {
+        objectId: object.objectId,
+        functionDeclaration: `function(insertedText, clearFirst, initialLength) {
+          const text = typeof this.value === "string"
+            ? this.value
+            : (this.textContent ?? "");
+          const normalizedInsertedText = this.tagName === "TEXTAREA"
+            ? insertedText.replace(/\\r\\n?/g, "\\n")
+            : insertedText;
+          const matches = clearFirst
+            ? text === normalizedInsertedText
+            : Number.isInteger(initialLength)
+              && text.length === initialLength + normalizedInsertedText.length
+              && text.endsWith(normalizedInsertedText);
+          return { connected: this.isConnected, matches };
+        }`,
+        arguments: [
+          { value: text },
+          { value: options.clearFirst },
+          { value: selection.result?.value?.initialLength ?? null },
+        ],
+        returnByValue: true,
+      },
+      signal,
+    );
+    const result = verification.result?.value;
+    if (result?.connected === true && result.matches === true) {
+      return;
+    }
+    if (attempt < 2) {
+      await new Promise((resolve) => setTimeout(resolve, 50));
+    }
+  }
+
+  throw new CdpError(
+    "cdp_error",
+    "Text insertion did not persist in the editable element.",
+    {
+      cdpMethod: "Input.insertText",
+      cdpParams: { backendNodeId },
+    },
+  );
+}
+
 /**
  * Per-key descriptor used by {@link dispatchKeyPress}. Mirrors the
  * fields CDP's `Input.dispatchKeyEvent` accepts. `text` is set only
@@ -166,6 +333,116 @@ interface KeyDescriptor {
   text?: string;
 }
 
+interface ModifierDescriptor {
+  bit: number;
+  descriptor: KeyDescriptor;
+}
+
+const ALT_MODIFIER: ModifierDescriptor = {
+  bit: 1,
+  descriptor: { key: "Alt", code: "AltLeft", windowsVirtualKeyCode: 18 },
+};
+const CONTROL_MODIFIER: ModifierDescriptor = {
+  bit: 2,
+  descriptor: {
+    key: "Control",
+    code: "ControlLeft",
+    windowsVirtualKeyCode: 17,
+  },
+};
+const META_MODIFIER: ModifierDescriptor = {
+  bit: 4,
+  descriptor: { key: "Meta", code: "MetaLeft", windowsVirtualKeyCode: 91 },
+};
+const SHIFT_MODIFIER: ModifierDescriptor = {
+  bit: 8,
+  descriptor: {
+    key: "Shift",
+    code: "ShiftLeft",
+    windowsVirtualKeyCode: 16,
+  },
+};
+
+const MODIFIER_DESCRIPTORS: Record<string, ModifierDescriptor> = {
+  alt: ALT_MODIFIER,
+  option: ALT_MODIFIER,
+  control: CONTROL_MODIFIER,
+  ctrl: CONTROL_MODIFIER,
+  meta: META_MODIFIER,
+  command: META_MODIFIER,
+  cmd: META_MODIFIER,
+  shift: SHIFT_MODIFIER,
+};
+
+const PUNCTUATION_KEYS = [
+  {
+    unshifted: "`",
+    shifted: "~",
+    code: "Backquote",
+    windowsVirtualKeyCode: 192,
+  },
+  {
+    unshifted: "-",
+    shifted: "_",
+    code: "Minus",
+    windowsVirtualKeyCode: 189,
+  },
+  {
+    unshifted: "=",
+    shifted: "+",
+    code: "Equal",
+    windowsVirtualKeyCode: 187,
+  },
+  {
+    unshifted: "[",
+    shifted: "{",
+    code: "BracketLeft",
+    windowsVirtualKeyCode: 219,
+  },
+  {
+    unshifted: "]",
+    shifted: "}",
+    code: "BracketRight",
+    windowsVirtualKeyCode: 221,
+  },
+  {
+    unshifted: "\\",
+    shifted: "|",
+    code: "Backslash",
+    windowsVirtualKeyCode: 220,
+  },
+  {
+    unshifted: ";",
+    shifted: ":",
+    code: "Semicolon",
+    windowsVirtualKeyCode: 186,
+  },
+  {
+    unshifted: "'",
+    shifted: '"',
+    code: "Quote",
+    windowsVirtualKeyCode: 222,
+  },
+  {
+    unshifted: ",",
+    shifted: "<",
+    code: "Comma",
+    windowsVirtualKeyCode: 188,
+  },
+  {
+    unshifted: ".",
+    shifted: ">",
+    code: "Period",
+    windowsVirtualKeyCode: 190,
+  },
+  {
+    unshifted: "/",
+    shifted: "?",
+    code: "Slash",
+    windowsVirtualKeyCode: 191,
+  },
+] as const;
+
 /**
  * Subset of the US keyboard layout used to populate
  * `Input.dispatchKeyEvent` params. Without these fields, sites that
@@ -182,7 +459,7 @@ const KEY_DESCRIPTORS: Record<string, KeyDescriptor> = {
     windowsVirtualKeyCode: 13,
     text: "\r",
   },
-  Tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9, text: "\t" },
+  Tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9 },
   Escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
   Backspace: {
     key: "Backspace",
@@ -251,6 +528,17 @@ function resolveKeyDescriptor(key: string): KeyDescriptor | null {
   if (fromMap) {
     return fromMap;
   }
+  const punctuation = PUNCTUATION_KEYS.find(
+    ({ unshifted, shifted }) => key === unshifted || key === shifted,
+  );
+  if (punctuation) {
+    return {
+      key,
+      code: punctuation.code,
+      windowsVirtualKeyCode: punctuation.windowsVirtualKeyCode,
+      text: key,
+    };
+  }
   if (key.length !== 1) {
     return null;
   }
@@ -291,60 +579,168 @@ function resolveKeyDescriptor(key: string): KeyDescriptor | null {
   return null;
 }
 
+function describeKeyEvent(
+  descriptor: KeyDescriptor,
+  type: "rawKeyDown" | "char" | "keyUp",
+  modifiers: number,
+  unmodifiedText = descriptor.text,
+): Record<string, unknown> {
+  const params: Record<string, unknown> = {
+    type,
+    key: descriptor.key,
+    code: descriptor.code,
+    windowsVirtualKeyCode: descriptor.windowsVirtualKeyCode,
+  };
+  if (modifiers !== 0) {
+    params.modifiers = modifiers;
+  }
+  if (type === "char" && descriptor.text !== undefined) {
+    params.text = descriptor.text;
+    params.unmodifiedText = unmodifiedText;
+  }
+  return params;
+}
+
+const SHIFTED_CHARACTERS: Record<string, string> = {
+  "1": "!",
+  "2": "@",
+  "3": "#",
+  "4": "$",
+  "5": "%",
+  "6": "^",
+  "7": "&",
+  "8": "*",
+  "9": "(",
+  "0": ")",
+  ...Object.fromEntries(
+    PUNCTUATION_KEYS.map(({ unshifted, shifted }) => [unshifted, shifted]),
+  ),
+};
+
+function applyShift(descriptor: KeyDescriptor): KeyDescriptor {
+  if (descriptor.text === undefined) {
+    return descriptor;
+  }
+  const shifted =
+    descriptor.text >= "a" && descriptor.text <= "z"
+      ? descriptor.text.toUpperCase()
+      : (SHIFTED_CHARACTERS[descriptor.text] ?? descriptor.text);
+  return { ...descriptor, key: shifted, text: shifted };
+}
+
 /**
- * Press a single key (keyDown + optional `char` + keyUp). Resolves
- * the key name to a {@link KeyDescriptor} so CDP receives the right
- * `code` / `windowsVirtualKeyCode` / `text` fields — required by
- * sites that check `event.keyCode` (e.g. Enter-to-submit) or
- * `event.code`. For printable keys we also dispatch a `char` event
- * between keyDown and keyUp so the character is actually inserted
- * into focused inputs.
+ * Press a key or modifier chord using the same `Modifier+Key` spelling as
+ * Playwright. Printable keys use rawKeyDown + char + keyUp, with text only on
+ * the char event, so the browser inserts exactly one character.
  */
 export async function dispatchKeyPress(
   cdp: CdpClient,
   key: string,
   signal?: AbortSignal,
 ): Promise<void> {
-  const desc = resolveKeyDescriptor(key);
-  if (!desc) {
-    // Unknown multi-character key (e.g. F-keys we have not mapped).
-    // Fall back to the minimal payload so callers still see a
-    // keyDown/keyUp pair, and warn so we can extend the map.
-
-    console.warn(
-      `dispatchKeyPress: no descriptor for key "${key}", sending minimal event`,
-    );
-    await cdp.send("Input.dispatchKeyEvent", { type: "keyDown", key }, signal);
-    await cdp.send("Input.dispatchKeyEvent", { type: "keyUp", key }, signal);
-    return;
+  const parts = key === "+" ? [key] : key.split("+");
+  const primaryKey = parts.at(-1) ?? "";
+  const modifierTokens = parts.slice(0, -1);
+  const modifiers: ModifierDescriptor[] = [];
+  let modifierBits = 0;
+
+  for (const token of modifierTokens) {
+    const modifier = MODIFIER_DESCRIPTORS[token.toLowerCase()];
+    if (!modifier || (modifierBits & modifier.bit) !== 0) {
+      throw new CdpError("cdp_error", `Unsupported key chord: ${key}`);
+    }
+    modifierBits |= modifier.bit;
+    modifiers.push(modifier);
   }
 
-  const baseParams: Record<string, unknown> = {
-    key: desc.key,
-    code: desc.code,
-    windowsVirtualKeyCode: desc.windowsVirtualKeyCode,
-  };
-  if (desc.text !== undefined) {
-    baseParams.text = desc.text;
+  const standaloneModifier = MODIFIER_DESCRIPTORS[primaryKey.toLowerCase()];
+  if (standaloneModifier && (modifierBits & standaloneModifier.bit) !== 0) {
+    throw new CdpError("cdp_error", `Unsupported key chord: ${key}`);
+  }
+  const unmodifiedDescriptor =
+    standaloneModifier?.descriptor ?? resolveKeyDescriptor(primaryKey);
+  if (!unmodifiedDescriptor) {
+    throw new CdpError("cdp_error", `Unsupported key: ${primaryKey || key}`);
   }
+  const descriptor =
+    (modifierBits & SHIFT_MODIFIER.bit) !== 0
+      ? applyShift(unmodifiedDescriptor)
+      : unmodifiedDescriptor;
+  const primaryDownModifiers = modifierBits | (standaloneModifier?.bit ?? 0);
 
-  await cdp.send(
-    "Input.dispatchKeyEvent",
-    { ...baseParams, type: "keyDown" },
-    signal,
-  );
-  if (desc.text !== undefined) {
+  let activeModifiers = 0;
+  const pressedModifiers: ModifierDescriptor[] = [];
+  let primaryPressed = false;
+  try {
+    for (const modifier of modifiers) {
+      const nextModifiers = activeModifiers | modifier.bit;
+      activeModifiers = nextModifiers;
+      pressedModifiers.push(modifier);
+      await cdp.send(
+        "Input.dispatchKeyEvent",
+        describeKeyEvent(modifier.descriptor, "rawKeyDown", nextModifiers),
+        signal,
+      );
+    }
+
+    const suppressText = (modifierBits & (1 | 2 | 4)) !== 0;
+    primaryPressed = true;
     await cdp.send(
       "Input.dispatchKeyEvent",
-      { ...baseParams, type: "char" },
+      describeKeyEvent(descriptor, "rawKeyDown", primaryDownModifiers),
       signal,
     );
+    if (descriptor.text !== undefined && !suppressText) {
+      await cdp.send(
+        "Input.dispatchKeyEvent",
+        describeKeyEvent(
+          descriptor,
+          "char",
+          modifierBits,
+          unmodifiedDescriptor.text,
+        ),
+        signal,
+      );
+    }
+    await cdp.send(
+      "Input.dispatchKeyEvent",
+      describeKeyEvent(descriptor, "keyUp", modifierBits),
+      signal,
+    );
+    primaryPressed = false;
+
+    while (pressedModifiers.length > 0) {
+      const modifier = pressedModifiers.at(-1)!;
+      const nextModifiers = activeModifiers & ~modifier.bit;
+      await cdp.send(
+        "Input.dispatchKeyEvent",
+        describeKeyEvent(modifier.descriptor, "keyUp", nextModifiers),
+        signal,
+      );
+      activeModifiers = nextModifiers;
+      pressedModifiers.pop();
+    }
+  } finally {
+    if (primaryPressed) {
+      try {
+        await cdp.send(
+          "Input.dispatchKeyEvent",
+          describeKeyEvent(descriptor, "keyUp", activeModifiers),
+        );
+      } catch {}
+    }
+
+    while (pressedModifiers.length > 0) {
+      const modifier = pressedModifiers.pop()!;
+      activeModifiers &= ~modifier.bit;
+      try {
+        await cdp.send(
+          "Input.dispatchKeyEvent",
+          describeKeyEvent(modifier.descriptor, "keyUp", activeModifiers),
+        );
+      } catch {}
+    }
   }
-  await cdp.send(
-    "Input.dispatchKeyEvent",
-    { ...baseParams, type: "keyUp" },
-    signal,
-  );
 }
 
 /** Dispatch a wheel scroll delta at the given viewport point. */

package/src/tools/browser/cdp-client/cdp-inspect-client.ts

@@ -406,14 +406,7 @@ export class CdpInspectClient implements ScopedCdpClient {
     httpError: unknown,
     signal: AbortSignal,
   ): Promise<AttachedSession> {
-    let wsUrl: string;
-    try {
-      wsUrl = this.helpers.buildBrowserWsUrl(host, port);
-    } catch (err) {
-      // buildBrowserWsUrl enforces loopback — if it throws, the host
-      // is non-loopback and we should not attempt any WS connection.
-      throw err;
-    }
+    const wsUrl = this.helpers.buildBrowserWsUrl(host, port);
 
     let transport: CdpWsTransport;
     try {

package/src/tools/client-os.ts

@@ -5,9 +5,15 @@ import {
   parseClientOs,
   supportsHostProxy,
 } from "../channels/types.js";
+import {
+  canUseVirtualDesktop,
+  supportsVirtualDesktopComputerUse,
+} from "../desktop/virtual-desktop-feature.js";
 import { assistantEventHub } from "../runtime/assistant-event-hub.js";
+import type { ToolContext } from "./types.js";
 
 interface HostClientContext {
+  trustClass?: ToolContext["trustClass"];
   clientOs?: ClientOs;
   transportInterface?: InterfaceId;
   sourceActorPrincipalId?: string;
@@ -85,6 +91,15 @@ export function supportsClientOsForSkillTool(
   toolName: string,
   context: HostClientContext,
 ): boolean {
+  if (
+    supportsVirtualDesktopComputerUse(toolName) &&
+    canUseVirtualDesktop({
+      ...context,
+      trustClass: context.trustClass ?? "unknown",
+    })
+  ) {
+    return true;
+  }
   if (supportedClientOs === undefined) {
     return true;
   }

package/src/tools/computer-use/skill-proxy-bridge.ts

@@ -2,7 +2,7 @@
  * Shared helper for computer-use skill wrapper scripts.
  *
  * Each wrapper calls forwardComputerUseProxyTool() to delegate execution to
- * the proxy resolver, which forwards the call to the connected desktop client.
+ * the proxy resolver, which selects the connected or virtual desktop.
  */
 
 import { formatDesktopAppRequired } from "../capability-offer.js";

package/src/tools/computer-use/target.ts

@@ -0,0 +1,42 @@
+import {
+  isVirtualDesktopPlatform,
+  shouldUseVirtualDesktop,
+} from "../../desktop/virtual-desktop-feature.js";
+import type { ExecutionTarget } from "../tool-types.js";
+
+export function computerUseTarget(
+  input: Record<string, unknown>,
+  context?: Parameters<typeof shouldUseVirtualDesktop>[0],
+): "connected-computer" | "assistant-desktop" {
+  // Availability is checked at dispatch, so revocation cannot switch computers.
+  const target =
+    input.target ??
+    (input.target_client_id === undefined &&
+    context &&
+    shouldUseVirtualDesktop(context, isVirtualDesktopPlatform())
+      ? "assistant-desktop"
+      : "connected-computer");
+  if (target !== "connected-computer" && target !== "assistant-desktop") {
+    throw new Error("Choose target connected-computer or assistant-desktop");
+  }
+  if (target === "assistant-desktop" && input.target_client_id !== undefined) {
+    throw new Error("target_client_id only applies to a connected computer");
+  }
+  if (target === "connected-computer" && input.observation_id !== undefined) {
+    throw new Error("observation_id only applies to the assistant desktop");
+  }
+  return target;
+}
+
+export function computerUseExecutionTarget(
+  input: Record<string, unknown>,
+  context?: Parameters<typeof shouldUseVirtualDesktop>[0],
+): ExecutionTarget {
+  try {
+    return computerUseTarget(input, context) === "assistant-desktop"
+      ? "sandbox"
+      : "host";
+  } catch {
+    return "host";
+  }
+}