@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/daemon/wake-conversation-ops.ts

@@ -22,7 +22,7 @@ import {
 import { syncMessageToDisk } from "../persistence/conversation-disk-view.js";
 import { backfillMessageIdOnLogs } from "../persistence/llm-request-log-store.js";
 import { resolveMediaSourceData } from "../providers/media-resolve.js";
-import type { Message } from "../providers/types.js";
+import type { Message, ToolDefinition } from "../providers/types.js";
 import { broadcastMessage } from "../runtime/assistant-event-hub.js";
 import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
 import type { CompletedBackgroundTool } from "../tools/background-tool-registry.js";
@@ -350,38 +350,70 @@ export async function persistWakeTriggerMessage(
   }
 }
 
+/**
+ * How a wake's tool allowlist is applied beyond the allowlist itself. Every
+ * field is set on the conversation together with the allowlist and restored
+ * with it.
+ */
+export interface WakeToolScopeOptions {
+  /** How the allowlist is enforced; see {@link SubagentToolGateMode}. */
+  gateMode?: SubagentToolGateMode;
+  /**
+   * Freezes the client-context inputs for tool-definition resolution
+   * (execution-gate-mode cache-parity wakes); see {@link WakeToolContextPin}.
+   * Its `requestOrigin`, when set, is stamped onto the conversation's per-turn
+   * origin so the permission checker's origin-scoped auto-grants fire.
+   */
+  toolContextPin?: WakeToolContextPin;
+  /**
+   * Skills whose bundled tools join the turn's active set
+   * (`allowedToolNames`) so they are callable without a prior `skill_load`.
+   */
+  preactivateSkillIds?: readonly string[];
+  /**
+   * Sent verbatim as the wake's wire tool array in place of the resolved one
+   * (the source conversation's recorded surface, replayed for provider
+   * prompt-cache parity). Changes nothing about what may execute.
+   */
+  wireToolDefinitions?: readonly ToolDefinition[];
+  /**
+   * The delegation section's rendered state recorded with the source's
+   * surface, replayed into the wake's system prompt alongside
+   * `wireToolDefinitions`. Changes nothing about what may execute.
+   */
+  delegateIndependentTasks?: boolean;
+}
+
 /**
  * Temporarily restrict the tools visible/executable during a wake by
  * reusing the conversation's subagent allowlist slot. Returns a restore
  * callback that reinstates the previous allowlist so the wake can release
  * the scope before any queued user turn is drained.
- *
- * `gateMode` controls how the allowlist is enforced — see
- * {@link SubagentToolGateMode}. `toolContextPin`, when provided
- * (execution-gate-mode cache-parity wakes), freezes the client-context
- * inputs for tool-definition resolution — see {@link WakeToolContextPin}.
- * Its `requestOrigin`, when set, is stamped onto the conversation's per-turn
- * origin so the permission checker's origin-scoped auto-grants fire for the
- * wake. `preactivateSkillIds`, when non-empty, activates those skills' bundled
- * tools in the turn's active set (`allowedToolNames`) for the wake so they are
- * callable without a prior `skill_load`. All are set and restored alongside the
- * allowlist.
  */
 export function scopeWakeAllowedTools(
   conversation: Conversation,
   tools: ReadonlySet<string>,
-  gateMode: SubagentToolGateMode = "wire",
-  toolContextPin?: WakeToolContextPin,
-  preactivateSkillIds?: readonly string[],
+  {
+    gateMode = "wire",
+    toolContextPin,
+    preactivateSkillIds,
+    wireToolDefinitions,
+    delegateIndependentTasks,
+  }: WakeToolScopeOptions = {},
 ): () => void {
   const previous = conversation.subagentAllowedTools;
   const previousGateMode = conversation.subagentToolGateMode;
   const previousToolContextPin = conversation.toolContextPin;
   const previousRequestOrigin = conversation.currentTurnRequestOrigin;
   const previousPreactivated = conversation.preactivatedSkillIds;
+  const previousWireToolReplay = conversation.wireToolReplay;
+  const previousDelegateIndependentTasksReplay =
+    conversation.delegateIndependentTasksReplay;
   conversation.setSubagentAllowedTools(new Set(tools));
   conversation.subagentToolGateMode = gateMode;
   conversation.toolContextPin = toolContextPin;
+  conversation.wireToolReplay = wireToolDefinitions;
+  conversation.delegateIndependentTasksReplay = delegateIndependentTasks;
   if (toolContextPin?.requestOrigin !== undefined) {
     conversation.currentTurnRequestOrigin = toolContextPin.requestOrigin;
   }
@@ -394,5 +426,8 @@ export function scopeWakeAllowedTools(
     conversation.toolContextPin = previousToolContextPin;
     conversation.currentTurnRequestOrigin = previousRequestOrigin;
     conversation.setPreactivatedSkillIds(previousPreactivated);
+    conversation.wireToolReplay = previousWireToolReplay;
+    conversation.delegateIndependentTasksReplay =
+      previousDelegateIndependentTasksReplay;
   };
 }

package/src/desktop/__tests__/fake-desktop.ts

@@ -70,6 +70,7 @@ export function newFakeDesktop(options: FakeDesktopOptions) {
     which: (binary) =>
       options.missingBinaries?.includes(binary) ? null : `/usr/bin/${binary}`,
     probeVncPort: async () => vncReady,
+    probeSessionBus: async () => true,
     resolveChromePath: () => chromiumPath(),
     renderWallpaper: options.renderWallpaper ?? (async () => null),
     killProcessGroup: (child, signal) => {

package/src/desktop/desktop-accessibility-script.ts

@@ -0,0 +1,88 @@
+/** Runs inside the virtual desktop's Linux session, using the AT-SPI D-Bus API. */
+export const DESKTOP_ACCESSIBILITY_SCRIPT = String.raw`
+import json, os, sys, time
+import dbus
+
+ACCESSIBLE = "org.a11y.atspi.Accessible"
+COMPONENT = "org.a11y.atspi.Component"
+PROPERTIES = "org.freedesktop.DBus.Properties"
+ROOT = "/org/a11y/atspi/accessible/root"
+LIMIT = 300
+DEADLINE = time.monotonic() + 2
+
+def call(bus, ref, interface, method, *args):
+    remaining = DEADLINE - time.monotonic()
+    if remaining <= 0:
+        raise TimeoutError("Accessibility query exceeded its time limit")
+    obj = bus.get_object(str(ref[0]), str(ref[1]), introspect=False)
+    return obj.get_dbus_method(method, interface)(*args, timeout=min(0.2, remaining))
+
+def read(bus, ref, depth=0):
+    name = call(bus, ref, PROPERTIES, "Get", ACCESSIBLE, "Name")
+    child_count = call(bus, ref, PROPERTIES, "Get", ACCESSIBLE, "ChildCount")
+    role = str(call(bus, ref, ACCESSIBLE, "GetRoleName"))
+    states = call(bus, ref, ACCESSIBLE, "GetState")
+    bits = sum(int(word) << (32 * i) for i, word in enumerate(states))
+    showing = bool(bits & (1 << 25)) and bool(bits & (1 << 30))
+    bounds = None
+    if showing and not bits & (1 << 6):
+        try:
+            bounds = [int(n) for n in call(bus, ref, COMPONENT, "GetExtents", dbus.UInt32(0))]
+        except dbus.DBusException:
+            pass
+    return {
+        "bus": str(ref[0]), "path": str(ref[1]), "depth": depth,
+        "name": str(name)[:200], "role": role[:80],
+        "bounds": bounds,
+        "states": [label for bit, label in [(4, "checked"), (8, "enabled"), (12, "focused"), (23, "selected")] if bits & (1 << bit)],
+    }, int(child_count)
+
+def main():
+    request = json.loads(sys.argv[1])
+    session = dbus.bus.BusConnection(os.environ["DBUS_SESSION_BUS_ADDRESS"])
+    service = ("org.a11y.Bus", "/org/a11y/bus")
+    call(session, service, PROPERTIES, "Set", "org.a11y.Status", "IsEnabled", dbus.Boolean(True, variant_level=1))
+    address = str(call(session, service, "org.a11y.Bus", "GetAddress"))
+    bus = dbus.bus.BusConnection(address)
+    bus_id = str(call(bus, ("org.freedesktop.DBus", "/org/freedesktop/DBus"), "org.freedesktop.DBus", "GetId"))
+    if request["operation"] == "resolve":
+        if request["busId"] != bus_id:
+            raise ValueError("Accessibility session changed. Observe again.")
+        target = request["target"]
+        node, _ = read(bus, (target["bus"], target["path"]))
+        if node["name"] != target["name"] or node["role"] != target["role"] or not node["bounds"]:
+            raise ValueError("Accessibility element changed or is hidden. Observe again.")
+        print(json.dumps(node))
+        return
+    pending = [("org.a11y.atspi.Registry", ROOT, 0)]
+    seen, nodes = set(), []
+    truncated = False
+    while pending and len(seen) < LIMIT and time.monotonic() < DEADLINE:
+        name, path, depth = pending.pop()
+        ref = (name, path)
+        if ref in seen or path == "/org/a11y/atspi/null":
+            continue
+        seen.add(ref)
+        try:
+            node, count = read(bus, ref, depth)
+            if node["bounds"]:
+                nodes.append(node)
+            if depth < 20:
+                children = []
+                for i in range(min(count, LIMIT - len(seen))):
+                    child = call(bus, ref, ACCESSIBLE, "GetChildAtIndex", dbus.Int32(i))
+                    children.append((str(child[0]) or name, str(child[1]), depth + 1))
+                pending.extend(reversed(children))
+                truncated = truncated or count > len(children)
+            elif count:
+                truncated = True
+        except (dbus.DBusException, TimeoutError):
+            truncated = True
+    print(json.dumps({"busId": bus_id, "nodes": nodes, "truncated": truncated or bool(pending)}))
+
+try:
+    main()
+except Exception as error:
+    print(str(error), file=sys.stderr)
+    sys.exit(1)
+`;

package/src/desktop/desktop-accessibility.test.ts

@@ -0,0 +1,160 @@
+import { describe, expect, mock, test } from "bun:test";
+
+import { DesktopAccessibility } from "./desktop-accessibility.js";
+import {
+  performDesktopComputerUse,
+  planDesktopComputerUse,
+} from "./desktop-computer-use.js";
+
+const node = {
+  bus: ":1.2",
+  path: "/org/a11y/atspi/accessible/7",
+  name: "Results",
+  role: "scroll pane",
+  depth: 2,
+  states: ["enabled"],
+  bounds: [100, 200, 400, 300],
+};
+const signal = () => new AbortController().signal;
+
+describe("desktop accessibility", () => {
+  test("observations publish IDs and actions resolve current, clipped bounds", async () => {
+    const run = mock(async (request: object) =>
+      "target" in request
+        ? { ...node, bounds: [-20, 300, 200, 300] }
+        : { busId: "bus-1", nodes: [node], truncated: false },
+    );
+    const reader = new DesktopAccessibility(run);
+    expect(
+      (await reader.observe("unix:abstract=test", signal())).axTree,
+    ).toContain('[1] scroll pane "Results"');
+    reader.bindObservation("obs-1");
+    expect(await reader.resolve(1, "obs-1", signal())).toEqual({
+      x: 89,
+      y: 449,
+    });
+    expect(run.mock.calls[1][0]).toMatchObject({
+      operation: "resolve",
+      busId: "bus-1",
+      target: node,
+    });
+    await expect(
+      reader.resolve(1, "old-observation", signal()),
+    ).rejects.toThrow("stale");
+    await expect(reader.resolve(2, "obs-1", signal())).rejects.toThrow("stale");
+    expect(run).toHaveBeenCalledTimes(2);
+  });
+
+  test("an unavailable tree preserves screenshot fallback and invalidates old IDs", async () => {
+    const run = mock(async () => ({
+      busId: "bus-1",
+      nodes: [node],
+      truncated: false,
+    }));
+    const reader = new DesktopAccessibility(run);
+    await reader.observe("bus", signal());
+    reader.bindObservation("obs-1");
+    run.mockRejectedValue(new Error("bus disconnected"));
+    expect(await reader.observe("bus", signal())).toMatchObject({
+      userGuidance: expect.stringContaining("coordinates"),
+    });
+    await expect(reader.resolve(1, "obs-1", signal())).rejects.toThrow("stale");
+  });
+
+  test("cancellation is not hidden by screenshot fallback", async () => {
+    const reader = new DesktopAccessibility(async () => {
+      throw new Error("aborted");
+    });
+    const abort = new AbortController();
+    abort.abort(new Error("cancelled"));
+    await expect(reader.observe("bus", abort.signal)).rejects.toThrow(
+      "cancelled",
+    );
+  });
+
+  test("a hidden or removed target sends no input", async () => {
+    const reader = new DesktopAccessibility(async (request) => {
+      if ("target" in request) {
+        throw new Error("Object no longer exists");
+      }
+      return { busId: "bus-1", nodes: [node], truncated: false };
+    });
+    await reader.observe("bus", signal());
+    reader.bindObservation("obs-1");
+    const input = mock(async () => {});
+    const result = await performDesktopComputerUse(
+      planDesktopComputerUse("computer_use_scroll", {
+        element_id: 1,
+        direction: "down",
+        amount: 2,
+      }),
+      signal(),
+      {
+        input,
+        resolveElement: reader.resolve.bind(reader),
+        capture: async () => ({ screenshot: "image" }),
+      },
+      "obs-1",
+    );
+    expect(input).not.toHaveBeenCalled();
+    expect(result.executionError).toContain("Observe again");
+    expect(result.screenshot).toBe("image");
+  });
+
+  test("sequence IDs are resolved immediately before each step using the shared input driver", async () => {
+    let top = 200;
+    const reader = new DesktopAccessibility(async (request) =>
+      "target" in request
+        ? { ...node, bounds: [100, top, 400, 100] }
+        : { busId: "bus-1", nodes: [node], truncated: false },
+    );
+    await reader.observe("bus", signal());
+    reader.bindObservation("obs-1");
+    const input = mock(async (_args: string[], _signal?: AbortSignal) => {
+      top += 100;
+    });
+    const capture = mock(async () => ({ screenshot: "image" }));
+    await performDesktopComputerUse(
+      planDesktopComputerUse("computer_use_sequence", {
+        actions: [
+          { action: "click", element_id: 1 },
+          { action: "scroll", element_id: 1, direction: "down", amount: 2 },
+        ],
+      }),
+      signal(),
+      { input, capture, resolveElement: reader.resolve.bind(reader) },
+      "obs-1",
+    );
+    expect(input.mock.calls).toEqual([
+      [
+        [
+          "mousemove",
+          "299",
+          "249",
+          "click",
+          "--repeat",
+          "1",
+          "--delay",
+          "100",
+          "1",
+        ],
+        expect.any(AbortSignal),
+      ],
+      [
+        [
+          "mousemove",
+          "299",
+          "349",
+          "click",
+          "--repeat",
+          "2",
+          "--delay",
+          "50",
+          "5",
+        ],
+        expect.any(AbortSignal),
+      ],
+    ]);
+    expect(capture).toHaveBeenCalledTimes(1);
+  });
+});

package/src/desktop/desktop-accessibility.ts

@@ -0,0 +1,169 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+import { DESKTOP_ACCESSIBILITY_SCRIPT } from "./desktop-accessibility-script.js";
+import {
+  DESKTOP_DISPLAY,
+  DESKTOP_HEIGHT,
+  DESKTOP_WIDTH,
+} from "./desktop-display.js";
+
+interface AccessibleElement {
+  bus: string;
+  path: string;
+  name: string;
+  role: string;
+  depth: number;
+  states: string[];
+  bounds: [number, number, number, number];
+}
+interface Snapshot {
+  busId: string;
+  nodes: AccessibleElement[];
+  truncated: boolean;
+}
+type Query = (
+  request: object,
+  address: string,
+  signal: AbortSignal,
+) => Promise<unknown>;
+const exec = promisify(execFile);
+const query: Query = async (request, address, signal) => {
+  const { stdout } = await exec(
+    "/usr/bin/python3",
+    ["-c", DESKTOP_ACCESSIBILITY_SCRIPT, JSON.stringify(request)],
+    {
+      env: {
+        PATH: process.env.PATH,
+        LANG: "C.UTF-8",
+        DISPLAY: DESKTOP_DISPLAY,
+        DBUS_SESSION_BUS_ADDRESS: address,
+      },
+      signal,
+      timeout: 3_000,
+      maxBuffer: 512 * 1024,
+      windowsHide: true,
+    },
+  );
+  return JSON.parse(stdout);
+};
+
+function center(
+  bounds: AccessibleElement["bounds"],
+): { x: number; y: number } | undefined {
+  const [x, y, width, height] = bounds;
+  if (
+    ![x, y, width, height].every(Number.isFinite) ||
+    width <= 0 ||
+    height <= 0
+  ) {
+    return undefined;
+  }
+  const left = Math.max(0, x);
+  const top = Math.max(0, y);
+  const right = Math.min(DESKTOP_WIDTH, x + width);
+  const bottom = Math.min(DESKTOP_HEIGHT, y + height);
+  if (right <= left || bottom <= top) {
+    return undefined;
+  }
+  return {
+    x: Math.floor((left + right - 1) / 2),
+    y: Math.floor((top + bottom - 1) / 2),
+  };
+}
+
+export class DesktopAccessibility {
+  private elements = new Map<number, AccessibleElement>();
+  private observationId?: string;
+  private busId?: string;
+  private address?: string;
+
+  constructor(private readonly run: Query = query) {}
+
+  async observe(
+    address: string,
+    signal: AbortSignal,
+  ): Promise<{ axTree?: string; userGuidance?: string }> {
+    signal.throwIfAborted();
+    this.elements.clear();
+    this.busId = undefined;
+    this.observationId = undefined;
+    this.address = address;
+    try {
+      const snapshot = (await this.run(
+        { operation: "observe" },
+        address,
+        signal,
+      )) as Snapshot;
+      signal.throwIfAborted();
+      this.busId = snapshot.busId;
+      const lines: string[] = [];
+      for (const node of snapshot.nodes) {
+        if (!center(node.bounds)) {
+          continue;
+        }
+        const id = this.elements.size + 1;
+        this.elements.set(id, node);
+        lines.push(
+          `${"  ".repeat(Math.min(node.depth, 20))}[${id}] ${node.role} ${JSON.stringify(node.name)} ${node.states.join(" ")}`.trimEnd(),
+        );
+      }
+      return {
+        axTree: lines.length ? lines.join("\n") : undefined,
+        ...(!lines.length || snapshot.truncated
+          ? {
+              userGuidance:
+                "Accessibility information is incomplete. Use screenshot coordinates for elements not listed.",
+            }
+          : {}),
+      };
+    } catch {
+      signal.throwIfAborted();
+      this.elements.clear();
+      return {
+        userGuidance:
+          "Accessibility is unavailable. Use screenshot coordinates for this observation.",
+      };
+    }
+  }
+
+  bindObservation(id: string): void {
+    this.observationId = id;
+  }
+
+  async resolve(
+    id: number,
+    observationId: string,
+    signal: AbortSignal,
+  ): Promise<{ x: number; y: number }> {
+    signal.throwIfAborted();
+    const target = this.elements.get(id);
+    if (
+      !target ||
+      observationId !== this.observationId ||
+      !this.address ||
+      !this.busId
+    ) {
+      throw new Error(
+        "Accessibility element ID is missing or stale. Observe again before acting.",
+      );
+    }
+    try {
+      const current = (await this.run(
+        { operation: "resolve", busId: this.busId, target },
+        this.address,
+        signal,
+      )) as AccessibleElement;
+      signal.throwIfAborted();
+      const point = center(current.bounds);
+      if (point) {
+        return point;
+      }
+    } catch {
+      signal.throwIfAborted();
+    }
+    throw new Error(
+      "Accessibility element is unavailable or outside the screen. Observe again before acting.",
+    );
+  }
+}