@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/daemon/message-provenance.ts

@@ -8,14 +8,19 @@
  * actor filters the history down to rows whose provenance is itself
  * non-guardian.
  *
- * This lives in its own low-dependency module (types plus the zod-only
- * trust-class leaf) so both the conversation lifecycle (history load) and the
+ * This lives in its own low-dependency module (types plus the gateway-client
+ * trust-class contract) so both the conversation lifecycle (history load) and the
  * context compactor (image manifest) can apply the identical filter without
  * creating an import cycle through `conversation-lifecycle` ↔
  * `window-manager` ↔ `compactor`.
+ *
+ * It also owns the author field (`actorAuthorProvenance`), which says who
+ * wrote a row rather than whose turn wrote it.
  */
+import { type TrustClass, TrustClassSchema } from "@vellumai/gateway-client";
+
 import type { MessageRow } from "../persistence/conversation-crud.js";
-import { type TrustClass, trustClassSchema } from "../runtime/trust-class.js";
+import type { TrustContext } from "./trust-context-types.js";
 
 export function parseProvenanceTrustClass(
   metadata: string | null,
@@ -25,7 +30,7 @@ export function parseProvenanceTrustClass(
   }
   try {
     const parsed = JSON.parse(metadata) as { provenanceTrustClass?: unknown };
-    const result = trustClassSchema.safeParse(parsed?.provenanceTrustClass);
+    const result = TrustClassSchema.safeParse(parsed?.provenanceTrustClass);
     if (result.success) {
       return result.data;
     }
@@ -56,3 +61,22 @@ export function filterMessagesForUntrustedActor(
 ): MessageRow[] {
   return messages.filter((m) => isRowVisibleToUntrustedActor(m.metadata));
 }
+
+/** The persisted author field: the contact who wrote the row. */
+export interface ActorAuthorProvenance {
+  provenanceContactId?: string;
+}
+
+/**
+ * Author fields for a row the trust context's actor wrote themselves: their
+ * own message or reaction. `provenanceFromTrustContext` describes the turn and
+ * is also stamped on the assistant's replies, tool results, and notices, so
+ * those rows must never carry these.
+ */
+export function actorAuthorProvenance(
+  trustContext: TrustContext | undefined,
+): ActorAuthorProvenance {
+  return trustContext?.requesterContactId
+    ? { provenanceContactId: trustContext.requesterContactId }
+    : {};
+}

package/src/daemon/mode-session-startup-recovery.ts

@@ -0,0 +1,23 @@
+import { setModeSessionRecoveryHealthy } from "../config/session-groups-gate.js";
+import { recoverActiveConversationModeSessions } from "../persistence/conversation-mode-sessions.js";
+import { setDbReady } from "./daemon-readiness.js";
+
+export type ModeSessionStartupRecoveryResult =
+  | { ok: true; interruptedCount: number }
+  | { ok: false; error: unknown };
+
+/** Runs after migrations; auxiliary recovery failure disables tracking for this boot. */
+export function recoverModeSessionsBeforeDbReady(
+  recover: () => number = recoverActiveConversationModeSessions,
+): ModeSessionStartupRecoveryResult {
+  try {
+    const interruptedCount = recover();
+    setModeSessionRecoveryHealthy(true);
+    return { ok: true, interruptedCount };
+  } catch (error) {
+    setModeSessionRecoveryHealthy(false);
+    return { ok: false, error };
+  } finally {
+    setDbReady(true);
+  }
+}

package/src/daemon/mode-session-tracking.ts

@@ -0,0 +1,54 @@
+import type { ModeSession } from "../api/mode-session.js";
+import { getLogger } from "../util/logger.js";
+import type {
+  ConversationModeSessionCoordinator,
+  ModeSessionSourceHandle,
+} from "./conversation-mode-session.js";
+
+const log = getLogger("mode-session-tracking");
+
+/** Optional bookkeeping must not interrupt accepted actions or content. */
+export function bestEffortModeSessionTracking<T>(
+  operation: string,
+  track: () => T,
+): T | undefined {
+  try {
+    return track();
+  } catch (err) {
+    log.warn({ err, operation }, "Mode-session tracking failed");
+    return undefined;
+  }
+}
+
+/** Retire a refused source while already accepted turn output keeps its owner. */
+export function claimModeSessionTurn(
+  coordinator: Pick<
+    ConversationModeSessionCoordinator,
+    "claimTurn" | "retireSource"
+  >,
+  turnId: string,
+  handle: ModeSessionSourceHandle,
+  at: number,
+): ModeSession | undefined {
+  let owner: ModeSession | undefined;
+  try {
+    owner = coordinator.claimTurn(turnId, handle, at);
+  } catch (err) {
+    bestEffortModeSessionTracking("failed admission cleanup", () =>
+      coordinator.retireSource(handle, {
+        status: "interrupted",
+        endReason: "tracking_failed",
+      }),
+    );
+    throw err;
+  }
+  if (!owner || owner.id !== handle.id) {
+    bestEffortModeSessionTracking("refused admission cleanup", () =>
+      coordinator.retireSource(handle, {
+        status: "interrupted",
+        endReason: "tracking_refused",
+      }),
+    );
+  }
+  return owner;
+}

package/src/daemon/process-message.ts

@@ -67,12 +67,19 @@ import {
   preactivateHostProxySkills,
   shouldAttachHostProxyForCapability,
 } from "./host-proxy-preactivation.js";
+import { actorAuthorProvenance } from "./message-provenance.js";
 import type { SubagentToolGateMode } from "./tool-setup-types.js";
-import { restingTrust } from "./trust-context-types.js";
+import { restingTrust, type TrustContext } from "./trust-context-types.js";
 
 const log = getLogger("process-message");
 
 type ProcessMessageOptions = ConversationCreateOptions & {
+  /**
+   * The person whose own inbound message this turn persists, set only by
+   * channel ingress and its retry replay. Names the author on the persisted
+   * user row; see `PersistMessageOptions.author`.
+   */
+  author?: TrustContext;
   /** Per-turn observer for live agent-loop events. Does not replace SSE broadcast. */
   onEvent?: (msg: AssistantEvent) => void;
   /** IDs of user-uploaded attachments to resolve and include in the turn. */
@@ -539,7 +546,7 @@ export async function processMessage(
       ...(Object.keys(imageSourcePaths).length > 0 ? { imageSourcePaths } : {}),
     };
     const userMetaWithSlack = withChannelEnvelopes(
-      serverChannelMeta,
+      { ...serverChannelMeta, ...actorAuthorProvenance(options?.author) },
       slackMeta,
       providerMeta,
     );
@@ -638,7 +645,7 @@ export async function processMessage(
         : {}),
     };
     const compactUserMeta = withChannelEnvelopes(
-      compactChannelMeta,
+      { ...compactChannelMeta, ...actorAuthorProvenance(options?.author) },
       slackMeta,
       providerMeta,
     );
@@ -698,7 +705,7 @@ export async function processMessage(
         : {}),
     };
     const cleanUserMeta = withChannelEnvelopes(
-      cleanChannelMeta,
+      { ...cleanChannelMeta, ...actorAuthorProvenance(options?.author) },
       slackMeta,
       providerMeta,
     );
@@ -744,6 +751,7 @@ export async function processMessage(
       requestId,
       metadata: persistMetadata,
       displayContent: options?.displayContent,
+      ...(options?.author ? { author: options.author } : {}),
       ...(options?.skipUserMessageIndexing ? { skipIndexing: true } : {}),
       ...(ingressKey ? { clientMessageId: ingressKey } : {}),
     },
@@ -825,6 +833,7 @@ export async function processMessageInBackground(
       requestId,
       metadata: persistMetadata,
       displayContent: options?.displayContent,
+      ...(options?.author ? { author: options.author } : {}),
       ...(ingressKey ? { clientMessageId: ingressKey } : {}),
     },
   );

package/src/daemon/virtual-desktop-context.ts

@@ -0,0 +1,21 @@
+import type { ToolContext } from "../tools/types.js";
+import type { Conversation } from "./conversation.js";
+import { resolveTurnClientOs } from "./conversation-client-surface.js";
+import { FALLBACK_TURN_TRUST, resolveTrustClass } from "./trust-context.js";
+import { turnActorPrincipalId } from "./turn-actor.js";
+
+export function virtualDesktopContext(
+  ctx: Conversation,
+  signal?: AbortSignal,
+): ToolContext {
+  return {
+    workingDir: ctx.workingDir,
+    conversationId: ctx.conversationId,
+    trustClass: resolveTrustClass(
+      ctx.getTurnOrRestingTrust?.() ?? FALLBACK_TURN_TRUST,
+    ),
+    ...resolveTurnClientOs(ctx),
+    sourceActorPrincipalId: turnActorPrincipalId(ctx),
+    signal,
+  };
+}

package/src/desktop/__tests__/fake-desktop.ts

@@ -70,6 +70,7 @@ export function newFakeDesktop(options: FakeDesktopOptions) {
     which: (binary) =>
       options.missingBinaries?.includes(binary) ? null : `/usr/bin/${binary}`,
     probeVncPort: async () => vncReady,
+    probeSessionBus: async () => true,
     resolveChromePath: () => chromiumPath(),
     renderWallpaper: options.renderWallpaper ?? (async () => null),
     killProcessGroup: (child, signal) => {

package/src/desktop/desktop-accessibility-script.ts

@@ -0,0 +1,88 @@
+/** Runs inside the virtual desktop's Linux session, using the AT-SPI D-Bus API. */
+export const DESKTOP_ACCESSIBILITY_SCRIPT = String.raw`
+import json, os, sys, time
+import dbus
+
+ACCESSIBLE = "org.a11y.atspi.Accessible"
+COMPONENT = "org.a11y.atspi.Component"
+PROPERTIES = "org.freedesktop.DBus.Properties"
+ROOT = "/org/a11y/atspi/accessible/root"
+LIMIT = 300
+DEADLINE = time.monotonic() + 2
+
+def call(bus, ref, interface, method, *args):
+    remaining = DEADLINE - time.monotonic()
+    if remaining <= 0:
+        raise TimeoutError("Accessibility query exceeded its time limit")
+    obj = bus.get_object(str(ref[0]), str(ref[1]), introspect=False)
+    return obj.get_dbus_method(method, interface)(*args, timeout=min(0.2, remaining))
+
+def read(bus, ref, depth=0):
+    name = call(bus, ref, PROPERTIES, "Get", ACCESSIBLE, "Name")
+    child_count = call(bus, ref, PROPERTIES, "Get", ACCESSIBLE, "ChildCount")
+    role = str(call(bus, ref, ACCESSIBLE, "GetRoleName"))
+    states = call(bus, ref, ACCESSIBLE, "GetState")
+    bits = sum(int(word) << (32 * i) for i, word in enumerate(states))
+    showing = bool(bits & (1 << 25)) and bool(bits & (1 << 30))
+    bounds = None
+    if showing and not bits & (1 << 6):
+        try:
+            bounds = [int(n) for n in call(bus, ref, COMPONENT, "GetExtents", dbus.UInt32(0))]
+        except dbus.DBusException:
+            pass
+    return {
+        "bus": str(ref[0]), "path": str(ref[1]), "depth": depth,
+        "name": str(name)[:200], "role": role[:80],
+        "bounds": bounds,
+        "states": [label for bit, label in [(4, "checked"), (8, "enabled"), (12, "focused"), (23, "selected")] if bits & (1 << bit)],
+    }, int(child_count)
+
+def main():
+    request = json.loads(sys.argv[1])
+    session = dbus.bus.BusConnection(os.environ["DBUS_SESSION_BUS_ADDRESS"])
+    service = ("org.a11y.Bus", "/org/a11y/bus")
+    call(session, service, PROPERTIES, "Set", "org.a11y.Status", "IsEnabled", dbus.Boolean(True, variant_level=1))
+    address = str(call(session, service, "org.a11y.Bus", "GetAddress"))
+    bus = dbus.bus.BusConnection(address)
+    bus_id = str(call(bus, ("org.freedesktop.DBus", "/org/freedesktop/DBus"), "org.freedesktop.DBus", "GetId"))
+    if request["operation"] == "resolve":
+        if request["busId"] != bus_id:
+            raise ValueError("Accessibility session changed. Observe again.")
+        target = request["target"]
+        node, _ = read(bus, (target["bus"], target["path"]))
+        if node["name"] != target["name"] or node["role"] != target["role"] or not node["bounds"]:
+            raise ValueError("Accessibility element changed or is hidden. Observe again.")
+        print(json.dumps(node))
+        return
+    pending = [("org.a11y.atspi.Registry", ROOT, 0)]
+    seen, nodes = set(), []
+    truncated = False
+    while pending and len(seen) < LIMIT and time.monotonic() < DEADLINE:
+        name, path, depth = pending.pop()
+        ref = (name, path)
+        if ref in seen or path == "/org/a11y/atspi/null":
+            continue
+        seen.add(ref)
+        try:
+            node, count = read(bus, ref, depth)
+            if node["bounds"]:
+                nodes.append(node)
+            if depth < 20:
+                children = []
+                for i in range(min(count, LIMIT - len(seen))):
+                    child = call(bus, ref, ACCESSIBLE, "GetChildAtIndex", dbus.Int32(i))
+                    children.append((str(child[0]) or name, str(child[1]), depth + 1))
+                pending.extend(reversed(children))
+                truncated = truncated or count > len(children)
+            elif count:
+                truncated = True
+        except (dbus.DBusException, TimeoutError):
+            truncated = True
+    print(json.dumps({"busId": bus_id, "nodes": nodes, "truncated": truncated or bool(pending)}))
+
+try:
+    main()
+except Exception as error:
+    print(str(error), file=sys.stderr)
+    sys.exit(1)
+`;

package/src/desktop/desktop-accessibility.test.ts

@@ -0,0 +1,160 @@
+import { describe, expect, mock, test } from "bun:test";
+
+import { DesktopAccessibility } from "./desktop-accessibility.js";
+import {
+  performDesktopComputerUse,
+  planDesktopComputerUse,
+} from "./desktop-computer-use.js";
+
+const node = {
+  bus: ":1.2",
+  path: "/org/a11y/atspi/accessible/7",
+  name: "Results",
+  role: "scroll pane",
+  depth: 2,
+  states: ["enabled"],
+  bounds: [100, 200, 400, 300],
+};
+const signal = () => new AbortController().signal;
+
+describe("desktop accessibility", () => {
+  test("observations publish IDs and actions resolve current, clipped bounds", async () => {
+    const run = mock(async (request: object) =>
+      "target" in request
+        ? { ...node, bounds: [-20, 300, 200, 300] }
+        : { busId: "bus-1", nodes: [node], truncated: false },
+    );
+    const reader = new DesktopAccessibility(run);
+    expect(
+      (await reader.observe("unix:abstract=test", signal())).axTree,
+    ).toContain('[1] scroll pane "Results"');
+    reader.bindObservation("obs-1");
+    expect(await reader.resolve(1, "obs-1", signal())).toEqual({
+      x: 89,
+      y: 449,
+    });
+    expect(run.mock.calls[1][0]).toMatchObject({
+      operation: "resolve",
+      busId: "bus-1",
+      target: node,
+    });
+    await expect(
+      reader.resolve(1, "old-observation", signal()),
+    ).rejects.toThrow("stale");
+    await expect(reader.resolve(2, "obs-1", signal())).rejects.toThrow("stale");
+    expect(run).toHaveBeenCalledTimes(2);
+  });
+
+  test("an unavailable tree preserves screenshot fallback and invalidates old IDs", async () => {
+    const run = mock(async () => ({
+      busId: "bus-1",
+      nodes: [node],
+      truncated: false,
+    }));
+    const reader = new DesktopAccessibility(run);
+    await reader.observe("bus", signal());
+    reader.bindObservation("obs-1");
+    run.mockRejectedValue(new Error("bus disconnected"));
+    expect(await reader.observe("bus", signal())).toMatchObject({
+      userGuidance: expect.stringContaining("coordinates"),
+    });
+    await expect(reader.resolve(1, "obs-1", signal())).rejects.toThrow("stale");
+  });
+
+  test("cancellation is not hidden by screenshot fallback", async () => {
+    const reader = new DesktopAccessibility(async () => {
+      throw new Error("aborted");
+    });
+    const abort = new AbortController();
+    abort.abort(new Error("cancelled"));
+    await expect(reader.observe("bus", abort.signal)).rejects.toThrow(
+      "cancelled",
+    );
+  });
+
+  test("a hidden or removed target sends no input", async () => {
+    const reader = new DesktopAccessibility(async (request) => {
+      if ("target" in request) {
+        throw new Error("Object no longer exists");
+      }
+      return { busId: "bus-1", nodes: [node], truncated: false };
+    });
+    await reader.observe("bus", signal());
+    reader.bindObservation("obs-1");
+    const input = mock(async () => {});
+    const result = await performDesktopComputerUse(
+      planDesktopComputerUse("computer_use_scroll", {
+        element_id: 1,
+        direction: "down",
+        amount: 2,
+      }),
+      signal(),
+      {
+        input,
+        resolveElement: reader.resolve.bind(reader),
+        capture: async () => ({ screenshot: "image" }),
+      },
+      "obs-1",
+    );
+    expect(input).not.toHaveBeenCalled();
+    expect(result.executionError).toContain("Observe again");
+    expect(result.screenshot).toBe("image");
+  });
+
+  test("sequence IDs are resolved immediately before each step using the shared input driver", async () => {
+    let top = 200;
+    const reader = new DesktopAccessibility(async (request) =>
+      "target" in request
+        ? { ...node, bounds: [100, top, 400, 100] }
+        : { busId: "bus-1", nodes: [node], truncated: false },
+    );
+    await reader.observe("bus", signal());
+    reader.bindObservation("obs-1");
+    const input = mock(async (_args: string[], _signal?: AbortSignal) => {
+      top += 100;
+    });
+    const capture = mock(async () => ({ screenshot: "image" }));
+    await performDesktopComputerUse(
+      planDesktopComputerUse("computer_use_sequence", {
+        actions: [
+          { action: "click", element_id: 1 },
+          { action: "scroll", element_id: 1, direction: "down", amount: 2 },
+        ],
+      }),
+      signal(),
+      { input, capture, resolveElement: reader.resolve.bind(reader) },
+      "obs-1",
+    );
+    expect(input.mock.calls).toEqual([
+      [
+        [
+          "mousemove",
+          "299",
+          "249",
+          "click",
+          "--repeat",
+          "1",
+          "--delay",
+          "100",
+          "1",
+        ],
+        expect.any(AbortSignal),
+      ],
+      [
+        [
+          "mousemove",
+          "299",
+          "349",
+          "click",
+          "--repeat",
+          "2",
+          "--delay",
+          "50",
+          "5",
+        ],
+        expect.any(AbortSignal),
+      ],
+    ]);
+    expect(capture).toHaveBeenCalledTimes(1);
+  });
+});

package/src/desktop/desktop-accessibility.ts

@@ -0,0 +1,169 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+import { DESKTOP_ACCESSIBILITY_SCRIPT } from "./desktop-accessibility-script.js";
+import {
+  DESKTOP_DISPLAY,
+  DESKTOP_HEIGHT,
+  DESKTOP_WIDTH,
+} from "./desktop-display.js";
+
+interface AccessibleElement {
+  bus: string;
+  path: string;
+  name: string;
+  role: string;
+  depth: number;
+  states: string[];
+  bounds: [number, number, number, number];
+}
+interface Snapshot {
+  busId: string;
+  nodes: AccessibleElement[];
+  truncated: boolean;
+}
+type Query = (
+  request: object,
+  address: string,
+  signal: AbortSignal,
+) => Promise<unknown>;
+const exec = promisify(execFile);
+const query: Query = async (request, address, signal) => {
+  const { stdout } = await exec(
+    "/usr/bin/python3",
+    ["-c", DESKTOP_ACCESSIBILITY_SCRIPT, JSON.stringify(request)],
+    {
+      env: {
+        PATH: process.env.PATH,
+        LANG: "C.UTF-8",
+        DISPLAY: DESKTOP_DISPLAY,
+        DBUS_SESSION_BUS_ADDRESS: address,
+      },
+      signal,
+      timeout: 3_000,
+      maxBuffer: 512 * 1024,
+      windowsHide: true,
+    },
+  );
+  return JSON.parse(stdout);
+};
+
+function center(
+  bounds: AccessibleElement["bounds"],
+): { x: number; y: number } | undefined {
+  const [x, y, width, height] = bounds;
+  if (
+    ![x, y, width, height].every(Number.isFinite) ||
+    width <= 0 ||
+    height <= 0
+  ) {
+    return undefined;
+  }
+  const left = Math.max(0, x);
+  const top = Math.max(0, y);
+  const right = Math.min(DESKTOP_WIDTH, x + width);
+  const bottom = Math.min(DESKTOP_HEIGHT, y + height);
+  if (right <= left || bottom <= top) {
+    return undefined;
+  }
+  return {
+    x: Math.floor((left + right - 1) / 2),
+    y: Math.floor((top + bottom - 1) / 2),
+  };
+}
+
+export class DesktopAccessibility {
+  private elements = new Map<number, AccessibleElement>();
+  private observationId?: string;
+  private busId?: string;
+  private address?: string;
+
+  constructor(private readonly run: Query = query) {}
+
+  async observe(
+    address: string,
+    signal: AbortSignal,
+  ): Promise<{ axTree?: string; userGuidance?: string }> {
+    signal.throwIfAborted();
+    this.elements.clear();
+    this.busId = undefined;
+    this.observationId = undefined;
+    this.address = address;
+    try {
+      const snapshot = (await this.run(
+        { operation: "observe" },
+        address,
+        signal,
+      )) as Snapshot;
+      signal.throwIfAborted();
+      this.busId = snapshot.busId;
+      const lines: string[] = [];
+      for (const node of snapshot.nodes) {
+        if (!center(node.bounds)) {
+          continue;
+        }
+        const id = this.elements.size + 1;
+        this.elements.set(id, node);
+        lines.push(
+          `${"  ".repeat(Math.min(node.depth, 20))}[${id}] ${node.role} ${JSON.stringify(node.name)} ${node.states.join(" ")}`.trimEnd(),
+        );
+      }
+      return {
+        axTree: lines.length ? lines.join("\n") : undefined,
+        ...(!lines.length || snapshot.truncated
+          ? {
+              userGuidance:
+                "Accessibility information is incomplete. Use screenshot coordinates for elements not listed.",
+            }
+          : {}),
+      };
+    } catch {
+      signal.throwIfAborted();
+      this.elements.clear();
+      return {
+        userGuidance:
+          "Accessibility is unavailable. Use screenshot coordinates for this observation.",
+      };
+    }
+  }
+
+  bindObservation(id: string): void {
+    this.observationId = id;
+  }
+
+  async resolve(
+    id: number,
+    observationId: string,
+    signal: AbortSignal,
+  ): Promise<{ x: number; y: number }> {
+    signal.throwIfAborted();
+    const target = this.elements.get(id);
+    if (
+      !target ||
+      observationId !== this.observationId ||
+      !this.address ||
+      !this.busId
+    ) {
+      throw new Error(
+        "Accessibility element ID is missing or stale. Observe again before acting.",
+      );
+    }
+    try {
+      const current = (await this.run(
+        { operation: "resolve", busId: this.busId, target },
+        this.address,
+        signal,
+      )) as AccessibleElement;
+      signal.throwIfAborted();
+      const point = center(current.bounds);
+      if (point) {
+        return point;
+      }
+    } catch {
+      signal.throwIfAborted();
+    }
+    throw new Error(
+      "Accessibility element is unavailable or outside the screen. Observe again before acting.",
+    );
+  }
+}