@vellumai/assistant 0.12.4-dev.202609241715.2cfa60b → 0.12.4-dev.202609241859.ddab752

package/docs/desktop-browser-cli.md

@@ -53,8 +53,9 @@ Non-platform assistants keep their connected-computer behavior.
 Explicit `target: "connected-computer"` or `target_client_id` selects a connected
 computer on every client. The selected target never falls back to another
 computer when unavailable. The virtual desktop uses a fixed 1440x810 display.
-Observations return full-screen screenshots and mouse actions use the same
-screen coordinates.
+Observations return full-screen screenshots and mouse actions use screen
+points. Screenshot metadata uses the transport-optimized image dimensions and
+provides per-axis conversion from image pixels to those screen points.
 Viewer resizing scales the canvas locally without changing the desktop layout.
 
 Observe first and pass the returned `observation_id` with each native action.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.12.4-dev.202609241715.2cfa60b",
+  "version": "0.12.4-dev.202609241859.ddab752",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/host-cu-proxy.test.ts

@@ -1186,6 +1186,85 @@ describe("HostCuProxy", () => {
 
       expect(result.content).toContain("2560x1440 px");
       expect(result.content).toContain("1280x720 pt");
+      expect(result.content).toContain("x = round(image_x * 0.500000)");
+      expect(result.content).toContain("y = round(image_y * 0.500000)");
+    });
+
+    test("derives separate screen-point scales for a downscaled screenshot", () => {
+      setup();
+
+      const result = proxy.formatObservation({
+        screenshot: "base64data",
+        screenshotWidthPx: 835,
+        screenshotHeightPx: 540,
+        screenWidthPt: 1728,
+        screenHeightPt: 1117,
+      });
+
+      const xScale = Number(result.content.match(/image_x \* ([\d.]+)/)?.[1]);
+      const yScale = Number(result.content.match(/image_y \* ([\d.]+)/)?.[1]);
+      expect(Math.round(675 * xScale)).toBe(1397);
+      expect(Math.round(330 * yScale)).toBe(683);
+      expect(result.content).toContain("screen points, not screenshot pixels");
+    });
+
+    test.each([
+      { capture_window_id: 12 },
+      { captureWindowId: 12 },
+      { captureDisplayId: 2 },
+    ])(
+      "does not scale a scoped screenshot by desktop dimensions: %j",
+      (input) => {
+        setup();
+
+        const result = proxy.formatObservation(
+          {
+            screenshot: "base64data",
+            screenshotWidthPx: 800,
+            screenshotHeightPx: 600,
+            screenWidthPt: 1600,
+            screenHeightPt: 1200,
+          },
+          undefined,
+          false,
+          { toolName: "computer_use_observe", input },
+        );
+
+        expect(result.content).toContain("screenshot is capture-relative");
+        expect(result.content).not.toContain("round(image_");
+      },
+    );
+
+    test.each([undefined, 0, -1, Number.NaN, Number.POSITIVE_INFINITY])(
+      "does not invent a mapping for an unavailable or invalid dimension: %s",
+      (screenWidthPt) => {
+        setup();
+
+        const result = proxy.formatObservation({
+          screenshot: "base64data",
+          screenshotWidthPx: 800,
+          screenshotHeightPx: 600,
+          screenWidthPt,
+          screenHeightPt: 1200,
+        });
+
+        expect(result.content).toContain("coordinate mapping is unavailable");
+        expect(result.content).not.toContain("round(image_");
+      },
+    );
+
+    test("omits coordinate guidance when no screenshot accompanies dimensions", () => {
+      setup();
+
+      const result = proxy.formatObservation({
+        axTree: "Button [1]",
+        screenshotWidthPx: 800,
+        screenshotHeightPx: 600,
+        screenWidthPt: 1600,
+        screenHeightPt: 1200,
+      });
+
+      expect(result.content).not.toContain("coordinate");
     });
 
     test("escapes </ax-tree> in AX tree content", () => {

package/src/__tests__/host-cu-routes-targeted.test.ts

@@ -17,6 +17,8 @@
  */
 import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
 
+import sharp from "sharp";
+
 // ── Module mocks ─────────────────────────────────────────────────────────────
 // Must be registered before the host-cu-routes module is loaded.
 
@@ -487,7 +489,14 @@ describe("handleHostCuResult: request body validation", () => {
     expect(pendingStore.has(requestId)).toBe(true);
   });
 
-  test("forwards a full observation payload unchanged", async () => {
+  test("forwards observation metadata with dimensions from the screenshot bytes", async () => {
+    const screenshot = (
+      await sharp({
+        create: { width: 835, height: 540, channels: 3, background: "white" },
+      })
+        .jpeg()
+        .toBuffer()
+    ).toString("base64");
     const requestId = "req-cu-full-observation";
     registerPending(requestId);
 
@@ -496,7 +505,7 @@ describe("handleHostCuResult: request body validation", () => {
         requestId,
         axTree: "Button [1]",
         axDiff: "+Button [1]",
-        screenshot: "base64png",
+        screenshot,
         screenshotWidthPx: 1600,
         screenshotHeightPx: 1000,
         screenWidthPt: 800,
@@ -512,9 +521,9 @@ describe("handleHostCuResult: request body validation", () => {
     expect(cuResolveSpy[0].payload).toMatchObject({
       axTree: "Button [1]",
       axDiff: "+Button [1]",
-      screenshot: "base64png",
-      screenshotWidthPx: 1600,
-      screenshotHeightPx: 1000,
+      screenshot,
+      screenshotWidthPx: 835,
+      screenshotHeightPx: 540,
       screenWidthPt: 800,
       screenHeightPt: 500,
       executionResult: "Clicked",

package/src/config/bundled-skills/computer-use/SKILL.md

@@ -37,6 +37,26 @@ The tree is walked to a limited depth to keep steps fast, and says when it was
 cut off. If the element you need is not in it, call `computer_use_observe` with
 `full_tree: true`.
 
+## Coordinates and live editing
+
+Prefer element IDs from the latest observation. Click, scroll and drag
+coordinates are **screen points**, not screenshot pixels. For a desktop
+screenshot, use the per-axis conversion in its observation metadata. If you
+crop or resize the image, map back to the original screenshot first. Observe
+again after the layout changes; an older image is not a current target.
+
+During live editing, use the app's zoom and track-height controls, then inspect
+the fresh screenshot. Avoid installing image libraries or repeatedly cropping
+and measuring screenshot pixels just to locate a UI control or waveform.
+Reserve scripted image or media analysis for tasks that actually require it.
+If an action does not produce the expected visible result, check focus and the
+coordinate mapping before repeating it.
+
+Waveform pixels do not reveal spoken words. If an edit depends on audio you
+cannot hear or a boundary you cannot see, ask for a timestamp, a user-positioned
+playhead, or the source media. Do not guess the cut point or claim an edit
+succeeded without observing its result.
+
 ## Scripting apps (macOS)
 
 Reach for `computer_use_run_applescript` first when an app can be driven by

package/src/config/bundled-skills/computer-use/TOOLS.json

@@ -21,7 +21,7 @@
           },
           "full_tree": {
             "type": "boolean",
-            "description": "On connected personal computers, walk the accessibility tree to full depth. The virtual desktop uses a bounded tree; use screenshot coordinates when an element is not listed."
+            "description": "On connected personal computers, walk the accessibility tree to full depth. The virtual desktop uses a bounded tree; use coordinates converted from the screenshot metadata when an element is not listed."
           },
           "target_client_id": {
             "type": "string",
@@ -57,11 +57,11 @@
           },
           "x": {
             "type": "integer",
-            "description": "X coordinate on screen (fallback when no element_id)"
+            "description": "X in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
           },
           "y": {
             "type": "integer",
-            "description": "Y coordinate on screen (fallback when no element_id)"
+            "description": "Y in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
           },
           "reasoning": {
             "type": "string",
@@ -170,11 +170,11 @@
           },
           "x": {
             "type": "integer",
-            "description": "X coordinate on screen (fallback when no element_id)"
+            "description": "X in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
           },
           "y": {
             "type": "integer",
-            "description": "Y coordinate on screen (fallback when no element_id)"
+            "description": "Y in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
           },
           "direction": {
             "type": "string",
@@ -222,11 +222,11 @@
           },
           "x": {
             "type": "integer",
-            "description": "Source X coordinate (fallback when no element_id)"
+            "description": "Source X in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
           },
           "y": {
             "type": "integer",
-            "description": "Source Y coordinate (fallback when no element_id)"
+            "description": "Source Y in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
           },
           "to_element_id": {
             "type": "integer",
@@ -234,11 +234,11 @@
           },
           "to_x": {
             "type": "integer",
-            "description": "Destination X coordinate (fallback when no to_element_id)"
+            "description": "Destination X in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no to_element_id)."
           },
           "to_y": {
             "type": "integer",
-            "description": "Destination Y coordinate (fallback when no to_element_id)"
+            "description": "Destination Y in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no to_element_id)."
           },
           "reasoning": {
             "type": "string",
@@ -408,11 +408,11 @@
                 },
                 "x": {
                   "type": "integer",
-                  "description": "click, double_click, right_click, scroll: X coordinate on screen (fallback when no element_id)"
+                  "description": "click, double_click, right_click, scroll: X in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
                 },
                 "y": {
                   "type": "integer",
-                  "description": "click, double_click, right_click, scroll: Y coordinate on screen (fallback when no element_id)"
+                  "description": "click, double_click, right_click, scroll: Y in screen points, not screenshot pixels. Apply the latest observation's coordinate mapping (fallback when no element_id)."
                 },
                 "direction": {
                   "type": "string",

package/src/daemon/computer-use-observation.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, test } from "bun:test";
+
+import sharp from "sharp";
+
+import { parseImageDimensions } from "../context/image-dimensions.js";
+import { prepareComputerUseObservation } from "./computer-use-observation.js";
+import { HostCuProxy } from "./host-cu-proxy.js";
+
+async function screenshot(width: number, height: number): Promise<string> {
+  const bytes = await sharp({
+    create: { width, height, channels: 3, background: "white" },
+  })
+    .jpeg()
+    .toBuffer();
+  return bytes.toString("base64");
+}
+
+describe("computer-use observation transport", () => {
+  test("derives metadata from emitted screenshot bytes", async () => {
+    const image = await screenshot(835, 540);
+    const observation = await prepareComputerUseObservation({
+      screenshot: image,
+      screenshotWidthPx: 1728,
+      screenshotHeightPx: 1117,
+      screenWidthPt: 1728,
+      screenHeightPt: 1117,
+    });
+
+    expect(observation.screenshot).toBe(image);
+    expect(observation.screenshotWidthPx).toBe(835);
+    expect(observation.screenshotHeightPx).toBe(540);
+  });
+
+  test("formats the optimized dimensions on the virtual desktop path", async () => {
+    const image = await screenshot(3200, 1800);
+    const proxy = new HostCuProxy();
+    const result = await proxy.executeLocal(
+      "computer_use_observe",
+      {},
+      async () => ({
+        screenshot: image,
+        screenshotWidthPx: 3200,
+        screenshotHeightPx: 1800,
+        screenWidthPt: 1600,
+        screenHeightPt: 900,
+      }),
+    );
+
+    const block = result.contentBlocks?.[0];
+    expect(block?.type).toBe("image");
+    if (block?.type !== "image" || block.source.type !== "base64") {
+      throw new Error("Expected screenshot bytes");
+    }
+    const dimensions = parseImageDimensions(block.source);
+    expect(dimensions).toEqual({ width: 1568, height: 882 });
+    const repeated = await prepareComputerUseObservation({
+      screenshot: block.source.data,
+    });
+    expect(repeated.screenshot).toBe(block.source.data);
+    expect(result.content).toContain("1568x882 px");
+    expect(result.content).toContain("x = round(image_x * 1.020408)");
+    expect(result.content).toContain("y = round(image_y * 1.020408)");
+    proxy.dispose();
+  });
+
+  test("does not trust dimension metadata when image bytes cannot be read", async () => {
+    const observation = await prepareComputerUseObservation({
+      screenshot: "invalid",
+      screenshotWidthPx: 800,
+      screenshotHeightPx: 600,
+    });
+
+    expect(observation.screenshotWidthPx).toBeUndefined();
+    expect(observation.screenshotHeightPx).toBeUndefined();
+  });
+
+  test("leaves observations without a screenshot alone", async () => {
+    const observation = { axTree: "[1] AXButton Save" };
+    expect(await prepareComputerUseObservation(observation)).toBe(observation);
+  });
+});

package/src/daemon/computer-use-observation.ts

@@ -0,0 +1,23 @@
+import { optimizeImageForTransport } from "../agent/image-optimize.js";
+import { parseImageDimensions } from "../context/image-dimensions.js";
+import type { CuObservationResult } from "./host-cu-proxy.js";
+
+/** Keep coordinate metadata aligned with the image sent to the model. */
+export async function prepareComputerUseObservation(
+  observation: CuObservationResult,
+): Promise<CuObservationResult> {
+  if (!observation.screenshot) {
+    return observation;
+  }
+  const image = await optimizeImageForTransport(
+    observation.screenshot,
+    "image/jpeg",
+  );
+  const dimensions = parseImageDimensions(image.data, image.mediaType);
+  return {
+    ...observation,
+    screenshot: image.data,
+    screenshotWidthPx: dimensions?.width,
+    screenshotHeightPx: dimensions?.height,
+  };
+}

package/src/daemon/host-cu-proxy.ts

@@ -28,6 +28,7 @@ import { POINT_AT_PROXY_TOOL } from "../tools/computer-use/skill-proxy-bridge.js
 import type { ToolExecutionResult } from "../tools/types.js";
 import { AssistantError, ErrorCode } from "../util/errors.js";
 import { getLogger } from "../util/logger.js";
+import { prepareComputerUseObservation } from "./computer-use-observation.js";
 import { resolveHostCuTarget } from "./host-cu-target.js";
 import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
 
@@ -615,7 +616,7 @@ export class HostCuProxy {
     this.recordAction(toolName, input, reasoning);
     this._previousAXTree = undefined;
     this._consecutiveUnchangedSteps = 0;
-    const observation = await execute();
+    const observation = await prepareComputerUseObservation(await execute());
     return this.formatObservation(observation, undefined, false, {
       toolName,
       input,
@@ -791,7 +792,7 @@ export class HostCuProxy {
       );
     }
 
-    const screenshotMeta = this.formatScreenshotMetadata(obs);
+    const screenshotMeta = this.formatScreenshotMetadata(obs, action);
     if (screenshotMeta.length > 0) {
       parts.push("");
       parts.push(...screenshotMeta);
@@ -898,7 +899,10 @@ export class HostCuProxy {
     }
   }
 
-  private formatScreenshotMetadata(obs: CuObservationResult): string[] {
+  private formatScreenshotMetadata(
+    obs: CuObservationResult,
+    action: ActionIdentity | undefined,
+  ): string[] {
     if (!obs.screenshot) {
       return [];
     }
@@ -914,6 +918,33 @@ export class HostCuProxy {
         `Screen metadata: ${obs.screenWidthPt}x${obs.screenHeightPt} pt`,
       );
     }
+    if (action && hasCaptureTarget(action.input)) {
+      lines.push(
+        "This screenshot is capture-relative. Full-screen dimensions do not map its pixels to action coordinates. Use accessibility element IDs, or request an unscoped desktop observation before using coordinates.",
+      );
+    } else if (
+      [
+        obs.screenshotWidthPx,
+        obs.screenshotHeightPx,
+        obs.screenWidthPt,
+        obs.screenHeightPt,
+      ].every(
+        (dimension) =>
+          typeof dimension === "number" &&
+          Number.isFinite(dimension) &&
+          dimension > 0,
+      )
+    ) {
+      const scaleX = obs.screenWidthPt! / obs.screenshotWidthPx!;
+      const scaleY = obs.screenHeightPt! / obs.screenshotHeightPx!;
+      lines.push(
+        `Action coordinates are screen points, not screenshot pixels. Convert this image's pixels: x = round(image_x * ${scaleX.toFixed(6)}), y = round(image_y * ${scaleY.toFixed(6)}). Prefer accessibility element IDs when available.`,
+      );
+    } else {
+      lines.push(
+        "Screenshot-to-screen coordinate mapping is unavailable. Use accessibility element IDs or obtain an observation with both screenshot and screen dimensions before using coordinates.",
+      );
+    }
     return lines;
   }
 }

package/src/desktop/desktop-computer-use.test.ts

@@ -1,5 +1,7 @@
 import { describe, expect, mock, spyOn, test } from "bun:test";
 
+import sharp from "sharp";
+
 import { run as click } from "../config/bundled-skills/computer-use/tools/computer-use-click.js";
 import * as configLoader from "../config/loader.js";
 import { HostCuProxy } from "../daemon/host-cu-proxy.js";
@@ -14,12 +16,20 @@ import {
 } from "./desktop-computer-use.js";
 import * as sessionManager from "./desktop-session-manager.js";
 
+const SCREENSHOT = (
+  await sharp({
+    create: { width: 1440, height: 810, channels: 3, background: "white" },
+  })
+    .jpeg()
+    .toBuffer()
+).toString("base64");
+
 function driver(userGuidance?: string) {
   return {
     input: mock(async (_args: string[], _signal?: AbortSignal) => {}),
     capture: mock(async (_signal: AbortSignal) => ({
       userGuidance,
-      screenshot: "anBlZw==",
+      screenshot: SCREENSHOT,
       screenshotWidthPx: 1440,
       screenshotHeightPx: 810,
       screenWidthPt: 1440,
@@ -52,7 +62,7 @@ describe("virtual desktop computer use", () => {
         signal(),
         backend,
       );
-      expect(result.screenshot).toBe("anBlZw==");
+      expect(result.screenshot).toBe(SCREENSHOT);
     }
     expect(backend.input).not.toHaveBeenCalled();
     expect(backend.capture).toHaveBeenCalledTimes(2);
@@ -161,7 +171,7 @@ describe("virtual desktop computer use", () => {
     );
     expect(backend.input).toHaveBeenCalledTimes(1);
     expect(result.executionError).toBe("Input failed");
-    expect(result.screenshot).toBe("anBlZw==");
+    expect(result.screenshot).toBe(SCREENSHOT);
   });
 
   test("done releases only its conversation without starting the desktop", async () => {
@@ -316,7 +326,7 @@ describe("virtual desktop computer use", () => {
         source: {
           type: "base64",
           media_type: "image/jpeg",
-          data: "anBlZw==",
+          data: SCREENSHOT,
         },
       },
     ]);

package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts

@@ -1771,6 +1771,89 @@ describe("LiveVoiceSession Flux end-of-turn during the STT dial", () => {
     await session.close("client_end");
   }, 10_000);
 
+  test("rearms an empty fallback cycle while the previous reply is active", async () => {
+    const abort = mock();
+    const { frames, session, transcribers, turnCalls } = createHarness({
+      fluxConfig: FLUX_ON,
+      startVoiceTurn: async (turnOptions) => {
+        turnOptions.callbacks?.assistant_text_delta?.(
+          makeTextDelta("Working on the edit."),
+        );
+        return { turnId: "editing-turn", abort };
+      },
+    });
+
+    try {
+      await session.start();
+      await waitFor(() => transcribers.length === 1);
+      const transcriber = transcribers[0]!;
+      await session.handleBinaryAudio(LOUD_CHUNK);
+      transcriber.startOfTurn(0);
+      transcriber.endOfTurn("edit the clip", 0);
+      await waitFor(() => turnCalls.length === 1);
+
+      const speechStarts = countFrames(frames, "speech_started");
+      await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
+      // No provider speech start or transcript arrives for this local onset.
+      await waitFor(() => transcriber.stopped);
+      await waitFor(
+        () => transcribers.length >= 2,
+        "An empty fallback cycle blocked the replacement stream behind the reply",
+      );
+      expect(countFrames(frames, "utterance_discarded")).toBe(1);
+      expect(countFrames(frames, "speech_started")).toBe(speechStarts);
+      expect(countFrames(frames, "turn_cancelled")).toBe(0);
+      expect(abort).not.toHaveBeenCalled();
+      expect(turnCalls).toHaveLength(1);
+
+      const replacement = transcribers.at(-1)!;
+      await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
+      expect(replacement.received.length).toBeGreaterThan(0);
+      replacement.startOfTurn(0);
+      await waitFor(() => countFrames(frames, "turn_cancelled") === 1);
+      expect(countFrames(frames, "speech_started")).toBe(speechStarts + 1);
+      expect(abort).toHaveBeenCalledTimes(1);
+
+      replacement.endOfTurn("stop editing", 0);
+      await waitFor(() => turnCalls.length === 2);
+      expect(turnCalls[1]?.content).toBe("stop editing");
+    } finally {
+      await session.close("client_end");
+    }
+  }, 10_000);
+
+  test("keeps a nonempty fallback transcript queued behind the previous reply", async () => {
+    const abort = mock();
+    const { frames, session, transcribers, turnCalls } = createHarness({
+      fluxConfig: FLUX_ON,
+      startVoiceTurn: async () => ({ turnId: "editing-turn", abort }),
+    });
+
+    try {
+      await session.start();
+      await waitFor(() => transcribers.length === 1);
+      const transcriber = transcribers[0]!;
+      await session.handleBinaryAudio(LOUD_CHUNK);
+      transcriber.startOfTurn(0);
+      transcriber.endOfTurn("edit the clip", 0);
+      await waitFor(() => turnCalls.length === 1);
+
+      transcriber.pendingFlushText = "show me the result";
+      await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
+      await waitFor(() => transcriber.stopped);
+      await flushAsyncCallbacks();
+      expect(turnCalls).toHaveLength(1);
+      expect(countFrames(frames, "utterance_discarded")).toBe(0);
+      expect(abort).not.toHaveBeenCalled();
+
+      turnCalls[0]?.callbacks?.message_complete?.(makeMessageComplete());
+      await waitFor(() => turnCalls.length === 2);
+      expect(turnCalls[1]?.content).toBe("show me the result");
+    } finally {
+      await session.close("client_end");
+    }
+  }, 10_000);
+
   test("routes a closing stream's flush into the turn it released", async () => {
     const { session, transcribers, turnCalls } = createHarness({
       fluxConfig: FLUX_ON,

package/src/live-voice/live-voice-session.ts

@@ -5945,11 +5945,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     ) {
       return;
     }
-    // One assistant turn at a time: a server_vad utterance that closes while
-    // the previous turn is still speaking waits; rearmAfterTurn retries it.
-    if (this.activeAssistantTurn) {
-      return;
-    }
     if (utterance.phase !== "transcriber_closed") {
       return;
     }
@@ -5972,6 +5967,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       return;
     }
 
+    // Empty cycles must rearm input even during a reply so provider speech
+    // starts can still interrupt it. Only a real transcript waits its turn.
+    if (this.activeAssistantTurn) {
+      return;
+    }
+
     await this.launchAssistantTurn(utterance, content);
   }
 

package/src/runtime/routes/host-cu-routes.ts

@@ -6,6 +6,7 @@
  */
 import { z } from "zod";
 
+import { prepareComputerUseObservation } from "../../daemon/computer-use-observation.js";
 import { findConversation } from "../../daemon/conversation-registry.js";
 import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
 import { SAME_ACTOR_FORBIDDEN_DESCRIPTION } from "../auth/same-actor.js";
@@ -112,7 +113,7 @@ async function handleHostCuResult({ body, headers }: RouteHandlerArgs) {
     throw new NotFoundError("No host CU proxy for conversation");
   }
 
-  conversation.hostCuProxy.processObservation(requestId, {
+  const observation = await prepareComputerUseObservation({
     axTree,
     axDiff,
     screenshot,
@@ -126,6 +127,7 @@ async function handleHostCuResult({ body, headers }: RouteHandlerArgs) {
     userGuidance,
     timings,
   });
+  conversation.hostCuProxy.processObservation(requestId, observation);
 
   return { accepted: true };
 }