@vellumai/assistant 0.12.2-dev.202609211820.22f71b0 → 0.12.2-dev.202609212016.fb4c53e

package/Dockerfile

@@ -101,6 +101,69 @@ RUN apt-get update && apt-get install -y \
     xdg-utils \
     && rm -rf /var/lib/apt/lists/*
 
+# Desktop system packages persist through container replacement as image layers.
+RUN set -eu; \
+    wezterm_version=20240203-110809-5046fc22; \
+    case "$(dpkg --print-architecture)" in \
+        amd64) wezterm_suffix=Debian12.deb; wezterm_sha256=d3a5c97093fbc0a87e8f9616e44efc4c9503cfc495fd1cfe4ffeff88578d15f8 ;; \
+        arm64) wezterm_suffix=Debian12.arm64.deb; wezterm_sha256=351f8791c6e9561d687afa63834a2132de84800dad6e7c3d035cc6518d6c3744 ;; \
+        *) echo "Unsupported desktop architecture" >&2; exit 1 ;; \
+    esac; \
+    curl --fail --location --proto '=https' --proto-redir '=https' \
+        --retry 2 --max-time 300 --output /tmp/wezterm.deb \
+        "https://github.com/wezterm/wezterm/releases/download/${wezterm_version}/wezterm-${wezterm_version}.${wezterm_suffix}"; \
+    printf '%s  /tmp/wezterm.deb\n' "$wezterm_sha256" | sha256sum --check -; \
+    apt-get update; \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        adwaita-icon-theme \
+        at-spi2-core \
+        bamfdaemon \
+        dbus-daemon \
+        dbus-x11 \
+        feh \
+        fonts-liberation \
+        gnome-mines \
+        gvfs \
+        libcurl4 \
+        libegl1 \
+        libgl1-mesa-dri \
+        libgtk-3-0 \
+        libvulkan1 \
+        openbox \
+        plank \
+        python3 \
+        python3-dbus \
+        scrot \
+        thunar \
+        tigervnc-common \
+        tigervnc-standalone-server \
+        tumbler \
+        xauth \
+        xcompmgr \
+        xdotool \
+        xfonts-base \
+        xterm \
+        /tmp/wezterm.deb; \
+    wezterm --version; \
+    rm -rf /tmp/wezterm.deb /var/lib/apt/lists/* /var/cache/apt/archives/*.deb
+
+# Extract Chrome without running package scripts or registering its repository.
+RUN set -eu; \
+    chrome_version=153.0.8010.36-1; \
+    chrome_arch="$(dpkg --print-architecture)"; \
+    case "$chrome_arch" in \
+        amd64) chrome_sha256=9bb44e33031c2f2857cf36b4343051a12f93058e4b781e3c76313df87f6c8d32 ;; \
+        arm64) chrome_sha256=1fcf6ec51a9d52e26ff1ad807225f725be55069ad2b68c5d69361a2746665188 ;; \
+        *) echo "Unsupported Chrome architecture" >&2; exit 1 ;; \
+    esac; \
+    curl --fail --location --proto '=https' --proto-redir '=https' \
+        --retry 2 --max-time 300 --output /tmp/chrome.deb \
+        "https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${chrome_version}_${chrome_arch}.deb"; \
+    printf '%s  /tmp/chrome.deb\n' "$chrome_sha256" | sha256sum --check -; \
+    dpkg-deb --extract /tmp/chrome.deb /; \
+    /opt/google/chrome/chrome --version; \
+    rm /tmp/chrome.deb
+
 # Copy bun binary from builder instead of re-installing
 COPY --from=builder /root/.bun/bin/bun /usr/local/bin/bun
 RUN ln -sf /usr/local/bin/bun /usr/local/bin/bunx

package/docs/desktop-browser-cli.md

@@ -1,6 +1,8 @@
 # Virtual desktop browser CLI
 
-`assistant browser --virtual-desktop` uses the browser CLI's shared operation handlers against the Chrome process owned by `DesktopSessionManager`. It requires a platform-hosted container, the `assistant-desktop` flag, completed automatic desktop installation and an identified guardian conversation. Availability is checked before starting or reusing control and after asynchronous startup; loss of the flag or installation readiness cancels active control. The first eligible browser command waits for the shared installation job, starts Chrome, and executes the original action exactly once. Cancellation and feature revocation stop the pending action while shared installation can finish for other viewers. Browser CLI requests allow ten minutes, with an eight-minute setup wait limit. Eligible web conversations receive browser guidance in the bash tool description, including the configured shell timeout; native clients retain their existing tool description. Opening the picture-in-picture panel also starts setup and shows progress automatically; failed installs retain a retry action. The persistent profile remains `data/desktop-profile`; no data migration or extension installation is required.
+`assistant browser --virtual-desktop` uses the browser CLI's shared operation handlers against the Chrome process owned by `DesktopSessionManager`. It requires a platform-hosted container, the `assistant-desktop` flag, image-provided desktop components and an identified guardian conversation. Availability is checked before starting or reusing control and after asynchronous startup; losing readiness or the feature flag cancels active control. The first eligible browser command starts Chrome and executes its action without downloading or installing dependencies. Opening the picture-in-picture panel checks readiness and starts the viewer directly. The persistent profile remains `data/desktop-profile`.
+
+The assistant Dockerfile installs desktop system packages, pinned WezTerm and pinned Google Chrome before Bun and application source. Architecture-specific SHA-256 checksums verify both downloads. Chrome is extracted to `/opt/google/chrome` without running package scripts or registering its repository. Missing components require replacing the assistant image. Runtime code only checks readiness. `POST /v1/desktop/setup` remains a read-only compatibility endpoint for released clients; current clients only use GET.
 
 Automatic browser selection in platform-hosted web conversations uses the streamed Chrome when the desktop flag is enabled and the actor is an identified guardian. The macOS and Windows apps keep their existing browser selection and fallback behavior; they use the streamed desktop only with explicit `--virtual-desktop`. Their shared renderer reports a `web` transport, so selection also reads the active turn's frozen `clientOs`; it does not change transport identity or host capability checks. Explicit `--virtual-desktop`, `--browser-mode`, `--target-client-id` and active-tab requests override the default. Existing personal-browser sessions and tab pins remain on their selected browser. Tab commands use the same streamed-browser default as page commands. `--browser-mode local` selects assistant-side Playwright, not the user's Chrome.
 
@@ -16,7 +18,7 @@ flowchart LR
   Chrome --> Stream[Existing desktop stream]
 ```
 
-The Terminal dock launcher opens [WezTerm](https://wezterm.org/), with a plain dark native title bar, minimize/maximize/close controls, a new-tab button and split panes. Use Ctrl+Shift+T for a tab, Ctrl+Shift+D to split side by side, and Ctrl+Shift+E to split top and bottom. The desktop seeds an editable `data/desktop-panel/wezterm/wezterm.lua` configuration with software rendering and a dark theme. The managed launcher keeps its stable `data/desktop-panel/applications/xterm.desktop` path so existing dock pins remain valid. Setup installs an exact-version, SHA-256-verified WezTerm package for Linux x64 or ARM64, plus its rendering libraries.
+The Terminal dock launcher opens [WezTerm](https://wezterm.org/), with a plain dark native title bar, minimize/maximize/close controls, a new-tab button and split panes. Use Ctrl+Shift+T for a tab, Ctrl+Shift+D to split side by side, and Ctrl+Shift+E to split top and bottom. The desktop seeds an editable `data/desktop-panel/wezterm/wezterm.lua` configuration with software rendering and a dark theme. The managed launcher keeps its stable `data/desktop-panel/applications/xterm.desktop` path so existing dock pins remain valid. The image includes an exact-version, SHA-256-verified WezTerm package for Linux x64 or ARM64, plus its rendering libraries.
 
 Chrome and WezTerm use explicit desktop window classes shared with their launcher entries. The desktop session bus and Plank share `XDG_DATA_HOME` so the D-Bus-activated window matcher can resolve the generated launchers and group running windows under their pinned icons.
 
@@ -34,9 +36,9 @@ The client records key and mouse presses before dispatch. On release it opens a
 
 `--use-active-tab` and personal browser targeting are rejected with `--virtual-desktop`. Download waiting is unsupported. Browser operations are bounded to two minutes and share the desktop lease's action budget and idle expiry.
 
-Validation: focused client tests exercise shared snapshot/click behavior, namespace isolation, stale references, target changes, cancellation and uncertain-input cleanup. Lease tests cover browser ownership, cancellation and cleanup independently of native input. The Linux smoke script exercises real Chrome, the CLI and visible pointer feedback.
+Validation: focused client tests exercise shared snapshot/click behavior, namespace isolation, stale references, target changes, cancellation and uncertain-input cleanup. Lease tests cover browser ownership, cancellation and cleanup independently of native input. The Linux smoke script exercises real Chrome, the CLI and visible pointer feedback. For image replacement coverage, run `scripts/smoke-desktop-browser-cli.ts` in two disposable containers with the same workspace volume and networking disabled from first use. Both runs require baked components to be ready before navigation and assert that no installation progress is emitted. Each run needs a temporary `ASSISTANT_IPC_SOCKET_DIR`.
 
-The desktop header icon pulses in the assistant's avatar accent color while a browser automation lease is active, including between browser commands. It shares the progress indicator's accent and neutral fallback. Reduced-motion clients show a solid accent. Setup status exposes the optional `automationActive` field; `desktop_activity_changed` events refresh it on acquisition and cancellation or release. Installation progress retains its `assistant:self:desktop` sync invalidations. The indicator reads status without starting installation, and reconnects refetch the current lease state.
+The desktop header icon pulses in the assistant's avatar accent color while a browser automation lease is active, including between browser commands. It shares the progress indicator's accent and neutral fallback. Reduced-motion clients show a solid accent. Setup status exposes the optional `automationActive` field; `desktop_activity_changed` events refresh it on acquisition and cancellation or release. The indicator reads status, and reconnects refetch the current lease state.
 
 Desktop streaming checks the current in-memory gateway feature flags during connection startup. Once connected, frame and drain callbacks do not check feature flags or load workspace configuration. Turning the flag off prevents new connections; an existing stream continues until it closes or the desktop stops.
 
@@ -50,8 +52,10 @@ Non-platform assistants keep their connected-computer behavior.
 
 Explicit `target: "connected-computer"` or `target_client_id` selects a connected
 computer on every client. The selected target never falls back to another
-computer when unavailable. Virtual desktop observations return full-screen
-screenshots and mouse actions use screen coordinates.
+computer when unavailable. The virtual desktop uses a fixed 1440x810 display.
+Observations return full-screen screenshots and mouse actions use the same
+screen coordinates.
+Viewer resizing scales the canvas locally without changing the desktop layout.
 
 Observe first and pass the returned `observation_id` with each native action.
 Browser commands, user handoff, errors, and interruption require a fresh

package/openapi.yaml

@@ -12012,7 +12012,7 @@ paths:
                 additionalProperties: false
     post:
       operationId: desktop_setup_post
-      summary: Install virtual desktop components
+      summary: Check virtual desktop readiness (legacy POST)
       tags:
         - desktop
       responses:
@@ -21363,6 +21363,100 @@ paths:
                                       - redirectCount
                                       - durationMs
                                     additionalProperties: false
+                                  recall:
+                                    type: object
+                                    properties:
+                                      query:
+                                        type: string
+                                      depth:
+                                        type: string
+                                        enum:
+                                          - fast
+                                          - standard
+                                          - deep
+                                      sources:
+                                        type: array
+                                        items:
+                                          type: string
+                                          enum:
+                                            - memory
+                                            - conversations
+                                            - workspace
+                                      answer:
+                                        type: string
+                                      evidence:
+                                        type: array
+                                        items:
+                                          type: object
+                                          properties:
+                                            source:
+                                              type: string
+                                              enum:
+                                                - memory
+                                                - conversations
+                                                - workspace
+                                            title:
+                                              type: string
+                                            locator:
+                                              type: string
+                                            excerpt:
+                                              type: string
+                                            timestampMs:
+                                              type: number
+                                            path:
+                                              type: string
+                                            conversationId:
+                                              type: string
+                                            messageId:
+                                              type: string
+                                          required:
+                                            - source
+                                            - title
+                                            - locator
+                                            - excerpt
+                                          additionalProperties: false
+                                      searchedSources:
+                                        type: array
+                                        items:
+                                          type: object
+                                          properties:
+                                            source:
+                                              type: string
+                                              enum:
+                                                - memory
+                                                - conversations
+                                                - workspace
+                                            status:
+                                              type: string
+                                              enum:
+                                                - searched
+                                                - degraded
+                                            evidenceCount:
+                                              type: number
+                                            error:
+                                              type: string
+                                          required:
+                                            - source
+                                            - status
+                                            - evidenceCount
+                                          additionalProperties: false
+                                    required:
+                                      - query
+                                      - depth
+                                      - sources
+                                      - evidence
+                                      - searchedSources
+                                    additionalProperties: false
+                                  remember:
+                                    type: object
+                                    properties:
+                                      facts:
+                                        type: array
+                                        items:
+                                          type: string
+                                    required:
+                                      - facts
+                                    additionalProperties: false
                                 additionalProperties: false
                               scopeOptions:
                                 type: array
@@ -21883,6 +21977,100 @@ paths:
                                               - redirectCount
                                               - durationMs
                                             additionalProperties: false
+                                          recall:
+                                            type: object
+                                            properties:
+                                              query:
+                                                type: string
+                                              depth:
+                                                type: string
+                                                enum:
+                                                  - fast
+                                                  - standard
+                                                  - deep
+                                              sources:
+                                                type: array
+                                                items:
+                                                  type: string
+                                                  enum:
+                                                    - memory
+                                                    - conversations
+                                                    - workspace
+                                              answer:
+                                                type: string
+                                              evidence:
+                                                type: array
+                                                items:
+                                                  type: object
+                                                  properties:
+                                                    source:
+                                                      type: string
+                                                      enum:
+                                                        - memory
+                                                        - conversations
+                                                        - workspace
+                                                    title:
+                                                      type: string
+                                                    locator:
+                                                      type: string
+                                                    excerpt:
+                                                      type: string
+                                                    timestampMs:
+                                                      type: number
+                                                    path:
+                                                      type: string
+                                                    conversationId:
+                                                      type: string
+                                                    messageId:
+                                                      type: string
+                                                  required:
+                                                    - source
+                                                    - title
+                                                    - locator
+                                                    - excerpt
+                                                  additionalProperties: false
+                                              searchedSources:
+                                                type: array
+                                                items:
+                                                  type: object
+                                                  properties:
+                                                    source:
+                                                      type: string
+                                                      enum:
+                                                        - memory
+                                                        - conversations
+                                                        - workspace
+                                                    status:
+                                                      type: string
+                                                      enum:
+                                                        - searched
+                                                        - degraded
+                                                    evidenceCount:
+                                                      type: number
+                                                    error:
+                                                      type: string
+                                                  required:
+                                                    - source
+                                                    - status
+                                                    - evidenceCount
+                                                  additionalProperties: false
+                                            required:
+                                              - query
+                                              - depth
+                                              - sources
+                                              - evidence
+                                              - searchedSources
+                                            additionalProperties: false
+                                          remember:
+                                            type: object
+                                            properties:
+                                              facts:
+                                                type: array
+                                                items:
+                                                  type: string
+                                            required:
+                                              - facts
+                                            additionalProperties: false
                                         additionalProperties: false
                                       scopeOptions:
                                         type: array

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.12.2-dev.202609211820.22f71b0",
+  "version": "0.12.2-dev.202609212016.fb4c53e",
   "license": "MIT",
   "type": "module",
   "exports": {

package/scripts/smoke-desktop-browser-cli.ts

@@ -14,26 +14,18 @@ import { registerBrowserCommand } from "../src/cli/commands/browser.js";
 import { DesktopAutomationLease } from "../src/desktop/desktop-automation-lease.js";
 import {
   desktopChromePath,
-  desktopDependencyInstaller,
+  desktopDependencies,
 } from "../src/desktop/desktop-dependencies.js";
 import { DesktopSessionManager } from "../src/desktop/desktop-session-manager.js";
 import { getAssistantSocketPath } from "../src/ipc/socket-path.js";
 
-if (
-  process.platform !== "linux" ||
-  !process.argv[2] ||
-  !process.env.ASSISTANT_IPC_SOCKET_DIR
-) {
+if (process.platform !== "linux" || !process.env.ASSISTANT_IPC_SOCKET_DIR) {
   throw new Error(
-    "Run in disposable Linux with desktop packages, a temporary ASSISTANT_IPC_SOCKET_DIR and a Chrome executable argument or --install for a cold install.",
+    "Run in a disposable Linux assistant image with a temporary ASSISTANT_IPC_SOCKET_DIR.",
   );
 }
-const coldInstall = process.argv[2] === "--install";
-const executable = coldInstall ? desktopChromePath() : process.argv[2];
-if (coldInstall) {
-  assert.equal(desktopDependencyInstaller.getStatus().state, "required");
-  assert.equal(Bun.which("Xtigervnc"), null);
-}
+const executable = desktopChromePath();
+assert.equal(desktopDependencies.getStatus().state, "ready");
 const directory = await mkdtemp(join(tmpdir(), "desktop-browser-cli-"));
 const manager = new DesktopSessionManager({
   resolveChromePath: async () => executable,
@@ -44,9 +36,7 @@ const manager = new DesktopSessionManager({
 const control = new DesktopAutomationLease({
   notify: async () => {},
   enabled: () => true,
-  ready: () =>
-    !coldInstall || desktopDependencyInstaller.getStatus().state === "ready",
-  ensureReady: (signal) => desktopDependencyInstaller.ensureReady(signal),
+  ready: () => desktopDependencies.getStatus().state === "ready",
   manager: () => manager,
 });
 const context = {
@@ -182,16 +172,10 @@ try {
     "--allow-private-network",
   );
   assert.equal(navigations, 1);
-  if (coldInstall) {
-    assert.equal(desktopDependencyInstaller.getStatus().state, "ready");
-    assert(
-      setupNotifications >= 3,
-      "Installation progress must reach the viewer",
-    );
-    console.error(
-      "PASS: one CLI navigate installed desktop and Chrome from scratch, then loaded the requested page once",
-    );
-  }
+  assert.equal(setupNotifications, 0);
+  console.error(
+    "PASS: baked desktop opened the requested page without installation",
+  );
   const snapshot = await cli("snapshot");
   await cli(
     "type",