@vellumai/assistant 0.12.3-dev.202609222214.c4d457d → 0.12.3-dev.202609222313.c4916bb

package/node_modules/@vellumai/avatar-manifest/src/__tests__/notification-avatar.test.ts

@@ -67,8 +67,31 @@ describe("notificationAvatarDiscHex", () => {
 });
 
 describe("notificationAvatarSvg", () => {
+  test("fills the entire circular crop with a custom image", () => {
+    const svg = notificationAvatarSvg({
+      kind: "image",
+      innerPngBase64: PNG_BASE64,
+      accentHex: "#E9642F",
+    });
+
+    expect(attributes("image", svg)).toMatchObject({
+      x: "0",
+      y: "0",
+      width: "256",
+      height: "256",
+      preserveAspectRatio: "xMidYMid slice",
+      "clip-path": "url(#notification-avatar-disc)",
+    });
+    expect(allAttributes("circle", svg)[1]).toEqual({
+      cx: "128",
+      cy: "128",
+      r: "128",
+    });
+  });
+
   test("draws one disc and one inset avatar at the default size", () => {
     const svg = notificationAvatarSvg({
+      kind: "character",
       innerPngBase64: PNG_BASE64,
       accentHex: "#E9642F",
     });
@@ -100,6 +123,7 @@ describe("notificationAvatarSvg", () => {
     // upload fill the square instead of letterboxing is the pair of attributes
     // asserted here.
     const svg = notificationAvatarSvg({
+      kind: "character",
       innerPngBase64: PNG_BASE64,
       accentHex: "#E9642F",
     });
@@ -121,6 +145,7 @@ describe("notificationAvatarSvg", () => {
 
   test("scales the geometry to a custom size", () => {
     const svg = notificationAvatarSvg({
+      kind: "character",
       innerPngBase64: PNG_BASE64,
       accentHex: "#E9642F",
       size: 100,
@@ -142,6 +167,7 @@ describe("notificationAvatarSvg", () => {
 
   test("carries a non-PNG inner raster with its own media type", () => {
     const svg = notificationAvatarSvg({
+      kind: "character",
       innerPngBase64: PNG_BASE64,
       innerMediaType: "image/jpeg",
       accentHex: "#E9642F",
@@ -154,6 +180,7 @@ describe("notificationAvatarSvg", () => {
 
   test("paints the fallback disc when there is no accent", () => {
     const svg = notificationAvatarSvg({
+      kind: "character",
       innerPngBase64: PNG_BASE64,
       accentHex: null,
     });
@@ -164,8 +191,21 @@ describe("notificationAvatarSvg", () => {
 });
 
 describe("notificationAvatarGeometry", () => {
+  test("custom images reach the disc edge at any output size", () => {
+    expect(notificationAvatarGeometry("image")).toEqual({
+      radius: 128,
+      offset: 0,
+      inner: 256,
+    });
+    expect(notificationAvatarGeometry("image", 100)).toEqual({
+      radius: 50,
+      offset: 0,
+      inner: 100,
+    });
+  });
+
   test("derives the disc, the border and the avatar edge from the size", () => {
-    expect(notificationAvatarGeometry(100)).toEqual({
+    expect(notificationAvatarGeometry("character", 100)).toEqual({
       radius: 50,
       offset: 11,
       inner: 78,
@@ -173,15 +213,19 @@ describe("notificationAvatarGeometry", () => {
   });
 
   test("measures the default size when given none", () => {
-    expect(notificationAvatarGeometry()).toEqual(
-      notificationAvatarGeometry(NOTIFICATION_AVATAR_SIZE),
+    expect(notificationAvatarGeometry("character")).toEqual(
+      notificationAvatarGeometry("character", NOTIFICATION_AVATAR_SIZE),
     );
   });
 
   test("is the geometry the SVG is drawn with", () => {
     const size = 100;
-    const { radius, offset, inner } = notificationAvatarGeometry(size);
+    const { radius, offset, inner } = notificationAvatarGeometry(
+      "character",
+      size,
+    );
     const svg = notificationAvatarSvg({
+      kind: "character",
       innerPngBase64: PNG_BASE64,
       accentHex: null,
       size,

package/node_modules/@vellumai/avatar-manifest/src/notification-avatar.ts

@@ -15,11 +15,12 @@
  */
 
 import { isAvatarAccentHex } from "./accent.js";
+import type { AvatarKind } from "./manifest.js";
 
 /** Output edge in pixels: large enough for an iOS notification thumbnail. */
 export const NOTIFICATION_AVATAR_SIZE = 256;
 
-/** Free disc on each side, as a fraction of the edge; the avatar gets the rest. */
+/** Free disc on each side of a character avatar, as a fraction of the edge. */
 export const NOTIFICATION_AVATAR_INSET = 0.11;
 
 /** The disc when the assistant has no accent. */
@@ -48,7 +49,7 @@ export const NOTIFICATION_AVATAR_MAX_LOCAL_BYTES = 512 * 1024;
  * Bumped whenever the drawing above changes, so a sync keyed on it re-uploads
  * a disc rendered by an older spec even when the avatar itself is unchanged.
  */
-export const NOTIFICATION_AVATAR_SPEC_VERSION = 1;
+export const NOTIFICATION_AVATAR_SPEC_VERSION = 2;
 
 /** The id the inner raster's clip path is referenced by inside the document. */
 const CLIP_ID = "notification-avatar-disc";
@@ -74,24 +75,26 @@ export function notificationAvatarDiscHex(accentHex: string | null): string {
 
 /** Raster formats an `<image>` href carries here; what resvg and a canvas both decode. */
 export type NotificationAvatarMediaType =
-  | "image/png"
-  | "image/jpeg"
-  | "image/gif";
+  "image/png" | "image/jpeg" | "image/gif";
 
 /**
  * The three measurements the drawing derives from the edge: the disc's radius,
  * which is also its centre; the free border around the avatar; and the
  * avatar's own edge. Every rasterizer reads them from here, so the daemon's
  * SVG and the desktop canvas cannot drift.
+ * Custom images fill the disc; character avatars retain space around their
+ * silhouette.
  */
 export function notificationAvatarGeometry(
+  kind: Exclude<AvatarKind, "none">,
   size: number = NOTIFICATION_AVATAR_SIZE,
 ): { radius: number; offset: number; inner: number } {
-  const offset = size * NOTIFICATION_AVATAR_INSET;
+  const offset = kind === "character" ? size * NOTIFICATION_AVATAR_INSET : 0;
   return { radius: size / 2, offset, inner: size - 2 * offset };
 }
 
 export interface NotificationAvatarSvgOptions {
+  kind: Exclude<AvatarKind, "none">;
   /** The avatar raster to draw inside the disc, base64 with no data prefix. */
   innerPngBase64: string;
   /** What `innerPngBase64` holds; PNG unless an upload arrived as a JPEG or a GIF. */
@@ -107,7 +110,7 @@ function px(value: number): string {
 
 /**
  * The notification avatar as an SVG document: a filled disc with the avatar
- * drawn inset into it.
+ * filling it for custom images or inset for character avatars.
  *
  * The inner raster is cover-cropped (`xMidYMid slice`), matching what the web
  * canvas does, so a portrait or landscape upload fills the square instead of
@@ -119,12 +122,13 @@ function px(value: number): string {
  * rasterizer is known to render it.
  */
 export function notificationAvatarSvg({
+  kind,
   innerPngBase64,
   innerMediaType = "image/png",
   accentHex,
   size = NOTIFICATION_AVATAR_SIZE,
 }: NotificationAvatarSvgOptions): string {
-  const { radius, offset, inner } = notificationAvatarGeometry(size);
+  const { radius, offset, inner } = notificationAvatarGeometry(kind, size);
   const href = `data:${innerMediaType};base64,${innerPngBase64}`;
   const disc = `cx="${px(radius)}" cy="${px(radius)}" r="${px(radius)}"`;
   return (

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.12.3-dev.202609222214.c4d457d",
+  "version": "0.12.3-dev.202609222313.c4916bb",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/skill-load-inline-command.test.ts

@@ -109,6 +109,41 @@ async function executeSkillLoad(
 // ── Tests ─────────────────────────────────────────────────────────────────
 
 describe("skill_load inline command expansion", () => {
+  test("read-only preloading returns instructions without running inline commands", async () => {
+    writeSkill(
+      "readonly-example",
+      "readonly-example",
+      "Example skill",
+      "Context: !`echo example`",
+    );
+    const result = await skillLoadTool.execute(
+      { skill: "readonly-example" },
+      {
+        conversationId: "conversation-1",
+        workingDir: TEST_DIR,
+        trustClass: "guardian",
+      },
+      { readOnly: true },
+    );
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain("[inline command not executed]");
+    expect(runInlineCommandCalls).toHaveLength(0);
+    expect(mockAutoInstall).not.toHaveBeenCalled();
+  });
+
+  test("read-only preloading does not install missing skills", async () => {
+    const result = await skillLoadTool.execute(
+      { skill: "missing-readonly-skill" },
+      {
+        conversationId: "conversation-1",
+        workingDir: TEST_DIR,
+        trustClass: "guardian",
+      },
+      { readOnly: true },
+    );
+    expect(result.isError).toBe(true);
+    expect(mockAutoInstall).not.toHaveBeenCalled();
+  });
   beforeEach(() => {
     mkdirSync(join(TEST_DIR, "skills"), { recursive: true });
     runInlineCommandCalls.length = 0;

package/src/avatar/notification-avatar.test.ts

@@ -63,9 +63,6 @@ const TRAITS = { bodyShape: "blob", eyeStyle: "curious", color: "green" };
 const GREEN_DISC = { r: 0xe6, g: 0xf1, b: 0xe7 };
 /** #E9642F mixed 14% into white. */
 const ORANGE_DISC = { r: 0xfc, g: 0xe9, b: 0xe2 };
-/** The neutral disc every accent-less avatar wears. */
-const FALLBACK_DISC = { r: 0xec, g: 0xef, b: 0xea };
-
 /** On the disc and clear of the inset avatar square, which starts at y = 28.16. */
 const DISC_SAMPLE = { x: 128, y: 8 };
 /** Outside the inscribed disc, where the render leaves nothing at all. */
@@ -233,7 +230,7 @@ describe("renderNotificationAvatarPng", () => {
   );
 
   nativeTest(
-    "renders an uploaded image on the neutral disc when it has no accent",
+    "fills the circle with an uploaded image when it has no accent",
     async () => {
       await writeUpload("png", null);
 
@@ -245,7 +242,9 @@ describe("renderNotificationAvatarPng", () => {
       expect(image.height).toBe(256);
       expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
       expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
-        ...FALLBACK_DISC,
+        r: 0x20,
+        g: 0x60,
+        b: 0xc0,
         a: 255,
       });
       expect(image.at(128, 128)).toEqual({ r: 0x20, g: 0x60, b: 0xc0, a: 255 });
@@ -253,23 +252,34 @@ describe("renderNotificationAvatarPng", () => {
     NATIVE_RENDER_TEST_TIMEOUT_MS,
   );
 
-  nativeTest(
-    "cover-crops a wide upload and clips it to the disc",
-    async () => {
-      // 4:1, so a letterboxing render would leave disc colour above and below
-      // the avatar inside the inset square.
-      await writeUpload("png", null, { width: 256, height: 64 });
-
-      const image = await pixels((await renderNotificationAvatarPng())!);
-      // Inside the inset square (which starts at 28.16) near its top edge:
-      // filled by a cover-crop, disc colour by a letterbox.
-      expect(image.at(128, 35)).toEqual({ r: 0x20, g: 0x60, b: 0xc0, a: 255 });
-      // Inside the inset square but outside the disc, so only the clip can
-      // keep the avatar out of it.
-      expect(image.at(30, 30).a).toBe(0);
-    },
-    NATIVE_RENDER_TEST_TIMEOUT_MS,
-  );
+  for (const [shape, width, height] of [
+    ["landscape", 256, 64],
+    ["portrait", 64, 256],
+  ] as const) {
+    nativeTest(
+      `cover-crops a ${shape} upload to the full disc`,
+      async () => {
+        await writeUpload("png", null, { width, height });
+
+        const image = await pixels((await renderNotificationAvatarPng())!);
+        for (const [x, y] of [
+          [128, 8],
+          [8, 128],
+          [128, 248],
+          [248, 128],
+        ]) {
+          expect(image.at(x!, y!)).toEqual({
+            r: 0x20,
+            g: 0x60,
+            b: 0xc0,
+            a: 255,
+          });
+        }
+        expect(image.at(30, 30).a).toBe(0);
+      },
+      NATIVE_RENDER_TEST_TIMEOUT_MS,
+    );
+  }
 
   nativeTest(
     "transcodes a WebP upload, which resvg cannot decode on its own",
@@ -281,20 +291,18 @@ describe("renderNotificationAvatarPng", () => {
 
       const image = await pixels(png!);
       expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
-      expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
-        ...FALLBACK_DISC,
-        a: 255,
-      });
       // WebP is lossy, so the transcode moves the flat fill by a channel step
       // or two; what matters is that the avatar is drawn at all.
-      const centre = image.at(128, 128);
-      expect(centre.a).toBe(255);
-      for (const [actual, expected] of [
-        [centre.r, 0x20],
-        [centre.g, 0x60],
-        [centre.b, 0xc0],
-      ]) {
-        expect(Math.abs(actual! - expected!)).toBeLessThanOrEqual(4);
+      for (const point of [{ x: 128, y: 128 }, DISC_SAMPLE]) {
+        const pixel = image.at(point.x, point.y);
+        expect(pixel.a).toBe(255);
+        for (const [actual, expected] of [
+          [pixel.r, 0x20],
+          [pixel.g, 0x60],
+          [pixel.b, 0xc0],
+        ]) {
+          expect(Math.abs(actual! - expected!)).toBeLessThanOrEqual(4);
+        }
       }
     },
     NATIVE_RENDER_TEST_TIMEOUT_MS,

package/src/avatar/notification-avatar.ts

@@ -222,6 +222,7 @@ export async function renderNotificationAvatarPng(
     const Resvg = getResvg();
     const resvg = new Resvg(
       notificationAvatarSvg({
+        kind: state.kind,
         innerPngBase64: source.bytes.toString("base64"),
         innerMediaType: source.mediaType,
         accentHex: resolveNotificationAccentHex(state),

package/src/calls/__tests__/voice-session-bridge.test.ts

@@ -143,6 +143,10 @@ mock.module("../../persistence/conversation-crud.js", () => ({
   recordConversationPersistedSeq: () => {},
 }));
 
+import {
+  clearHubClients,
+  registerHubClient,
+} from "../../__tests__/helpers/hub-clients.js";
 import { setConfig } from "../../__tests__/helpers/set-config.js";
 import type { PreparedModelCall } from "../../agent/loop.js";
 import { selectWinningProfile } from "../../config/llm-resolver.js";
@@ -2326,21 +2330,30 @@ describe("front-door leg tool suppression", () => {
 });
 
 describe("desktop skill preactivation", () => {
+  afterEach(() => clearHubClients(assistantEventHub));
+
   async function preactivatedFor(turn: Record<string, unknown>): Promise<{
     skillIds: string[];
     proxyInterfaces: unknown[];
     skillIdsDuringLoop: string[];
+    promptDuringLoop: string | null;
   }> {
     const skillIds: string[] = [];
     const proxyInterfaces: unknown[] = [];
     let skillIdsDuringLoop: string[] = [];
+    let prompt: string | null = null;
+    let promptDuringLoop: string | null = null;
     const fake = makeFakeConversation({
       processing: false,
       runAgentLoop: async () => {
         skillIdsDuringLoop = [...skillIds];
+        promptDuringLoop = prompt;
       },
     });
     Object.assign(fake.conversation, {
+      setVoiceCallControlPrompt: (value: string | null) => {
+        prompt = value;
+      },
       addPreactivatedSkillId: (id: string) => {
         skillIds.push(id);
       },
@@ -2356,9 +2369,58 @@ describe("desktop skill preactivation", () => {
       ...turn,
     });
     await flushMicrotasks();
-    return { skillIds, proxyInterfaces, skillIdsDuringLoop };
+    return { skillIds, proxyInterfaces, skillIdsDuringLoop, promptDuringLoop };
   }
 
+  test("a shared-screen turn receives the annotation instructions and schemas before inference", async () => {
+    registerHubClient({
+      hub: assistantEventHub,
+      clientId: "annotation-client",
+      interfaceId: "macos",
+      actorPrincipalId: "user-123",
+      capabilities: ["host_cu", "host_cu_annotate"],
+    });
+    const result = await preactivatedFor({
+      routingLeg: "escalated",
+      macosDesktopSession: true,
+      screenSharing: true,
+      actorPrincipalId: "user-123",
+    });
+    expect(result.skillIdsDuringLoop).toContain("screen-annotation");
+    expect(result.promptDuringLoop).toContain("ID: screen-annotation");
+    expect(result.promptDuringLoop).toContain("Name the thing.");
+    expect(result.promptDuringLoop).toContain("screen_point_at");
+    expect(result.promptDuringLoop).toContain("screen_clear_marks");
+    expect(result.promptDuringLoop).toContain('"target"');
+    expect(result.promptDuringLoop).toContain("skill_execute");
+  });
+
+  test.each([
+    { screenSharing: false },
+    { actorPrincipalId: "other-user" },
+    { routingLeg: "front-door" },
+    { macosDesktopSession: false },
+  ])(
+    "does not preload annotation for an ineligible turn: %j",
+    async (override) => {
+      registerHubClient({
+        hub: assistantEventHub,
+        clientId: "annotation-client",
+        interfaceId: "macos",
+        actorPrincipalId: "user-123",
+        capabilities: ["host_cu", "host_cu_annotate"],
+      });
+      const result = await preactivatedFor({
+        routingLeg: "escalated",
+        macosDesktopSession: true,
+        screenSharing: true,
+        actorPrincipalId: "user-123",
+        ...override,
+      });
+      expect(result.promptDuringLoop).not.toContain("ID: screen-annotation");
+    },
+  );
+
   test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
     const result = await preactivatedFor({
       routingLeg: "escalated",

package/src/calls/voice-screen-annotation.ts

@@ -0,0 +1,44 @@
+import { shouldAttachHostProxyForCapability } from "../daemon/host-proxy-preactivation.js";
+import { skillLoadTool } from "../tools/skills/load.js";
+import type { ToolContext } from "../tools/types.js";
+import { getLogger } from "../util/logger.js";
+
+const log = getLogger("voice-screen-annotation");
+
+export async function loadVoiceScreenAnnotation(
+  context: ToolContext,
+): Promise<string | null> {
+  if (
+    !shouldAttachHostProxyForCapability(
+      "host_cu_annotate",
+      context.transportInterface,
+      context.sourceActorPrincipalId,
+    )
+  ) {
+    return null;
+  }
+  try {
+    const result = await skillLoadTool.execute(
+      { skill: "screen-annotation" },
+      context,
+      { readOnly: true },
+    );
+    if (result.isError) {
+      log.warn(
+        { conversationId: context.conversationId, error: result.content },
+        "Could not preload screen annotation",
+      );
+      return null;
+    }
+    return [
+      "The user is sharing their screen. skill_load has automatically loaded screen-annotation for this turn. Use its tools through skill_execute when pointing at the shared screen.",
+      result.content,
+    ].join("\n\n");
+  } catch (err) {
+    log.warn(
+      { err, conversationId: context.conversationId },
+      "Could not preload screen annotation",
+    );
+    return null;
+  }
+}

package/src/calls/voice-session-bridge.ts

@@ -32,7 +32,9 @@ import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
 import type { ModeSessionSourceHandle } from "../daemon/conversation-mode-session.js";
 import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
 import { getOrCreateConversation } from "../daemon/conversation-store.js";
+import { getEffectiveEnabledPluginSet } from "../daemon/conversation-tool-setup.js";
 import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
+import { resolveTrustClass } from "../daemon/trust-context.js";
 import type { TrustContext } from "../daemon/trust-context-types.js";
 import {
   newestPersistedSightFrame,
@@ -78,6 +80,7 @@ import {
 } from "./voice-control-protocol.js";
 import { judgeEscalation } from "./voice-escalation-judge.js";
 import type { VoiceEscalationTarget } from "./voice-escalation-target.js";
+import { loadVoiceScreenAnnotation } from "./voice-screen-annotation.js";
 import {
   createFrontDoorStreamGate,
   escalatedContinuationRule,
@@ -452,6 +455,8 @@ export interface VoiceTurnOptions {
    * session reports `macos` for its channel capabilities, iOS included.
    */
   macosDesktopSession?: boolean;
+  /** The desktop client currently shares a surface with this voice session. */
+  screenSharing?: boolean;
   /** Whether this is an inbound call (no outbound task). */
   isInbound: boolean;
   /** The outbound call task, if any. */
@@ -2108,6 +2113,23 @@ export async function startVoiceTurn(
           sourceInterface,
           sourceActorPrincipalId,
         );
+        if (opts.screenSharing === true) {
+          const annotation = await loadVoiceScreenAnnotation({
+            conversationId: opts.conversationId,
+            workingDir: conversation.workingDir,
+            trustClass: resolveTrustClass(opts.trustContext),
+            transportInterface: sourceInterface,
+            clientOs: "macos",
+            enabledPluginSet: getEffectiveEnabledPluginSet(conversation),
+            sourceActorPrincipalId,
+            signal: opts.signal,
+          });
+          if (annotation !== null) {
+            conversation.setVoiceCallControlPrompt(
+              [voiceCallControlPrompt, annotation].filter(Boolean).join("\n\n"),
+            );
+          }
+        }
       }
       await conversation.runAgentLoop(persistedContent, messageId, {
         ...(opts.subagentNotification?.cronRunId

package/src/live-voice/__tests__/live-voice-agent-turn.test.ts

@@ -238,6 +238,33 @@ function assistantDeltaTexts(frames: LiveVoiceServerFrame[]): string[] {
 }
 
 describe("LiveVoiceSession assistant turn", () => {
+  test.each([true, false])(
+    "passes current screen sharing state %p to the desktop voice bridge",
+    async (screenSharing) => {
+      const startVoiceTurn = mock(async (_options: VoiceTurnOptions) => ({
+        turnId: "bridge-turn-1",
+        abort: mock(),
+      }));
+      const { session, transcriber } = createSessionHarness({
+        startFrame: { ...START_FRAME, client: "macos" },
+        startVoiceTurn,
+      });
+      await session.start();
+      await session.handleClientFrame({
+        type: "update_config",
+        screenSharing: true,
+      });
+      await session.handleClientFrame({ type: "update_config", screenSharing });
+      transcriber.emit({ type: "final", text: "Show me the speed control" });
+      await session.handleClientFrame({ type: "ptt_release" });
+      expect(startVoiceTurn.mock.calls[0]?.[0]).toMatchObject({
+        macosDesktopSession: true,
+        screenSharing,
+      });
+      await session.close("client_end");
+    },
+  );
+
   test("runs final transcripts through the voice bridge and forwards ordered assistant events", async () => {
     const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
       options.callbacks?.assistant_text_delta?.({

package/src/live-voice/__tests__/protocol.test.ts

@@ -77,6 +77,28 @@ describe("parseLiveVoiceClientTextFrame", () => {
     expect(result).toEqual({ ok: true, frame });
   });
 
+  test.each([true, false])(
+    "parses screen sharing state %p",
+    (screenSharing) => {
+      const frame = { type: "update_config", screenSharing } as const;
+      expect(parseLiveVoiceClientTextFrame(JSON.stringify(frame))).toEqual({
+        ok: true,
+        frame,
+      });
+    },
+  );
+
+  test("rejects malformed screen sharing state", () => {
+    expect(
+      parseLiveVoiceClientTextFrame(
+        JSON.stringify({
+          type: "update_config",
+          screenSharing: "true",
+        }),
+      ).ok,
+    ).toBe(false);
+  });
+
   test("parses an update_config frame with a single field, omitting the other", () => {
     const result = parseLiveVoiceClientTextFrame(
       JSON.stringify({ type: "update_config", silenceThresholdMs: 900 }),

package/src/live-voice/live-voice-session.ts

@@ -1217,6 +1217,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   private releaseModeSessionResidency?: () => void;
   private cameraModeSessions?: CameraModeSessionProducer;
   private sightFrameSequence: Promise<void> = Promise.resolve();
+  private screenSharing = false;
   /**
    * Mirrors phase changes to the iOS Live Activity through the platform, for
    * the case the client cannot cover: an app backgrounded long enough for iOS
@@ -2084,13 +2085,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   }
 
   /**
-   * Apply a mid-session `update_config` frame: retune the live turn detector's
-   * pause ("pause before reply") and/or the barge-in guard ("interrupt
-   * sensitivity") without reconnecting. Each field is optional and independent;
-   * changes take effect from the next utterance. A no-op on manual (non-
-   * server_vad) sessions, which have no turn detector.
+   * Apply screen-sharing state and turn-detection tuning independently.
+   * Changes take effect from the next utterance.
    */
   private applyConfigUpdate(frame: LiveVoiceClientUpdateConfigFrame): void {
+    if (frame.screenSharing !== undefined) {
+      this.screenSharing = frame.screenSharing;
+    }
     if (frame.silenceThresholdMs !== undefined) {
       this.turnDetector?.setSilenceThresholdMs(frame.silenceThresholdMs);
       this.silenceThresholdMs = frame.silenceThresholdMs;
@@ -6602,7 +6603,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         userMessageInterface: "macos",
         assistantMessageInterface: "macos",
         ...(this.context.startFrame.client === "macos"
-          ? { macosDesktopSession: true }
+          ? { macosDesktopSession: true, screenSharing: this.screenSharing }
           : {}),
         voiceTelemetry: {
           sessionId: this.context.sessionId,

package/src/live-voice/protocol.ts

@@ -276,16 +276,16 @@ export interface LiveVoiceClientEndFrame {
 }
 
 /**
- * Mid-session tuning update: applies the same turn-detection knobs the start
- * frame carries to the *running* session, so the client can retune "pause
- * before reply" / "interrupt sensitivity" without reconnecting. Each field is
- * optional and independently applied; the same bounds as the start frame apply.
- * Only meaningful for `server_vad` sessions.
+ * Mid-session screen-sharing state and turn-detection tuning. Each field is
+ * optional and independently applied. Tuning uses the start frame's bounds
+ * and affects `server_vad` sessions.
  */
 export interface LiveVoiceClientUpdateConfigFrame {
   readonly type: "update_config";
   readonly silenceThresholdMs?: number;
   readonly bargeInMinSpeechMs?: number;
+  /** A shared desktop surface is available for annotation. */
+  readonly screenSharing?: boolean;
 }
 
 /**
@@ -1279,6 +1279,14 @@ function validateSightFrameTiming(
 function validateUpdateConfigFrame(
   value: Record<string, unknown>,
 ): LiveVoiceParseResult<LiveVoiceClientUpdateConfigFrame> {
+  if ("screenSharing" in value && typeof value.screenSharing !== "boolean") {
+    return protocolError(
+      "invalid_field",
+      "update_config field screenSharing must be a boolean",
+      "screenSharing",
+      "update_config",
+    );
+  }
   if (
     "silenceThresholdMs" in value &&
     !isIntInRange(
@@ -1315,6 +1323,9 @@ function validateUpdateConfigFrame(
     ok: true,
     frame: {
       type: "update_config",
+      ...(typeof value.screenSharing === "boolean"
+        ? { screenSharing: value.screenSharing }
+        : {}),
       ...(typeof value.silenceThresholdMs === "number"
         ? { silenceThresholdMs: value.silenceThresholdMs }
         : {}),

package/src/platform/sync-avatar.test.ts

@@ -709,6 +709,35 @@ describe("syncAvatarToPlatform", () => {
     expect(patches).toHaveLength(1);
   });
 
+  test("an older rendering spec re-uploads the same avatar once", async () => {
+    mockResvgAvailable = true;
+    syncAvatarToPlatform();
+    await settle();
+    const current = JSON.parse(readFileSync(syncStatePath, "utf-8"));
+    const previous = {
+      ...current,
+      key: current.key.replace(
+        `:${NOTIFICATION_AVATAR_SPEC_VERSION}:`,
+        `:${NOTIFICATION_AVATAR_SPEC_VERSION - 1}:`,
+      ),
+    };
+    expect(previous.key).not.toBe(current.key);
+    writeFileSync(syncStatePath, JSON.stringify(previous));
+
+    _resetSyncAvatarStateForTests();
+    syncAvatarToPlatform();
+    await settle();
+    _resetSyncAvatarStateForTests();
+    syncAvatarToPlatform();
+    await settle();
+
+    expect(patches).toHaveLength(2);
+    expect(patches[1].body.notification_avatar_base64).toBeDefined();
+    expect(JSON.parse(readFileSync(syncStatePath, "utf-8")).key).toBe(
+      current.key,
+    );
+  });
+
   test("a restart re-uploads when the destination or raster changed", async () => {
     syncAvatarToPlatform();
     await settle();

package/src/tools/skills/load.ts

@@ -157,6 +157,7 @@ export const skillLoadTool = {
   async execute(
     input: Record<string, unknown>,
     context: ToolContext,
+    options?: { readOnly?: boolean },
   ): Promise<ToolExecutionResult> {
     const selector = input.skill;
     if (typeof selector !== "string" || selector.trim().length === 0) {
@@ -171,13 +172,17 @@ export const skillLoadTool = {
     // command execution. Instructions are still returned so the assistant can
     // load the system-storage-cleanup skill (and any already-installed skill).
     const cleanupMode = context.diskPressureCleanupModeActive === true;
+    const readOnly = cleanupMode || options?.readOnly === true;
+    const inlineCommandStub = cleanupMode
+      ? INLINE_COMMAND_CLEANUP_STUB
+      : "[inline command not executed]";
 
     let loaded = loadSkillBySelector(selector);
 
     // Auto-install from catalog if the skill isn't found locally
     if (
       !loaded.skill &&
-      !cleanupMode &&
+      !readOnly &&
       (loaded.errorCode === "not_found" || loaded.errorCode === "empty_catalog")
     ) {
       try {
@@ -273,9 +278,8 @@ export const skillLoadTool = {
           break;
         }
 
-        // Under the disk-pressure lock, never auto-install missing includes
-        // (that writes to the workspace). Leave them advisory ("not loaded").
-        if (cleanupMode) {
+        // Read-only loads leave missing includes advisory ("not loaded").
+        if (readOnly) {
           break;
         }
 
@@ -349,17 +353,11 @@ export const skillLoadTool = {
     const hasInlineCommands =
       skill.inlineCommandExpansions && skill.inlineCommandExpansions.length > 0;
 
-    if (hasInlineCommands && cleanupMode) {
-      // Under the disk-pressure lock, loading a skill must not execute shell.
-      // Strip inline command tokens instead of rendering them; the rest of the
-      // instructions are still returned.
-      body = body.replace(
-        INLINE_COMMAND_TOKEN_PATTERN,
-        INLINE_COMMAND_CLEANUP_STUB,
-      );
+    if (hasInlineCommands && readOnly) {
+      body = body.replace(INLINE_COMMAND_TOKEN_PATTERN, inlineCommandStub);
       log.info(
         { skillId: skill.id },
-        "Skipped inline command expansion during disk pressure cleanup mode",
+        "Skipped inline command expansion during read-only skill load",
       );
     } else if (hasInlineCommands) {
       if (skill.source === "extra" || skill.source === "plugin") {
@@ -451,12 +449,10 @@ export const skillLoadTool = {
             childLoaded.skill.inlineCommandExpansions &&
             childLoaded.skill.inlineCommandExpansions.length > 0;
 
-          if (childHasInlineCommands && cleanupMode) {
-            // No shell execution under the disk-pressure lock — strip the
-            // child's inline command tokens rather than rendering them.
+          if (childHasInlineCommands && readOnly) {
             childBody = childBody.replace(
               INLINE_COMMAND_TOKEN_PATTERN,
-              INLINE_COMMAND_CLEANUP_STUB,
+              inlineCommandStub,
             );
           } else if (childHasInlineCommands) {
             if (