@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

package/src/ipc/__tests__/browser-ipc.test.ts

@@ -6,8 +6,19 @@
  * state or the daemon conversation store.
  */
 
-import { afterEach, describe, expect, mock, test } from "bun:test";
-
+import {
+  afterAll,
+  afterEach,
+  describe,
+  expect,
+  mock,
+  spyOn,
+  test,
+} from "bun:test";
+
+import { desktopDependencyInstaller } from "../../desktop/desktop-dependencies.js";
+import * as desktopFeature from "../../desktop/virtual-desktop-feature.js";
+import { browserManager } from "../../tools/browser/browser-manager.js";
 import type { ToolExecutionResult } from "../../tools/types.js";
 
 // ---------------------------------------------------------------------------
@@ -31,7 +42,10 @@ let mockOperationCalls: Array<{
 let mockConversation: {
   trustContext?: { trustClass: string };
   transportInterface?: string;
+  currentTurnClientOs?: string;
+  clientOs?: string;
   getTurnActorPrincipalId?: () => string | undefined;
+  abortController?: AbortController;
 } | null = null;
 
 let mockFindConversationCalls: string[] = [];
@@ -66,6 +80,47 @@ mock.module("../../daemon/conversation-registry.js", () => ({
   },
 }));
 
+let desktopEnabled = false;
+let desktopReady = false;
+let desktopFailure = false;
+const enabledSpy = spyOn(
+  desktopFeature,
+  "isVirtualDesktopEnabled",
+).mockImplementation(() => desktopEnabled);
+const readySpy = spyOn(
+  desktopDependencyInstaller,
+  "getStatus",
+).mockImplementation(() => ({ state: desktopReady ? "ready" : "required" }));
+afterAll(() => {
+  enabledSpy.mockRestore();
+  readySpy.mockRestore();
+});
+
+let desktopContext: import("../../tools/types.js").ToolContext | undefined;
+mock.module("../../desktop/desktop-browser-operations.js", () => ({
+  executeDesktopBrowserTabs: async (
+    _params: unknown,
+    context: import("../../tools/types.js").ToolContext,
+  ) => {
+    desktopContext = context;
+    return {
+      content: JSON.stringify({ ok: true, tabs: [{ tabId: 1 }] }),
+      isError: false,
+    };
+  },
+  executeDesktopBrowserOperation: async (
+    _operation: string,
+    _input: Record<string, unknown>,
+    context: import("../../tools/types.js").ToolContext,
+  ) => {
+    desktopContext = context;
+    return {
+      content: desktopFailure ? "Desktop is busy" : "desktop",
+      isError: desktopFailure,
+    };
+  },
+}));
+
 // Import after mocking — now from the shared routes location
 const { ROUTES, browserCliConversationKey } =
   await import("../../runtime/routes/browser-routes.js");
@@ -92,6 +147,11 @@ function callHandler(
 // ---------------------------------------------------------------------------
 
 afterEach(() => {
+  desktopEnabled = false;
+  desktopReady = false;
+  desktopFailure = false;
+  desktopContext = undefined;
+  browserManager.clearPreferredBackendKind("conv-default-browser");
   mockOperationResult = { content: "ok", isError: false };
   mockOperationCalls = [];
   mockConversation = null;
@@ -422,3 +482,171 @@ describe("browser_execute route", () => {
     expect(mockOperationCalls[0].input).toEqual({});
   });
 });
+
+test("desktop route preserves guardian ownership and live-turn cancellation", async () => {
+  const abortController = new AbortController();
+  mockConversation = {
+    trustContext: { trustClass: "guardian" },
+    getTurnActorPrincipalId: () => "user-123",
+    abortController,
+  };
+  const result = await callHandler(
+    { operation: "snapshot", desktop: true, conversationId: "conv-desktop" },
+    { "x-vellum-actor-principal-id": "user-other" },
+  );
+  expect(result).toMatchObject({ content: "desktop" });
+  expect(mockOperationCalls).toHaveLength(0);
+  expect(desktopContext).toMatchObject({
+    conversationId: "conv-desktop",
+    sourceActorPrincipalId: "user-123",
+    trustClass: "guardian",
+  });
+  expect(desktopContext?.signal?.aborted).toBe(false);
+  abortController.abort();
+  expect(desktopContext?.signal?.aborted).toBe(true);
+});
+
+function webConversation(clientOs = "web") {
+  desktopEnabled = true;
+  desktopReady = true;
+  mockConversation = {
+    trustContext: { trustClass: "guardian" },
+    transportInterface: "web",
+    currentTurnClientOs: clientOs,
+    getTurnActorPrincipalId: () => "user-123",
+  };
+}
+
+test("web browser commands default to the installed streamed Chrome", async () => {
+  webConversation();
+  const result = await callHandler({
+    operation: "navigate",
+    input: { url: "https://example.com" },
+    conversationId: "conv-default-browser",
+  });
+  expect(result).toMatchObject({ content: "desktop", isError: false });
+  expect(mockOperationCalls).toHaveLength(0);
+  expect(desktopContext?.clientOs).toBe("web");
+});
+
+test.each(["macos", "windows"])(
+  "%s renderer uses the personal browser despite its web transport",
+  async (clientOs) => {
+    webConversation(clientOs);
+    await callHandler({
+      operation: "snapshot",
+      conversationId: "conv-default-browser",
+    });
+    expect(mockOperationCalls).toHaveLength(1);
+    expect(desktopContext).toBeUndefined();
+  },
+);
+
+test("browser selection uses the active turn rather than a queued message's OS", async () => {
+  webConversation("macos");
+  mockConversation!.clientOs = "web";
+  await callHandler({
+    operation: "snapshot",
+    conversationId: "conv-default-browser",
+  });
+  expect(mockOperationCalls).toHaveLength(1);
+  expect(desktopContext).toBeUndefined();
+});
+
+test.each(["flag", "guardian", "actor"])(
+  "web preserves the existing browser when %s is unavailable",
+  async (missing) => {
+    webConversation();
+    if (missing === "flag") {
+      desktopEnabled = false;
+    }
+    if (missing === "guardian") {
+      mockConversation!.trustContext = { trustClass: "unknown" };
+    }
+    if (missing === "actor") {
+      mockConversation!.getTurnActorPrincipalId = () => undefined;
+    }
+    await callHandler({
+      operation: "snapshot",
+      conversationId: "conv-default-browser",
+    });
+    expect(mockOperationCalls).toHaveLength(1);
+    expect(desktopContext).toBeUndefined();
+  },
+);
+
+test.each([
+  { browser_mode: "local" },
+  { browser_mode: "playwright" },
+  { browser_mode: "extension" },
+  { browser_mode: "cdp-inspect" },
+  { target_client_id: "client-123" },
+  { use_active_tab: true },
+])("explicit browser choice %j wins over the web default", async (input) => {
+  webConversation();
+  await callHandler({
+    operation: "snapshot",
+    input,
+    conversationId: "conv-default-browser",
+  });
+  expect(mockOperationCalls).toHaveLength(1);
+  expect(desktopContext).toBeUndefined();
+});
+
+test("follow-up web commands retain an existing personal browser session", async () => {
+  webConversation();
+  browserManager.setPreferredBackendKind("conv-default-browser", "extension");
+  await callHandler({
+    operation: "snapshot",
+    conversationId: "conv-default-browser",
+  });
+  expect(mockOperationCalls).toHaveLength(1);
+  expect(desktopContext).toBeUndefined();
+});
+
+test("native clients can explicitly select the streamed browser", async () => {
+  webConversation("windows");
+  const result = await callHandler({
+    operation: "snapshot",
+    desktop: true,
+    conversationId: "conv-default-browser",
+  });
+  expect(result).toMatchObject({ content: "desktop" });
+  expect(desktopContext?.clientOs).toBe("windows");
+  expect(mockOperationCalls).toHaveLength(0);
+});
+
+test("web tab commands share the streamed browser default", async () => {
+  webConversation();
+  const { ROUTES } =
+    await import("../../runtime/routes/browser-tabs-routes.js");
+  const result = await ROUTES[0].handler({
+    body: { command: "list", conversationId: "conv-default-browser" },
+  });
+  expect(result).toEqual({ ok: true, tabs: [{ tabId: 1 }] });
+  expect(desktopContext?.clientOs).toBe("web");
+});
+
+test("a streamed browser failure does not switch to Playwright or personal Chrome", async () => {
+  webConversation();
+  desktopFailure = true;
+  const result = await callHandler({
+    operation: "click",
+    input: { selector: "#submit" },
+    conversationId: "conv-default-browser",
+  });
+  expect(result).toMatchObject({ isError: true, content: "Desktop is busy" });
+  expect(mockOperationCalls).toHaveLength(0);
+});
+
+test("first web browser use routes to virtual desktop setup before installation", async () => {
+  webConversation();
+  desktopReady = false;
+  await callHandler({
+    operation: "navigate",
+    input: { url: "https://example.com" },
+    conversationId: "conv-default-browser",
+  });
+  expect(desktopContext?.clientOs).toBe("web");
+  expect(mockOperationCalls).toHaveLength(0);
+});

package/src/ipc/__tests__/cancel-on-disconnect.test.ts

@@ -0,0 +1,54 @@
+import { afterEach, expect, test } from "bun:test";
+
+import { setDbReady } from "../../daemon/daemon-readiness.js";
+import type { RouteHandlerArgs } from "../../runtime/routes/types.js";
+import { AssistantIpcServer } from "../assistant-server.js";
+import { cliIpcCall } from "../cli-client.js";
+
+let server: AssistantIpcServer | undefined;
+afterEach(() => server?.stop());
+
+for (const cancelOnDisconnect of [true, false]) {
+  test(`timed-out IPC request cancels its action only when opted in: ${cancelOnDisconnect}`, async () => {
+    setDbReady(true);
+    server = new AssistantIpcServer();
+    const methods = (
+      server as unknown as {
+        methods: Map<string, (args: RouteHandlerArgs) => Promise<unknown>>;
+      }
+    ).methods;
+    const setup = Promise.withResolvers<void>();
+    const completed = Promise.withResolvers<void>();
+    let signal: AbortSignal | undefined;
+    let actions = 0;
+    methods.set("test_browser_setup", async (args) => {
+      signal = args.abortSignal;
+      await setup.promise;
+      if (!signal?.aborted) {
+        actions++;
+      }
+      completed.resolve();
+      return { ok: true };
+    });
+    await server.start();
+    try {
+      const result = await cliIpcCall(
+        "test_browser_setup",
+        {},
+        {
+          timeoutMs: 100,
+          cancelOnDisconnect,
+        },
+      );
+      expect(result.timedOut).toBe(true);
+      expect(signal).toBeDefined();
+      await Bun.sleep(20);
+      setup.resolve();
+      await completed.promise;
+      expect(signal?.aborted).toBe(cancelOnDisconnect);
+      expect(actions).toBe(cancelOnDisconnect ? 0 : 1);
+    } finally {
+      setup.resolve();
+    }
+  });
+}

package/src/ipc/assistant-server.ts

@@ -78,6 +78,7 @@ const log = getLogger("assistant-ipc-server");
 // ---------------------------------------------------------------------------
 
 export type IpcRequest = {
+  cancelOnDisconnect?: boolean;
   id: string;
   method: string;
   params?: Record<string, unknown>;
@@ -404,6 +405,16 @@ export class AssistantIpcServer {
       this.abortControllers.set(req.id, abortController);
     }
 
+    const onDisconnect = () => abortController?.abort();
+    if (req.cancelOnDisconnect === true) {
+      socket.once("close", onDisconnect);
+    }
+    const removeDisconnectListener = () => {
+      if (req.cancelOnDisconnect === true) {
+        socket.off("close", onDisconnect);
+      }
+    };
+
     try {
       const handlerArgs = {
         ...injectLocalActorHeader(req.params),
@@ -434,14 +445,17 @@ export class AssistantIpcServer {
               reader,
               this.buildErrorResponse(req.id, err),
             );
-          });
+          })
+          .finally(removeDisconnectListener);
       } else {
+        removeDisconnectListener();
         if (!isIpcStreamingResponse(result)) {
           this.abortControllers.delete(req.id);
         }
         this.sendResult(socket, reader, req.id, result);
       }
     } catch (err) {
+      removeDisconnectListener();
       this.abortControllers.delete(req.id);
       log.warn({ err, method: req.method }, "IPC handler error");
       this.sendResponse(socket, reader, this.buildErrorResponse(req.id, err));

package/src/ipc/cli-client.ts

@@ -60,9 +60,8 @@ export interface CliIpcCallResult<T = unknown> {
   errorDetails?: unknown;
   /**
    * Set when the call was abandoned because `timeoutMs` elapsed with no
-   * response. Distinct from every other `ok: false` shape: the request WAS
-   * delivered and the daemon may still be executing it — closing the client
-   * socket does not abort the handler. Callers that would otherwise retry a
+   * response. The request was delivered and may still be executing unless
+   * `cancelOnDisconnect` was requested. Callers that would otherwise retry a
    * transport failure must not retry this one until the original can no
    * longer be in flight.
    */
@@ -79,7 +78,11 @@ export interface CliIpcCallResult<T = unknown> {
 export async function cliIpcCall<T = unknown>(
   method: string,
   params?: Record<string, unknown>,
-  options?: { timeoutMs?: number; signal?: AbortSignal },
+  options?: {
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    cancelOnDisconnect?: boolean;
+  },
 ): Promise<CliIpcCallResult<T>> {
   if (options?.signal?.aborted) {
     throw options.signal.reason ?? new DOMException("Aborted", "AbortError");
@@ -180,7 +183,12 @@ export async function cliIpcCall<T = unknown>(
 
     socket.on("connect", () => {
       clearTimeout(connectTimer);
-      writeMessage(socket, { id: reqId, method, params });
+      writeMessage(socket, {
+        id: reqId,
+        method,
+        params,
+        ...(opts?.cancelOnDisconnect ? { cancelOnDisconnect: true } : {}),
+      });
 
       callTimer = setTimeout(() => {
         log.debug(

package/src/live-voice/__tests__/live-voice-agent-turn.test.ts

@@ -1043,6 +1043,28 @@ describe("LiveVoiceSession spoken session controls", () => {
     ).toMatchObject({ action: "mute", durationMs: 30_000 });
   });
 
+  test("a declared look sends its look control after the acknowledgement", async () => {
+    const { frames, session, getCallbacks } = createControlsHarness([
+      "look_screen",
+    ]);
+
+    await startReleasedTurn(session, getCallbacks);
+    emitTextDelta(
+      getCallbacks,
+      "Taking a look. What should I focus on? [LOOK:SCREEN]",
+    );
+    emitMessageComplete(getCallbacks);
+    await waitFor(() =>
+      frames.some((frame) => frame.type === "session_control"),
+    );
+
+    expect(
+      frames.find((frame) => frame.type === "session_control"),
+    ).toMatchObject({ action: "look_screen" });
+    // The marker is control, never caption text.
+    expect(assistantDeltaTexts(frames).join("").includes("[LOOK")).toBe(false);
+  });
+
   test("a control the client did not declare is never sent", async () => {
     const { frames, session, getCallbacks } = createControlsHarness(["mute"]);
 

package/src/live-voice/__tests__/live-voice-events.test.ts

@@ -259,9 +259,10 @@ describe("LiveVoiceSession archive and metrics events", () => {
       // has nothing to show and no connection to run, and its capability
       // digest already routes anything needing a tool to the escalated leg
       // (see live-voice-triage-escalate.test.ts). The start frame declares no
-      // client controls, so only the session's own update cadence is taught.
+      // client controls, so only the session's own update cadence and the
+      // no-looks line are taught.
       voiceControlPrompt:
-        'You are speaking in a local live voice session. Keep each reply to one to three spoken sentences. The caller is listening, not reading: lead with the answer, say it in the fewest words that are still complete, and stop. No preamble, no recap of the question, no closing offer of more. Your text is sent directly to a text-to-speech engine. Never use markdown formatting (asterisks, headers, backticks, links), bulleted or numbered lists, or emojis. Write plain conversational text only; if something is naturally a list, say it as one sentence. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work, or a card with an image or a link instead of reading an address out); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. The user can also control this call by asking you. Only when they clearly ask:\n- To hear fewer spoken progress updates while you work (for example "don\'t give me updates so often"), confirm that you will only check in now and then and will tell them when it is done, then end your reply with [UPDATES:FEWER]. If they later want regular updates back, confirm and end with [UPDATES:NORMAL].\nThe marker must be the very last thing in your reply. It is never spoken and does nothing anywhere else.',
+        'You are speaking in a local live voice session. Keep each reply to one to three spoken sentences. The caller is listening, not reading: lead with the answer, say it in the fewest words that are still complete, and stop. No preamble, no recap of the question, no closing offer of more. Your text is sent directly to a text-to-speech engine. Never use markdown formatting (asterisks, headers, backticks, links), bulleted or numbered lists, or emojis. Write plain conversational text only; if something is naturally a list, say it as one sentence. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work, or a card with an image or a link instead of reading an address out); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. The user can also control this call by asking you. Only when they clearly ask:\n- This call cannot turn on a screen share or the camera. If they ask you to look at their screen or at something and you have no other way to see it, say so briefly instead of pretending to look.\n- To hear fewer spoken progress updates while you work (for example "don\'t give me updates so often"), confirm that you will only check in now and then and will tell them when it is done, then end your reply with [UPDATES:FEWER]. If they later want regular updates back, confirm and end with [UPDATES:NORMAL].\nThe marker must be the very last thing in your reply. It is never spoken and does nothing anywhere else.',
     });
     callbacks?.assistant_text_delta?.(makeTextDelta("Hello there."));
     callbacks?.message_complete?.(makeMessageComplete());

package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts

@@ -350,6 +350,25 @@ describe("live-voice turn attribution", () => {
     expect(options.assistantMessageInterface).toBe("macos");
   });
 
+  test("a session opened from the macOS desktop client asks for the desktop skills", async () => {
+    const options = await captureTurnOptions({
+      ...START_FRAME,
+      client: "macos",
+    });
+
+    expect(options.macosDesktopSession).toBe(true);
+  });
+
+  test("a session from any other client does not ask for the desktop skills", async () => {
+    for (const startFrame of [{ ...START_FRAME, client: "ios" }, START_FRAME]) {
+      const options = await captureTurnOptions(
+        startFrame as LiveVoiceClientStartFrame,
+      );
+
+      expect(options.macosDesktopSession).toBeUndefined();
+    }
+  });
+
   test("omits the client when the start frame declares none", async () => {
     const options = await captureTurnOptions(START_FRAME);
 

package/src/live-voice/__tests__/protocol.test.ts

@@ -701,7 +701,15 @@ describe("parseLiveVoiceClientTextFrame", () => {
   test("keeps known session controls and drops unknown ones", () => {
     const result = validateLiveVoiceClientFrame({
       type: "start",
-      sessionControls: ["mute", "look_screen", "end", "mute", 7],
+      sessionControls: [
+        "mute",
+        "look_stop",
+        "look_camera",
+        "end",
+        "mute",
+        7,
+        "fly",
+      ],
       audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
     });
 
@@ -712,14 +720,14 @@ describe("parseLiveVoiceClientTextFrame", () => {
     // A newer client's controls cost it nothing on an older daemon.
     expect(result.frame).toMatchObject({
       type: "start",
-      sessionControls: ["end", "mute"],
+      sessionControls: ["end", "mute", "look_camera", "look_stop"],
     });
   });
 
   test.each([
     ["absent", {}],
     ["not an array", { sessionControls: "end" }],
-    ["all unknown", { sessionControls: ["look_screen"] }],
+    ["all unknown", { sessionControls: ["fly"] }],
   ])(
     "omits sessionControls from the start frame when %s",
     (_label, extra: Record<string, unknown>) => {

package/src/live-voice/__tests__/session-controls.test.ts

@@ -71,6 +71,40 @@ describe("sessionControlTeaching", () => {
     expect(teaching).toContain("Never emit any other bracketed marker.");
   });
 
+  test("a device that can show both asks which one a bare look means", () => {
+    const teaching = sessionControlTeaching(["look_screen", "look_camera"], {});
+
+    expect(teaching).toContain("[LOOK:SCREEN]");
+    expect(teaching).toContain("[LOOK:CAMERA]");
+    expect(teaching).toContain("ask which one instead of guessing");
+  });
+
+  test("a device that can look can be told to stop", () => {
+    const teaching = sessionControlTeaching(["look_screen", "look_stop"], {});
+
+    expect(teaching).toContain("[LOOK:STOP]");
+    expect(
+      requestedSessionControl("Stopping. [LOOK:STOP]", ["look_stop"]),
+    ).toEqual({ action: "look_stop" });
+  });
+
+  test("a device that can show one look needs no question", () => {
+    const teaching = sessionControlTeaching(["look_camera"], {});
+
+    expect(teaching).toContain("[LOOK:CAMERA]");
+    expect(teaching).not.toContain("[LOOK:SCREEN]");
+    expect(teaching).not.toContain("ask which one");
+    expect(teaching).not.toContain("cannot turn on a screen share");
+  });
+
+  // The silent failure in the report this exists for: asked to look, the
+  // assistant neither looked nor said it could not.
+  test("a device that can show neither says so rather than pretending", () => {
+    expect(sessionControlTeaching(["end"], {})).toContain(
+      "This call cannot turn on a screen share or the camera.",
+    );
+  });
+
   test("the front-door leg keeps its verdict tokens", () => {
     expect(sessionControlTeaching(["end"], { frontDoor: true })).not.toContain(
       "Never emit any other bracketed marker.",

package/src/live-voice/live-voice-metrics.ts

@@ -511,6 +511,15 @@ export class LiveVoiceMetricsCollector {
         turnId: turn.turnId,
         finishReason,
         ...aggregateFieldsForTurn(snapshotTurn(turn)),
+        // Stated even at zero, unlike the telemetry field this overrides.
+        // Zero is the single most informative value this count takes: it is
+        // what a turn asked for fewer updates looks like, and equally what a
+        // narrator that is failing every attempt looks like. Left absent, the
+        // two read identically to anyone reading the log, and "the feature is
+        // off" becomes indistinguishable from "the feature did its job".
+        // The wire payload keeps the omission (see optionalTurnFields), so
+        // turns that never engage narration are unchanged for telemetry.
+        progressUpdatesSpoken: turn.progressUpdatesSpoken,
       },
       "Live voice turn latency",
     );

package/src/live-voice/live-voice-session.ts

@@ -5322,6 +5322,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         // as telemetry instead, on `voiceTelemetry` below.
         userMessageInterface: "macos",
         assistantMessageInterface: "macos",
+        ...(this.context.startFrame.client === "macos"
+          ? { macosDesktopSession: true }
+          : {}),
         voiceTelemetry: {
           sessionId: this.context.sessionId,
           ...(this.context.startFrame.client

package/src/live-voice/protocol.ts

@@ -168,7 +168,13 @@ export interface LiveVoiceClientStartFrame {
   readonly sessionControls?: readonly LiveVoiceSessionControl[];
 }
 
-const LIVE_VOICE_SESSION_CONTROLS = ["end", "mute"] as const;
+const LIVE_VOICE_SESSION_CONTROLS = [
+  "end",
+  "mute",
+  "look_screen",
+  "look_camera",
+  "look_stop",
+] as const;
 
 /** A session control a client can carry out on the assistant's behalf. */
 export type LiveVoiceSessionControl =
@@ -633,6 +639,9 @@ export interface LiveVoiceMinimizeRoomServerFrame extends LiveVoiceServerFrameBa
  *   elapses; without, stay muted until the user unmutes. The timer is the
  *   client's: a muted microphone sends silence, so the daemon cannot hear an
  *   "unmute".
+ * - `look_screen`: start showing the call the user's screen.
+ * - `look_camera`: start showing the call what the camera sees.
+ * - `look_stop`: stop showing the call the screen and the camera.
  */
 export interface LiveVoiceSessionControlServerFrame extends LiveVoiceServerFrameBase {
   readonly type: "session_control";

package/src/live-voice/session-controls.ts

@@ -1,6 +1,9 @@
 import {
   END_CALL_MARKER,
   FEWER_UPDATES_MARKER,
+  LOOK_CAMERA_MARKER,
+  LOOK_SCREEN_MARKER,
+  LOOK_STOP_MARKER,
   MUTE_MARKER,
   NORMAL_UPDATES_MARKER,
   parseTerminalSessionControl,
@@ -24,6 +27,9 @@ import type { LiveVoiceSessionControl } from "./protocol.js";
 
 const CLIENT_CONTROL_LINES: Record<LiveVoiceSessionControl, string> = {
   end: `- To end the call (for example "I'm all done" or "okay, I'm gonna go"), say a brief goodbye, then end your reply with ${END_CALL_MARKER}. Being done with a task is not the same as leaving the call; end only when they are leaving.`,
+  look_screen: `- To look at their screen (for example "take a look at my screen" or "can you see what I'm looking at?"), say you are taking a look, then end your reply with ${LOOK_SCREEN_MARKER}. Their screen starts being shared with you once you finish speaking, so you cannot describe it yet: ask what they want you to look at, or say you will take it from their next words.`,
+  look_camera: `- To look through their camera (for example "look at this" or "can you see this?" while they hold something up), say you are taking a look, then end your reply with ${LOOK_CAMERA_MARKER}. The camera turns on once you finish speaking, so you cannot describe what it sees yet: ask them to show you, or say you will take it from their next words.`,
+  look_stop: `- To stop showing you their screen or camera (for example "stop sharing" or "you can stop looking now"), confirm in a few words, then end your reply with ${LOOK_STOP_MARKER}.`,
   mute: `- To mute their microphone (for example "mute for 30 seconds" or "mute yourself, I need to take this"), confirm in a few words, then end your reply with [MUTE:<seconds>] when they gave a duration or ${MUTE_MARKER} when they did not. While muted you cannot hear them, so mention they can unmute from the call controls unless the mute is timed.`,
 };
 
@@ -47,11 +53,33 @@ export function sessionControlTeaching(
   return [
     "The user can also control this call by asking you. Only when they clearly ask:",
     ...controls.map((control) => CLIENT_CONTROL_LINES[control]),
+    ...lookGuidance(controls),
     UPDATES_LINE,
     `The marker must be the very last thing in your reply. It is never spoken and does nothing anywhere else.${leg.frontDoor === true ? "" : " Never emit any other bracketed marker."}`,
   ].join("\n");
 }
 
+/**
+ * What to say about looking beyond the per-control lines: ask which when the
+ * device can do both and the request does not say, and say so plainly when it
+ * can do neither, rather than failing silently.
+ */
+function lookGuidance(controls: readonly LiveVoiceSessionControl[]): string[] {
+  const screen = controls.includes("look_screen");
+  const camera = controls.includes("look_camera");
+  if (screen && camera) {
+    return [
+      `- If they just say "take a look" and it is not clear whether they mean their screen or their camera, ask which one instead of guessing, and use no marker until they answer.`,
+    ];
+  }
+  if (!screen && !camera) {
+    return [
+      "- This call cannot turn on a screen share or the camera. If they ask you to look at their screen or at something and you have no other way to see it, say so briefly instead of pretending to look.",
+    ];
+  }
+  return [];
+}
+
 /** A session control the client carries out, sent as a `session_control` frame. */
 export type ClientSessionControlRequest = Exclude<
   SessionControlRequest,