@vellumai/assistant 0.11.1 → 0.11.2-staging.2

package/src/live-voice/__tests__/live-voice-events.test.ts

@@ -254,13 +254,13 @@ describe("LiveVoiceSession archive and metrics events", () => {
       userMessageInterface: "macos",
       assistantMessageInterface: "macos",
       // Pins the full production control prompt for the FRONT-DOOR leg (the
-      // first leg of every routed turn): the no-UI rule (voice turns are
-      // non-interactive — everything is conveyed in speech) and the shared
-      // no-setup-flows rule (no OAuth/browser flows mid-call). The [-1]
+      // first leg of every routed turn): the speech-first rule (which now
+      // permits a surface, since the room reveals one by itself) and the
+      // shared no-setup-flows rule (no OAuth/browser flows mid-call). The [-1]
       // room-minimize teaching is deliberately absent — only the escalated
       // leg learns it (see live-voice-triage-escalate.test.ts).
       voiceControlPrompt:
-        "You are speaking in a local live voice session. Keep replies brief and conversational. You cannot display cards, forms, or any on-screen UI during the call — convey everything in speech. " +
+        "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. " +
         VOICE_NO_SETUP_FLOWS_RULE,
     });
     callbacks?.assistant_text_delta?.(makeTextDelta("Hello there."));

package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts

@@ -190,7 +190,7 @@ describe("live-voice triage-and-escalate routing", () => {
     expect(escalated?.content).toBe(ESCALATION_CONTINUATION_CONTENT);
   });
 
-  test("the [-1] teaching reaches the escalated leg's prompt but never the front-door leg's", async () => {
+  test("the screen-reveal teaching reaches the escalated leg's prompt but never the front-door leg's", async () => {
     const { starter } = scriptedStartVoiceTurn({
       frontDoor: ["[1] ", "Let me think about that."],
       escalated: ["The detailed answer is 42."],
@@ -205,10 +205,14 @@ describe("live-voice triage-and-escalate routing", () => {
       starter.mock.calls[0]?.[0]?.voiceControlPrompt ?? "";
     const escalatedPrompt =
       starter.mock.calls[1]?.[0]?.voiceControlPrompt ?? "";
-    // The toolless fast leg has nothing to show and its decision rule promises
-    // verbatim speech — it must never learn the marker.
+    // The toolless fast leg has nothing to show, so it is never told the
+    // screen will be revealed.
+    expect(frontDoorPrompt).not.toContain("the overlay minimizes");
+    expect(escalatedPrompt).toContain("the overlay minimizes");
+    // No leg is taught a marker any more: the reveal is decided by whether a
+    // ui tool ran, never by anything the model emits.
     expect(frontDoorPrompt).not.toContain("[-1]");
-    expect(escalatedPrompt).toContain("[-1]");
+    expect(escalatedPrompt).not.toContain("[-1]");
   });
 
   test("the verdict token and any text past the bridge cap never reach the transcript", async () => {

package/src/live-voice/__tests__/live-voice-vad.test.ts

@@ -1207,8 +1207,16 @@ describe("LiveVoiceSession server VAD", () => {
     });
     // The barge-in utterance transcribes to nothing, so no follow-up turn
     // consumes the stash before the interrupt lands.
+    //
+    // "third question" is repeated because finals are handed out by transcriber
+    // creation order, and the session pre-arms a transcriber for the next
+    // utterance. The interrupt below can discard that pre-armed one and re-arm,
+    // which shifts the final utterance onto the following index — with a
+    // 3-entry list it would fall off the end and transcribe to the "utterance
+    // N" placeholder, so the wait for "third question" would time out rather
+    // than fail on the assertion. Both landing spots carry the same text.
     const { frames, session } = createHarness({
-      finals: ["first question", "", "third question"],
+      finals: ["first question", "", "third question", "third question"],
       startVoiceTurn,
       streamTtsAudio,
       spawnBackgroundContinuation,

package/src/live-voice/activity-label.ts

@@ -0,0 +1,169 @@
+/**
+ * Turns a tool call into one short user-facing line for the surfaces that show
+ * a running voice session.
+ *
+ * ## Why the copy lives here
+ *
+ * The iOS Live Activity has two drivers that must carry identical content: the
+ * live-voice socket, and an APNs push the daemon dispatches when the app is
+ * backgrounded and the web layer is suspended. Only one of them runs web code,
+ * so a label composed client-side could not be reproduced by the other. Both
+ * are therefore handed the same string, and this is where it is made.
+ *
+ * That does not contradict the rule that phase wording belongs to the web
+ * layer. The reason phase copy lives there is cadence: the iOS shell ships on
+ * App Store review while the web bundle deploys continuously, so a native
+ * `switch` would fossilize. This module deploys with the daemon, i.e.
+ * continuously, and it is the only layer that knows a tool ran at all.
+ *
+ * ## What makes a good label
+ *
+ * It is read on a Lock Screen, in a glance, by someone who is mid-conversation
+ * and not looking for detail. So: a present-participle phrase, no tool names,
+ * no arguments, no punctuation, and short enough to survive one line. Arguments
+ * are deliberately absent even though {@link summarizeToolInput} could supply
+ * them: a path or a URL on a Lock Screen is unreadable at a glance and can
+ * carry content the user would not choose to show a passer-by.
+ */
+
+/**
+ * The tools that put something on screen worth looking at.
+ *
+ * `app_open` belongs here with the ui-surface tools: it proxies to a connected
+ * client to bring an app up, which is the same thing from the user's side, and
+ * a list keyed on "ui_" would have missed it.
+ *
+ * `ui_dismiss` is deliberately absent, and is handled as its own case: it
+ * retires a surface, and revealing the screen to show the user something that
+ * is no longer there is the opposite of the point.
+ */
+const UI_REVEAL_TOOLS: ReadonlySet<string> = new Set([
+  "ui_show",
+  "ui_update",
+  "app_open",
+]);
+
+/** The tool that takes a surface away again. */
+const UI_DISMISS_TOOL = "ui_dismiss";
+
+/**
+ * Whether running `toolName` leaves something on screen the user should be
+ * shown, which is what makes a live-voice turn reveal the screen behind the
+ * call overlay.
+ */
+export function revealsUiSurface(toolName: string): boolean {
+  return UI_REVEAL_TOOLS.has(toolName);
+}
+
+/** Whether `toolName` takes a surface off the screen. */
+export function dismissesUiSurface(toolName: string): boolean {
+  return toolName === UI_DISMISS_TOOL;
+}
+
+/**
+ * Present-participle phrases for the built-in tools, by exact name.
+ *
+ * Grouped by what the user would say is happening, not by how the tool is
+ * implemented: `bash` and `host_bash` differ in where they run, which is not a
+ * distinction anyone glancing at a Lock Screen is making.
+ */
+const TOOL_ACTIVITY_LABELS: Readonly<Record<string, string>> = {
+  bash: "Running a command",
+  terminal: "Running a command",
+  host_bash: "Running a command",
+  file_read: "Reading a file",
+  host_file_read: "Reading a file",
+  file_write: "Writing a file",
+  host_file_write: "Writing a file",
+  file_edit: "Editing a file",
+  host_file_edit: "Editing a file",
+  host_file_transfer: "Moving a file",
+  file_search: "Searching files",
+  web_search: "Searching the web",
+  web_fetch: "Reading a page",
+  network_request: "Making a request",
+  browser: "Using the browser",
+  computer_use: "Using the computer",
+  memory: "Checking memory",
+  schedule: "Checking the schedule",
+  ui_show: "Putting something on screen",
+  ui_update: "Putting something on screen",
+  app_open: "Opening an app",
+};
+
+/**
+ * Prefix-matched fallbacks, tried in order when a tool has no exact entry.
+ *
+ * The tool vocabulary is open: plugins, MCP servers, and skills all contribute
+ * names this module has never seen, and a new built-in should not have to be
+ * registered here to avoid a blank island. Prefixes cover the families whose
+ * names are structured; {@link GENERIC_ACTIVITY_LABEL} covers the rest.
+ */
+const TOOL_ACTIVITY_LABEL_PREFIXES: ReadonlyArray<readonly [string, string]> = [
+  ["browser_", "Using the browser"],
+  ["computer_", "Using the computer"],
+  ["file_", "Working with files"],
+  ["host_file_", "Working with files"],
+  ["web_", "Looking something up"],
+  ["mcp_", "Using a connected app"],
+  ["skill_", "Running a skill"],
+  ["subagent", "Working on it"],
+];
+
+/**
+ * What an unrecognized tool reads as.
+ *
+ * Deliberately vague rather than derived from the tool's name. A humanized
+ * `slack_conversations_history` is worse than "Working on it": it is longer
+ * than the slot, it leaks an internal vocabulary, and it reads like an error
+ * message. The label's job is to show the turn is alive and doing something,
+ * and this does that job for every tool that will ever exist.
+ */
+export const GENERIC_ACTIVITY_LABEL = "Working on it";
+
+/**
+ * The line to show while `toolName` runs.
+ *
+ * Total, by construction: every tool gets a label, because the alternative is
+ * a surface that goes blank precisely when the assistant is busiest.
+ */
+export function activityLabelForTool(toolName: string): string {
+  const exact = TOOL_ACTIVITY_LABELS[toolName];
+  if (exact !== undefined) {
+    return exact;
+  }
+  for (const [prefix, label] of TOOL_ACTIVITY_LABEL_PREFIXES) {
+    if (toolName.startsWith(prefix)) {
+      return label;
+    }
+  }
+  return GENERIC_ACTIVITY_LABEL;
+}
+
+/**
+ * The line to show while the turn is *waiting* on the user for `toolName`,
+ * rather than running it.
+ *
+ * The distinction matters more here than anywhere else this module is read.
+ * `tool_use_start` fires before the approval gate blocks, so a surface that
+ * showed only {@link activityLabelForTool} would say "Running a command" for
+ * the whole time the turn was in fact doing nothing but waiting — the same
+ * misstatement the spoken narration was fixed to stop making. This is the
+ * island's version of that fix.
+ *
+ * It keeps the tool's own phrase and appends who is being waited on, because
+ * the alternative — a bare "Waiting for your approval" — asks the user to
+ * approve something the surface will not name. That phrase is as specific as
+ * the island's vocabulary gets: no tool names, no arguments, nothing a
+ * passer-by reading a Lock Screen should not see (see the module header).
+ * Anyone wanting the detail has the card itself, one tap away.
+ */
+export function approvalActivityLabel(toolName: string): string {
+  // A confirmation raised outside the tool pipeline (a proxy or network
+  // prompter) has no tool to name, and "Working on it — needs your okay" would
+  // be a worse sentence than the bare one: the turn is not working.
+  if (toolName.length === 0) {
+    return "Needs your okay";
+  }
+  return `${activityLabelForTool(toolName)} — needs your okay`;
+}

package/src/live-voice/live-activity-reporter.ts

@@ -109,21 +109,45 @@ export function phaseForFrame(
  */
 export class LiveActivityReporter {
   private lastPhase: LiveActivityPhase | null = null;
+  private lastDetail = "";
   private ended = false;
 
   constructor(private readonly conversationId: string) {}
 
   /**
-   * Report the phase a frame implies, if it changed. Fire-and-forget: callers
-   * are on the session's send path and must not await this.
+   * Report what a frame implies about the phase or the activity line, if
+   * either changed. Fire-and-forget: callers are on the session's send path
+   * and must not await this.
+   *
+   * Both travel in one dispatch because they are two fields of one content
+   * state: ActivityKit replaces the whole thing per push, so a dispatch that
+   * carried only the field that moved would blank the other.
    */
   report(frame: LiveVoiceServerFramePayload): void {
+    if (this.ended) {
+      return;
+    }
     const phase = phaseForFrame(frame, this.lastPhase);
-    if (phase === null || phase === this.lastPhase || this.ended) {
+    // The session sends this frame only on a change it wants surfaced, and
+    // sends an empty label when the turn stops working, so it is taken
+    // verbatim rather than derived.
+    const detail = frame.type === "activity" ? frame.label : this.lastDetail;
+    const phaseMoved = phase !== null && phase !== this.lastPhase;
+    if (!phaseMoved && detail === this.lastDetail) {
+      return;
+    }
+    if (phase !== null) {
+      this.lastPhase = phase;
+    }
+    this.lastDetail = detail;
+    // An activity line can arrive before any phase has been established (a
+    // turn whose first tool starts before the phase-bearing frame lands).
+    // There is no content state to attach it to yet, and the phase the next
+    // frame sets will carry it.
+    if (this.lastPhase === null) {
       return;
     }
-    this.lastPhase = phase;
-    void this.dispatch(phase, "update");
+    void this.dispatch(this.lastPhase, "update", this.lastDetail);
   }
 
   /**
@@ -139,13 +163,17 @@ export class LiveActivityReporter {
       return;
     }
     this.ended = true;
-    void this.dispatch("ending", "end");
+    // No detail on the way out: whatever the turn was doing, it is not doing
+    // it any more, and this state is the one that lingers on the Lock Screen
+    // through the dismissal window.
+    void this.dispatch("ending", "end", "");
   }
 
   /** `protected` so a test can observe what would be sent without sending it. */
   protected async dispatch(
     phase: LiveActivityPhase,
     event: "update" | "end",
+    detail: string,
   ): Promise<void> {
     try {
       const client = await VellumPlatformClient.create();
@@ -162,6 +190,7 @@ export class LiveActivityReporter {
           conversation_id: this.conversationId,
           phase,
           event,
+          detail,
         }),
       });
       if (!response.ok) {