@vellumai/assistant 0.11.2-staging.1 → 0.11.2-staging.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.2-staging.1",
+  "version": "0.11.2-staging.2",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/live-voice/__tests__/activity-label.test.ts

@@ -12,6 +12,7 @@ import { describe, expect, test } from "bun:test";
 
 import {
   activityLabelForTool,
+  approvalActivityLabel,
   GENERIC_ACTIVITY_LABEL,
 } from "../activity-label.js";
 
@@ -66,3 +67,29 @@ describe("activityLabelForTool", () => {
     }
   });
 });
+
+describe("approvalActivityLabel", () => {
+  // The turn is not running the tool, it is waiting to be allowed to. Saying
+  // the former is the misstatement this exists to stop — and the line has to
+  // name *something*, or the island asks the user to approve a blank.
+  test("keeps the tool's phrase and says who is being waited on", () => {
+    expect(approvalActivityLabel("bash")).toBe(
+      "Running a command — needs your okay",
+    );
+    expect(approvalActivityLabel("web_search")).toBe(
+      "Searching the web — needs your okay",
+    );
+  });
+
+  test("an unrecognized tool still reads as a wait", () => {
+    expect(approvalActivityLabel("some_vendor_tool")).toBe(
+      `${GENERIC_ACTIVITY_LABEL} — needs your okay`,
+    );
+  });
+
+  // A confirmation from a proxy or network prompter has no tool behind it, and
+  // "Working on it — needs your okay" would claim work that is not happening.
+  test("drops the phrase entirely when there is no tool to name", () => {
+    expect(approvalActivityLabel("")).toBe("Needs your okay");
+  });
+});

package/src/live-voice/__tests__/live-voice-agent-turn.test.ts

@@ -154,6 +154,21 @@ async function waitFor(
   throw new Error(message);
 }
 
+/**
+ * The most recent `activity` frame, or `undefined` when none has been sent.
+ * `findLast` is past this project's lib target, so the filter-and-take-last
+ * form stands in for it.
+ */
+function lastActivityFrame(
+  frames: LiveVoiceServerFrame[],
+): Extract<LiveVoiceServerFrame, { type: "activity" }> | undefined {
+  const activity = frames.filter(
+    (frame): frame is Extract<LiveVoiceServerFrame, { type: "activity" }> =>
+      frame.type === "activity",
+  );
+  return activity[activity.length - 1];
+}
+
 function createCapturingTurnStarter(): {
   startVoiceTurn: LiveVoiceTurnStarter;
   getCallbacks: () => VoiceTurnCallbacks | undefined;
@@ -819,6 +834,116 @@ describe("LiveVoiceSession room reveal", () => {
     await flushAsyncCallbacks();
   });
 
+  // `tool_use_start` fires before the approval gate blocks, so without this the
+  // island keeps saying "Running a command" for the whole wait — a claim about
+  // work in flight made at the one moment nothing is in flight. The request id
+  // is what turns that line from accurate into answerable: it is what the Live
+  // Activity's Approve and Deny send back.
+  test("a pending approval publishes an answerable activity line", async () => {
+    const { startVoiceTurn, getCallbacks, announceApprovalPending } =
+      createCapturingTurnStarter();
+    const { frames, session } = createSessionHarness({ startVoiceTurn });
+
+    await session.start();
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => frames.some((frame) => frame.type === "thinking"));
+
+    getCallbacks()?.tool_use_start?.("bash");
+    await waitFor(() =>
+      frames.some(
+        (frame) => frame.type === "activity" && frame.label.length > 0,
+      ),
+    );
+    announceApprovalPending();
+    await waitFor(() =>
+      frames.some(
+        (frame) =>
+          frame.type === "activity" && frame.approvalRequestId === "req-1",
+      ),
+    );
+
+    const waiting = lastActivityFrame(frames);
+    // The tool's own phrase, kept, plus who is being waited on — so the line
+    // beside the buttons names what is being approved without naming the tool
+    // or its arguments to a Lock Screen.
+    expect(waiting).toMatchObject({
+      label: "Running a command — needs your okay",
+      approvalRequestId: "req-1",
+    });
+  });
+
+  // However the decision is made — the card in the app, the daemon's own
+  // timeout, a superseding message — the buttons must go with it. They are
+  // rendered from the id, so retiring the id is what retires them.
+  test("resolving the approval retires the request id", async () => {
+    const {
+      startVoiceTurn,
+      getCallbacks,
+      announceApprovalPending,
+      announceApprovalsResolved,
+    } = createCapturingTurnStarter();
+    const { frames, session } = createSessionHarness({ startVoiceTurn });
+
+    await session.start();
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => frames.some((frame) => frame.type === "thinking"));
+
+    getCallbacks()?.tool_use_start?.("bash");
+    announceApprovalPending();
+    await waitFor(() =>
+      frames.some(
+        (frame) =>
+          frame.type === "activity" && frame.approvalRequestId === "req-1",
+      ),
+    );
+
+    announceApprovalsResolved();
+    await waitFor(
+      () => lastActivityFrame(frames)?.approvalRequestId === undefined,
+    );
+
+    const resumed = lastActivityFrame(frames);
+    // Back to the tool that is now genuinely running, with nothing to answer.
+    expect(resumed).toMatchObject({ label: "Running a command" });
+    expect(resumed?.approvalRequestId).toBeUndefined();
+  });
+
+  // A turn can start and finish other work while it is blocked. Each of those
+  // events republishes the activity line, and a republish composed as if
+  // nothing were pending would take the request id down with it — retiring the
+  // island's buttons while the turn was still waiting on them.
+  test("a parallel tool event mid-wait does not retire the buttons", async () => {
+    const { startVoiceTurn, getCallbacks, announceApprovalPending } =
+      createCapturingTurnStarter();
+    const { frames, session } = createSessionHarness({ startVoiceTurn });
+
+    await session.start();
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => frames.some((frame) => frame.type === "thinking"));
+
+    getCallbacks()?.tool_use_start?.("bash");
+    announceApprovalPending();
+    await waitFor(() =>
+      frames.some(
+        (frame) =>
+          frame.type === "activity" && frame.approvalRequestId === "req-1",
+      ),
+    );
+
+    getCallbacks()?.tool_use_start?.("web_search");
+    getCallbacks()?.tool_result?.({
+      toolName: "web_search",
+      resultPreview: "",
+    });
+    await flushAsyncCallbacks();
+
+    const latest = lastActivityFrame(frames);
+    expect(latest?.approvalRequestId).toBe("req-1");
+    // And still naming the tool the decision is about, not the one that ran
+    // alongside it.
+    expect(latest?.label).toBe("Running a command — needs your okay");
+  });
+
   test("dismissing a surface does not reveal the screen", async () => {
     const { frames, session, getCallbacks } = createEscalatedMarkerHarness();
 

package/src/live-voice/activity-label.ts

@@ -139,3 +139,31 @@ export function activityLabelForTool(toolName: string): string {
   }
   return GENERIC_ACTIVITY_LABEL;
 }
+
+/**
+ * The line to show while the turn is *waiting* on the user for `toolName`,
+ * rather than running it.
+ *
+ * The distinction matters more here than anywhere else this module is read.
+ * `tool_use_start` fires before the approval gate blocks, so a surface that
+ * showed only {@link activityLabelForTool} would say "Running a command" for
+ * the whole time the turn was in fact doing nothing but waiting — the same
+ * misstatement the spoken narration was fixed to stop making. This is the
+ * island's version of that fix.
+ *
+ * It keeps the tool's own phrase and appends who is being waited on, because
+ * the alternative — a bare "Waiting for your approval" — asks the user to
+ * approve something the surface will not name. That phrase is as specific as
+ * the island's vocabulary gets: no tool names, no arguments, nothing a
+ * passer-by reading a Lock Screen should not see (see the module header).
+ * Anyone wanting the detail has the card itself, one tap away.
+ */
+export function approvalActivityLabel(toolName: string): string {
+  // A confirmation raised outside the tool pipeline (a proxy or network
+  // prompter) has no tool to name, and "Working on it — needs your okay" would
+  // be a worse sentence than the bare one: the turn is not working.
+  if (toolName.length === 0) {
+    return "Needs your okay";
+  }
+  return `${activityLabelForTool(toolName)} — needs your okay`;
+}

package/src/live-voice/live-voice-session.ts

@@ -60,6 +60,7 @@ import { createAbortReason } from "../util/abort-reasons.js";
 import { getLogger } from "../util/logger.js";
 import {
   activityLabelForTool,
+  approvalActivityLabel,
   dismissesUiSurface,
   revealsUiSurface,
 } from "./activity-label.js";
@@ -491,10 +492,29 @@ interface ActiveAssistantTurn {
   // map to the same line sends one frame rather than one per call. Empty means
   // the client believes nothing is running, which is also where a turn ends.
   activityLabel: string;
-  // Set while the turn is blocked on a decision the user has to make. Suppresses
-  // progress narration, whose entire vocabulary ("still on it", "almost there")
-  // describes work in flight and would be false here.
-  awaitingApproval: boolean;
+  // The approval id that went out with it, so the de-duplication covers the
+  // whole frame rather than its wording. What the CLIENT believes, as against
+  // `pendingApproval`, which is what is true.
+  publishedApprovalRequestId: string | null;
+  // Set while the turn is blocked on a decision the user has to make, and null
+  // when it is not. Suppresses progress narration, whose entire vocabulary
+  // ("still on it", "almost there") describes work in flight and would be
+  // false here.
+  //
+  // Carries the request id so a surface that is not the app — the Live
+  // Activity's buttons — can answer *that* request rather than whatever is
+  // pending by the time the tap arrives, and the wording that named it, which
+  // is captured once at the reveal rather than recomputed: a parallel op
+  // starting mid-wait moves `currentActivityLabel` on, and the line beside an
+  // Approve button must keep naming the thing being approved.
+  //
+  // The FIRST one, on a turn that leaves two decisions pending at once: the
+  // wait is announced once (see `revealRoomForPendingApproval`) and the pair
+  // resolves as one. A tap answering the first after it has already been
+  // decided is dropped client-side by the id check, which is the safe end of
+  // a rare case — the island never silently answers a request other than the
+  // one it named.
+  pendingApproval: { requestId: string; label: string } | null;
   // A tts_audio frame actually went out to the client — latches on the first
   // forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
   ttsAudioStarted: boolean;
@@ -2349,6 +2369,54 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     return "";
   }
 
+  /**
+   * The line for a turn that is waiting on a decision about its newest running
+   * op.
+   *
+   * The approval gate sits behind `tool_use_start`, so the tool being waited on
+   * is the one the turn last said it was running — which is precisely why the
+   * wait has to be published at all: without it the surfaces keep showing that
+   * tool as *running* for the whole time it is doing nothing of the kind.
+   * Falls back to the tool-less phrase when no op is open, which is the case
+   * for a confirmation raised by a prompter outside the tool pipeline.
+   *
+   * Read once, at the reveal — see `pendingApproval` for why it is then held
+   * rather than recomputed.
+   */
+  private pendingApprovalLabel(turn: ActiveAssistantTurn): string {
+    for (let i = turn.progress.ops.length - 1; i >= 0; i -= 1) {
+      const op = turn.progress.ops[i];
+      if (op !== undefined && op.completedAtMs === undefined) {
+        return approvalActivityLabel(op.toolName);
+      }
+    }
+    return approvalActivityLabel("");
+  }
+
+  /**
+   * Publish whatever the turn's activity line should be right now: the
+   * decision it is waiting on if it is waiting, and its newest running tool if
+   * it is not.
+   *
+   * The single entry point for every caller that would otherwise reach for
+   * `currentActivityLabel` directly. A turn can start and finish other ops
+   * while it is blocked on an approval — a parallel `tool_use_start` or
+   * `tool_result` lands mid-wait — and each of those would otherwise publish a
+   * line composed as if nothing were pending, taking the request id down with
+   * it and retiring the island's buttons while the turn was still waiting.
+   */
+  private refreshActivity(turn: ActiveAssistantTurn): void {
+    if (turn.pendingApproval !== null) {
+      this.publishActivity(
+        turn,
+        turn.pendingApproval.label,
+        turn.pendingApproval.requestId,
+      );
+      return;
+    }
+    this.publishActivity(turn, this.currentActivityLabel(turn));
+  }
+
   /**
    * Open the room because a decision is waiting behind it.
    *
@@ -2362,12 +2430,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
    * The latch is cleared as well, so a turn that also showed a surface does
    * not send a second minimize once its speech ends; the room is already open.
    */
-  private revealRoomForPendingApproval(turn: ActiveAssistantTurn): void {
-    if (turn.awaitingApproval) {
+  private revealRoomForPendingApproval(
+    turn: ActiveAssistantTurn,
+    requestId: string,
+  ): void {
+    if (turn.pendingApproval !== null) {
       return;
     }
-    turn.awaitingApproval = true;
+    turn.pendingApproval = {
+      requestId,
+      label: this.pendingApprovalLabel(turn),
+    };
     turn.minimizeRequested = false;
+    // Say what the turn is actually doing on the surfaces that are not the
+    // app. Without this the island keeps showing the tool as running for the
+    // whole wait — and it is the surface most likely to be the only one the
+    // user can see, since the case this exists for is a phone put down.
+    // Carrying the request id is what makes the line answerable there rather
+    // than merely accurate.
+    this.refreshActivity(turn);
     void this.sendFrame(
       { type: "minimize_room", turnId: turn.turnId },
       () => !this.isClosed,
@@ -2381,7 +2462,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
 
   /** Clear the wait once a decision lands, so the turn narrates normally again. */
   private clearAwaitingApproval(turn: ActiveAssistantTurn): void {
-    turn.awaitingApproval = false;
+    turn.pendingApproval = null;
+    // Put the activity line back to whatever the turn resumed doing, and
+    // retire the request id with it, so the island's Approve/Deny buttons go
+    // away the moment the decision is no longer the user's to make — including
+    // when it was made somewhere else entirely (the card in the app, the
+    // 45-second fallback, a superseding message).
+    this.refreshActivity(turn);
   }
 
   /**
@@ -2394,14 +2481,37 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
    *
    * Fire-and-forget. An activity label is a flourish, and nothing about the
    * conversation may wait on one.
+   *
+   * Callers whose line depends on turn state go through
+   * {@link refreshActivity}; this is called directly only to clear the line
+   * outright, which a cancelled or finished turn does regardless of what it
+   * was waiting on.
    */
-  private publishActivity(turn: ActiveAssistantTurn, label: string): void {
-    if (turn.activityLabel === label) {
+  private publishActivity(
+    turn: ActiveAssistantTurn,
+    label: string,
+    approvalRequestId?: string,
+  ): void {
+    // De-duplicated on the request id as well as the wording. The two move
+    // independently: a wait can be entered and left without the tool line
+    // changing at all, and a label-only check would swallow the frame that
+    // retires the approval — leaving the island's buttons up with nothing
+    // behind them.
+    if (
+      turn.activityLabel === label &&
+      turn.publishedApprovalRequestId === (approvalRequestId ?? null)
+    ) {
       return;
     }
     turn.activityLabel = label;
+    turn.publishedApprovalRequestId = approvalRequestId ?? null;
     void this.sendFrame(
-      { type: "activity", turnId: turn.turnId, label },
+      {
+        type: "activity",
+        turnId: turn.turnId,
+        label,
+        ...(approvalRequestId !== undefined ? { approvalRequestId } : {}),
+      },
       () => !this.isClosed,
     );
   }
@@ -3499,7 +3609,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       ttsDone: false,
       minimizeRequested: false,
       activityLabel: "",
-      awaitingApproval: false,
+      publishedApprovalRequestId: null,
+      pendingApproval: null,
       ttsAudioStarted: false,
       finalized: false,
       speculativePending: opts?.speculative === true,
@@ -3701,8 +3812,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
           ...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
         }),
-        onApprovalPending: () => {
-          this.revealRoomForPendingApproval(activeTurn);
+        onApprovalPending: (requestId) => {
+          this.revealRoomForPendingApproval(activeTurn, requestId);
         },
         onApprovalsResolved: () => {
           this.clearAwaitingApproval(activeTurn);
@@ -3932,7 +4043,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
             });
             current.progress.opsSinceNarration += 1;
             current.progress.stateEpoch += 1;
-            this.publishActivity(current, this.currentActivityLabel(current));
+            this.refreshActivity(current);
             log.debug({ turnId, toolName }, "Live voice turn started tool use");
             // Definitive tool use means the turn is guaranteed slow: speak
             // the floor-holding ack now instead of waiting out the
@@ -4007,7 +4118,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
                 current.minimizeRequested = false;
               }
             }
-            this.publishActivity(current, this.currentActivityLabel(current));
+            this.refreshActivity(current);
             this.maybeNarrateProgress(current, trigger);
           },
         },
@@ -4382,7 +4493,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       // Nothing is in flight while a decision is pending, so every phrase
       // narration has would be a lie about who the call is waiting on. The
       // turn says so once, when it starts waiting, and is quiet after that.
-      !turn.awaitingApproval &&
+      turn.pendingApproval === null &&
       this.turnAudioIdle(turn)
     );
   }

package/src/live-voice/protocol.ts

@@ -236,6 +236,18 @@ export interface LiveVoiceActivityServerFrame extends LiveVoiceServerFrameBase {
   readonly type: "activity";
   readonly turnId: string;
   readonly label: string;
+  /**
+   * The confirmation this turn is blocked on, when the label describes a wait
+   * rather than work in flight. Absent otherwise.
+   *
+   * It travels so that a surface outside the app — the Live Activity's
+   * Approve/Deny buttons — can answer the request it was drawn against rather
+   * than whatever is pending by the time the press lands. Content on a Lock
+   * Screen can be seconds old, and a decision is the one thing that must not
+   * be re-pointed when it is: the id lets a client drop a press aimed at a
+   * request already answered, timed out, or superseded.
+   */
+  readonly approvalRequestId?: string;
 }
 
 export interface LiveVoiceAssistantTextDeltaServerFrame extends LiveVoiceServerFrameBase {