@vellumai/cli 0.11.6 → 0.11.7-staging.2

package/src/lib/live-voice/client.ts

@@ -0,0 +1,363 @@
+/**
+ * A live-voice client that never opens a microphone.
+ *
+ * One instance drives one session. It opens the gateway WebSocket, sends
+ * `start` with `textInput: true`, and from then on takes turns by typing them:
+ * a `text` frame joins the pipeline at the point speech-to-text would have
+ * handed over a finished transcript, so the reply streams back through the
+ * same turn runner, the same segmented TTS, and the same barge-in as a spoken
+ * one. Only the capture half is skipped.
+ *
+ * Modeled on `clients/web/src/domains/chat/voice/live-voice/live-voice-client.ts`
+ * and deliberately narrower: no audio upload, no push-to-talk, no mid-session
+ * config updates. After `end()` or a failure the instance is terminal.
+ */
+
+import {
+  LIVE_VOICE_AUDIO_FORMAT,
+  MAX_TEXT_TURN_CHARS,
+  parseServerFrame,
+  type LiveVoiceActivityFrame,
+  type LiveVoiceAssistantTextDeltaFrame,
+  type LiveVoiceReadyFrame,
+  type LiveVoiceStartFrame,
+  type LiveVoiceTtsAudioFrame,
+} from "./protocol.js";
+
+/** Fail the session if no `ready` frame arrives within this window. */
+const CONNECT_TIMEOUT_MS = 10_000;
+
+export interface LiveVoiceClientEvents {
+  /** The session is open. `textInput` says whether typed turns are accepted. */
+  ready(frame: LiveVoiceReadyFrame): void;
+  /** A chunk of the assistant's reply text, as the model produces it. */
+  textDelta(frame: LiveVoiceAssistantTextDeltaFrame): void;
+  /** A chunk of spoken audio. PCM16 mono at the frame's own sample rate. */
+  audio(frame: LiveVoiceTtsAudioFrame): void;
+  /** The turn's speech is fully sent. */
+  turnDone(turnId: string): void;
+  /** The turn was aborted; buffered audio for it must be dropped. */
+  turnCancelled(turnId: string): void;
+  /** A named piece of work the turn is doing ("" means idle again). */
+  activity(frame: LiveVoiceActivityFrame): void;
+  /** The daemon refused a typed turn. The session survives. */
+  textTurnRejected(reason: "unsupported" | "busy", message: string): void;
+  /** A recoverable mid-session error. The session survives. */
+  warning(message: string): void;
+  /** The session is over. `error` is set when it ended badly. */
+  closed(error?: string): void;
+}
+
+type EventName = keyof LiveVoiceClientEvents;
+
+export interface CliLiveVoiceClientOptions {
+  readonly url: string;
+  readonly token: string;
+  readonly conversationId?: string;
+  /** Override the socket factory (tests). */
+  readonly webSocketFactory?: (url: string, token: string) => WebSocket;
+  /** Override the connect timeout (tests). */
+  readonly connectTimeoutMs?: number;
+}
+
+type SessionState = "idle" | "connecting" | "active" | "closed";
+
+export class CliLiveVoiceClient {
+  private state: SessionState = "idle";
+  private ws: WebSocket | null = null;
+  private connectTimer: ReturnType<typeof setTimeout> | null = null;
+  private textInputSupported = false;
+  private audioInputLive = true;
+
+  private readonly options: CliLiveVoiceClientOptions;
+  private readonly listeners = new Map<
+    EventName,
+    Set<(...a: never[]) => void>
+  >();
+
+  constructor(options: CliLiveVoiceClientOptions) {
+    this.options = options;
+  }
+
+  on<E extends EventName>(event: E, handler: LiveVoiceClientEvents[E]): void {
+    const set = this.listeners.get(event) ?? new Set();
+    set.add(handler as (...a: never[]) => void);
+    this.listeners.set(event, set);
+  }
+
+  private emit<E extends EventName>(
+    event: E,
+    ...args: Parameters<LiveVoiceClientEvents[E]>
+  ): void {
+    for (const handler of this.listeners.get(event) ?? []) {
+      (handler as (...a: unknown[]) => void)(...args);
+    }
+  }
+
+  /** Whether this daemon accepts typed turns (set from the `ready` echo). */
+  get supportsTextInput(): boolean {
+    return this.textInputSupported;
+  }
+
+  /** Whether the session's speech-to-text leg came up. */
+  get hasAudioInput(): boolean {
+    return this.audioInputLive;
+  }
+
+  /**
+   * Open the socket. Resolves once it is opening; readiness arrives as the
+   * `ready` event, and failure as `closed` with an error.
+   */
+  connect(): void {
+    if (this.state !== "idle") {
+      return;
+    }
+    this.state = "connecting";
+
+    let ws: WebSocket;
+    try {
+      ws = this.options.webSocketFactory
+        ? this.options.webSocketFactory(this.options.url, this.options.token)
+        : openAuthenticatedSocket(this.options.url, this.options.token);
+    } catch (err) {
+      this.fail(
+        err instanceof Error
+          ? err.message
+          : "Failed to open the live-voice WebSocket",
+      );
+      return;
+    }
+    this.ws = ws;
+    ws.binaryType = "arraybuffer";
+    ws.onopen = () => this.handleOpen();
+    ws.onmessage = (event: MessageEvent) => this.handleMessage(event);
+    ws.onerror = () => this.fail("Live-voice WebSocket error");
+    ws.onclose = (event: CloseEvent) => this.handleClose(event);
+
+    this.connectTimer = setTimeout(() => {
+      if (this.state === "connecting") {
+        this.fail(
+          `Live-voice connection timed out after ${this.connectTimeoutMs()}ms`,
+        );
+      }
+    }, this.connectTimeoutMs());
+  }
+
+  /**
+   * Take a turn by typing it. Returns whether the frame went out.
+   *
+   * False means the turn was not taken at all and the caller must say so: the
+   * session is not active, the daemon predates typed turns, or the text is
+   * empty or over the cap. A turn the daemon *receives* but refuses (it is
+   * mid-reply) comes back later as `textTurnRejected`, not as false here.
+   */
+  sendText(text: string): boolean {
+    if (this.state !== "active" || !this.textInputSupported) {
+      return false;
+    }
+    const trimmed = text.trim();
+    if (trimmed.length === 0 || trimmed.length > MAX_TEXT_TURN_CHARS) {
+      return false;
+    }
+    return this.trySend(JSON.stringify({ type: "text", text: trimmed }));
+  }
+
+  /** Cut off the assistant mid-reply. */
+  interrupt(): void {
+    if (this.state !== "active") {
+      return;
+    }
+    this.trySend(JSON.stringify({ type: "interrupt" }));
+  }
+
+  /** End the session gracefully, then close the socket. Idempotent. */
+  end(): void {
+    if (this.state !== "connecting" && this.state !== "active") {
+      return;
+    }
+    // Best-effort: trySend no-ops unless the socket is OPEN, so a quick cancel
+    // during connect skips the (impossible) send and still closes cleanly.
+    this.trySend(JSON.stringify({ type: "end" }));
+    this.teardown();
+    this.emit("closed", undefined);
+  }
+
+  private connectTimeoutMs(): number {
+    return this.options.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;
+  }
+
+  private handleOpen(): void {
+    if (this.state !== "connecting") {
+      return;
+    }
+    const start: LiveVoiceStartFrame = {
+      type: "start",
+      audio: LIVE_VOICE_AUDIO_FORMAT,
+      textInput: true,
+      ...(this.options.conversationId
+        ? { conversationId: this.options.conversationId }
+        : {}),
+    };
+    this.trySend(JSON.stringify(start));
+  }
+
+  private handleMessage(event: MessageEvent): void {
+    if (this.state === "closed") {
+      return;
+    }
+    // Every server payload is JSON text; binary is not part of the contract.
+    if (typeof event.data !== "string") {
+      return;
+    }
+    const frame = parseServerFrame(event.data);
+    switch (frame.type) {
+      case "ready": {
+        if (this.state !== "connecting") {
+          return;
+        }
+        this.clearConnectTimer();
+        this.state = "active";
+        this.textInputSupported = frame.textInput === true;
+        // Absent means yes: a daemon predating the field refuses any session it
+        // cannot transcribe, so every session it readies can hear.
+        this.audioInputLive = frame.audioInput !== false;
+        this.emit("ready", frame);
+        return;
+      }
+      case "busy":
+        this.fail(
+          "The assistant already has a voice session open " +
+            `(${frame.activeSessionId}).`,
+        );
+        return;
+      case "assistant_text_delta":
+        this.emit("textDelta", frame);
+        return;
+      case "tts_audio":
+        this.emit("audio", frame);
+        return;
+      case "tts_done":
+        this.emit("turnDone", frame.turnId);
+        return;
+      case "turn_cancelled":
+        this.emit("turnCancelled", frame.turnId);
+        return;
+      case "activity":
+        this.emit("activity", frame);
+        return;
+      case "thinking":
+        return;
+      case "error": {
+        // `frameType` names what the error is about. A typed turn's rejection
+        // must not be filed anywhere else: `unknown_type` here is
+        // byte-identical to the rejection an older daemon gives every optional
+        // frame, and a `recoverable` refusal is a busy assistant rather than a
+        // transient blip, so both would be misread by the generic paths below.
+        if (frame.frameType === "text") {
+          this.emit(
+            "textTurnRejected",
+            frame.code === "unknown_type" ? "unsupported" : "busy",
+            frame.message,
+          );
+          return;
+        }
+        if (frame.recoverable === true && this.state === "active") {
+          this.emit("warning", frame.message);
+          return;
+        }
+        this.fail(frame.message);
+        return;
+      }
+      case "unhandled":
+      case "malformed":
+        // Capture-side frames a mic-less session never acts on, and additions
+        // from a newer daemon. Ignoring them is what keeps protocol growth
+        // from breaking an older CLI.
+        return;
+    }
+  }
+
+  private handleClose(event: CloseEvent): void {
+    if (this.state === "closed") {
+      return;
+    }
+    if (this.state === "connecting") {
+      this.fail(closeReason(event));
+      return;
+    }
+    this.teardown();
+    this.emit("closed", undefined);
+  }
+
+  private trySend(data: string): boolean {
+    const ws = this.ws;
+    // Sending on a CONNECTING socket throws, so guard on readyState rather
+    // than on session state alone.
+    if (ws && ws.readyState === 1) {
+      ws.send(data);
+      return true;
+    }
+    return false;
+  }
+
+  private fail(message: string): void {
+    if (this.state === "closed") {
+      return;
+    }
+    this.teardown();
+    this.emit("closed", message);
+  }
+
+  private teardown(): void {
+    this.state = "closed";
+    this.clearConnectTimer();
+    const ws = this.ws;
+    this.ws = null;
+    if (ws) {
+      ws.onopen = null;
+      ws.onmessage = null;
+      ws.onerror = null;
+      ws.onclose = null;
+      ws.close();
+    }
+  }
+
+  private clearConnectTimer(): void {
+    if (this.connectTimer !== null) {
+      clearTimeout(this.connectTimer);
+      this.connectTimer = null;
+    }
+  }
+}
+
+/**
+ * Open the gateway socket with the guardian token in an `Authorization`
+ * header.
+ *
+ * The gateway takes the credential either as a header or as a `?token=` query
+ * parameter. The browser client has to use the query form, because `WebSocket`
+ * there cannot set headers, but a CLI can and should: a query parameter lands in
+ * process listings, shell history, and gateway access logs, and this token is
+ * the bound guardian's.
+ */
+function openAuthenticatedSocket(url: string, token: string): WebSocket {
+  return new WebSocket(url, {
+    headers: { Authorization: `Bearer ${token}` },
+  } as unknown as string[]) as WebSocket;
+}
+
+function closeReason(event: CloseEvent): string {
+  // 1008/1011 with no reason is what an upgrade rejected at the gateway looks
+  // like by the time it reaches a WebSocket client: the 401/403 body never
+  // survives the handshake. Name the likely cause rather than printing a bare
+  // numeric code the user cannot act on.
+  const detail = event.reason?.trim();
+  if (detail) {
+    return `Live-voice session closed before it opened: ${detail}`;
+  }
+  return (
+    "Live-voice session closed before it opened " +
+    `(code ${event.code}). The gateway may have rejected the credential: ` +
+    "try 'vellum wake' to lease a fresh one."
+  );
+}

package/src/lib/live-voice/connection.ts

@@ -0,0 +1,133 @@
+/**
+ * Resolving the live-voice WebSocket URL and the credential to open it with.
+ *
+ * The endpoint is the **gateway's** `/v1/live-voice`, not the runtime's. That
+ * distinction is the whole of the auth story: the runtime's copy demands a
+ * gateway service token, which no client can mint, while the gateway's copy
+ * authenticates the caller and then mints that service token itself on the way
+ * upstream (`gateway/src/http/routes/live-voice-websocket.ts`).
+ *
+ * What the gateway wants is an actor edge JWT belonging to the *bound
+ * guardian*. Live voice is a guardian-only surface, because the daemon stamps
+ * each voice turn with the guardian's trust context and the proxy replaces the
+ * caller's identity upstream, so the pin at the gateway is the only place a
+ * non-guardian can be stopped.
+ *
+ * The CLI already holds exactly that credential and has since it first paired:
+ * `leaseGuardianToken` POSTs `/v1/guardian/init`, which resolves the *vellum*
+ * guardian principal and mints an access token subject
+ * `actor:<assistantId>:<guardianPrincipalId>`, the same principal
+ * `findVellumGuardian()` returns on the other side of the check. So the pin
+ * passes by construction, and there is nothing new to provision here.
+ */
+
+import {
+  formatAssistantLookupError,
+  formatAssistantReference,
+  lookupAssistantByIdentifier,
+  resolveTargetAssistant,
+} from "../assistant-config.js";
+import {
+  guardianTokenDueForRenewal,
+  loadGuardianToken,
+  refreshGuardianToken,
+} from "../guardian-token.js";
+
+export interface LiveVoiceConnection {
+  /** `ws(s)://…/v1/live-voice` on the assistant's gateway. */
+  readonly url: string;
+  /** Guardian access token, sent as `Authorization: Bearer`. */
+  readonly token: string;
+  readonly assistantId: string;
+  /** `name (id)` when they differ, else the id, for user-facing output. */
+  readonly reference: string;
+}
+
+export class LiveVoiceConnectionError extends Error {}
+
+/**
+ * Resolve the target assistant, its gateway URL, and a usable guardian token.
+ *
+ * Refreshes the token when it is already due for renewal, so a long session
+ * does not open on a credential that expires mid-conversation. A token that is
+ * merely *rejected* is not refreshed here, because that would disclose the
+ * long-lived refresh credential on demand, which matches how `AssistantClient`
+ * gates its own reactive refresh.
+ */
+export async function resolveLiveVoiceConnection(
+  assistantIdArg?: string,
+): Promise<LiveVoiceConnection> {
+  // An explicit target is looked up by id first and display name second, and
+  // an ambiguous display name is an error naming the candidates, per the
+  // package's assistant-targeting convention. With no target,
+  // `resolveTargetAssistant` applies the usual active/sole-entry fallback.
+  const entry = assistantIdArg
+    ? resolveNamedAssistant(assistantIdArg)
+    : resolveTargetAssistant();
+
+  // Platform-cloud assistants authenticate with a platform session token
+  // (`X-Session-Token`), and the live-voice upgrade accepts only an actor edge
+  // JWT or a velay attestation, neither of which the CLI has for a cloud
+  // instance. Say so plainly rather than opening a socket that 401s.
+  if (entry.cloud === "vellum") {
+    throw new LiveVoiceConnectionError(
+      `${formatAssistantReference(entry)} is a Vellum cloud assistant. ` +
+        "Voice sessions are supported for local and self-hosted assistants only.",
+    );
+  }
+
+  const baseUrl = (entry.localUrl || entry.runtimeUrl).replace(/\/+$/, "");
+  let parsed: URL;
+  try {
+    parsed = new URL(baseUrl);
+  } catch {
+    throw new LiveVoiceConnectionError(
+      `Assistant '${entry.assistantId}' has an unusable gateway URL: ${baseUrl}`,
+    );
+  }
+  parsed.protocol = parsed.protocol === "https:" ? "wss:" : "ws:";
+  parsed.pathname = "/v1/live-voice";
+
+  const token = await resolveGuardianAccessToken(entry.assistantId, baseUrl);
+  if (!token) {
+    throw new LiveVoiceConnectionError(
+      `No credential for '${entry.assistantId}'. Run 'vellum wake ` +
+        `${entry.assistantId}' to lease one.`,
+    );
+  }
+
+  return {
+    url: parsed.toString(),
+    token,
+    assistantId: entry.assistantId,
+    reference: formatAssistantReference(entry),
+  };
+}
+
+function resolveNamedAssistant(identifier: string) {
+  const result = lookupAssistantByIdentifier(identifier);
+  if (result.status !== "found") {
+    throw new LiveVoiceConnectionError(
+      formatAssistantLookupError(identifier, result),
+    );
+  }
+  return result.entry;
+}
+
+async function resolveGuardianAccessToken(
+  assistantId: string,
+  gatewayUrl: string,
+): Promise<string | undefined> {
+  const stored = loadGuardianToken(assistantId);
+  if (!stored?.accessToken) {
+    return undefined;
+  }
+  if (!guardianTokenDueForRenewal(stored)) {
+    return stored.accessToken;
+  }
+  const refreshed = await refreshGuardianToken(gatewayUrl, assistantId);
+  // A failed refresh is not fatal on its own: the stored token may still have
+  // life left (`refreshAfter` deliberately precedes expiry), so try it rather
+  // than refusing to open a session that would have worked.
+  return refreshed?.accessToken ?? stored.accessToken;
+}

package/src/lib/live-voice/pcm-player.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  isOnPath,
+  PcmPlayer,
+  wrapPcmAsWav,
+  type PlayerCommand,
+} from "./pcm-player.js";
+
+describe("wrapPcmAsWav", () => {
+  test("writes a canonical 44-byte mono PCM16 header for the given rate", () => {
+    const pcm = Buffer.alloc(160, 7);
+    const wav = wrapPcmAsWav(pcm, 24000);
+
+    expect(wav.length).toBe(44 + pcm.length);
+    expect(wav.subarray(0, 4).toString()).toBe("RIFF");
+    expect(wav.readUInt32LE(4)).toBe(36 + pcm.length);
+    expect(wav.subarray(8, 12).toString()).toBe("WAVE");
+    expect(wav.readUInt16LE(20)).toBe(1); // PCM
+    expect(wav.readUInt16LE(22)).toBe(1); // mono
+    expect(wav.readUInt32LE(24)).toBe(24000);
+    expect(wav.readUInt32LE(28)).toBe(48000); // byte rate = rate * 2
+    expect(wav.readUInt16LE(32)).toBe(2); // block align
+    expect(wav.readUInt16LE(34)).toBe(16); // bits per sample
+    expect(wav.readUInt32LE(40)).toBe(pcm.length);
+    expect(wav.subarray(44)).toEqual(pcm);
+  });
+
+  test("tracks the sample rate the frame declared, not a fixed one", () => {
+    expect(wrapPcmAsWav(Buffer.alloc(2), 16000).readUInt32LE(24)).toBe(16000);
+    expect(wrapPcmAsWav(Buffer.alloc(2), 44100).readUInt32LE(24)).toBe(44100);
+  });
+});
+
+describe("PcmPlayer playback lifetime", () => {
+  /** A player process whose lifetime a test controls. */
+  const sleeper = (seconds: number): PlayerCommand => ({
+    mode: "streaming",
+    name: "/bin/sh",
+    args: () => ["-c", `sleep ${seconds}`],
+  });
+
+  test("finish() waits for the player to exit, not for the last byte", async () => {
+    const player = new PcmPlayer(sleeper(0.4));
+    player.write(Buffer.alloc(64), 24000);
+    expect(player.isPlaying).toBe(true);
+
+    const started = performance.now();
+    await player.finish();
+    const elapsed = performance.now() - started;
+
+    // Handing the prompt back at the last byte would resolve immediately; the
+    // point of the await is that the speaker has actually gone quiet.
+    expect(elapsed).toBeGreaterThan(250);
+    expect(player.isPlaying).toBe(false);
+    player.dispose();
+  });
+
+  test("stop() cuts playback off and releases a pending finish()", async () => {
+    const player = new PcmPlayer(sleeper(5));
+    player.write(Buffer.alloc(64), 24000);
+
+    const started = performance.now();
+    const pending = player.finish();
+    player.stop();
+    await pending;
+
+    // A finish() that outlived its interrupt would wedge the prompt for the
+    // rest of the session, which is exactly the barge-in case.
+    expect(performance.now() - started).toBeLessThan(1000);
+    expect(player.isPlaying).toBe(false);
+    player.dispose();
+  });
+
+  test("finish() resolves immediately when there is no player", async () => {
+    const player = new PcmPlayer(null);
+    player.write(Buffer.alloc(64), 24000);
+    await player.finish();
+    expect(player.mode).toBe("silent");
+  });
+
+  test("a player that cannot be spawned does not wedge finish()", async () => {
+    const missing: PlayerCommand = {
+      mode: "streaming",
+      name: "definitely-not-a-real-player-binary",
+      args: () => [],
+    };
+    const player = new PcmPlayer(missing);
+    player.write(Buffer.alloc(64), 24000);
+    await player.finish();
+    expect(player.isPlaying).toBe(false);
+    player.dispose();
+  });
+});
+
+describe("isOnPath", () => {
+  test("finds a binary that exists and rejects one that does not", () => {
+    expect(isOnPath("sh")).toBe(true);
+    expect(isOnPath("definitely-not-a-real-binary-xyz")).toBe(false);
+  });
+});