@vellumai/assistant 0.6.5 → 0.6.6

package/src/providers/speech-to-text/deepgram-realtime.ts

@@ -53,6 +53,17 @@ const DEFAULT_CONNECT_TIMEOUT_MS = 10_000;
  */
 const DEFAULT_INACTIVITY_TIMEOUT_MS = 30_000;
 
+/**
+ * Default interval (ms) for emitting Deepgram `KeepAlive` control frames
+ * during silent stretches. Deepgram's server-side timeout closes the
+ * socket if no real audio content arrives for ~10s; raw silence PCM does
+ * not reset that timer, only an explicit `{"type":"KeepAlive"}` message
+ * does. Sending one every 5s keeps the socket alive through arbitrary
+ * pauses (think: 1:1 voice mode while the user is thinking) without any
+ * meaningful bandwidth cost.
+ */
+const DEFAULT_KEEPALIVE_INTERVAL_MS = 5_000;
+
 /**
  * Maximum WebSocket bufferedAmount (bytes) before sendAudio applies
  * backpressure by dropping frames. This prevents unbounded memory growth
@@ -87,6 +98,13 @@ export interface DeepgramRealtimeOptions {
   connectTimeoutMs?: number;
   /** Inactivity timeout in milliseconds. Default: 30_000. */
   inactivityTimeoutMs?: number;
+  /**
+   * Interval (ms) between Deepgram `KeepAlive` control frames sent during
+   * silent stretches. Default: 5_000. Set to 0 to disable (not recommended
+   * outside tests — the server-side socket will close after ~10s of
+   * silence).
+   */
+  keepaliveIntervalMs?: number;
   /** Audio sample rate in Hz (default: 16000). Passed through from the client WebSocket connection. */
   sampleRate?: number;
   /**
@@ -222,6 +240,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
   private readonly baseUrl: string;
   private readonly connectTimeoutMs: number;
   private readonly inactivityTimeoutMs: number;
+  private readonly keepaliveIntervalMs: number;
   private readonly sampleRate: number;
   /**
    * Whether speaker diarization is requested. Forwarded to the Deepgram
@@ -248,6 +267,13 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
   /** Close grace timer handle. */
   private closeGraceTimer: ReturnType<typeof setTimeout> | null = null;
 
+  /**
+   * Periodic keepalive timer. Fires every {@link keepaliveIntervalMs} while
+   * the socket is open and emits a Deepgram `KeepAlive` control frame so
+   * silent stretches do not trip Deepgram's server-side inactivity close.
+   */
+  private keepaliveTimer: ReturnType<typeof setInterval> | null = null;
+
   constructor(apiKey: string, options: DeepgramRealtimeOptions = {}) {
     this.apiKey = apiKey;
     this.model = options.model ?? DEFAULT_MODEL;
@@ -260,6 +286,8 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
       options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
     this.inactivityTimeoutMs =
       options.inactivityTimeoutMs ?? DEFAULT_INACTIVITY_TIMEOUT_MS;
+    this.keepaliveIntervalMs =
+      options.keepaliveIntervalMs ?? DEFAULT_KEEPALIVE_INTERVAL_MS;
     this.sampleRate = options.sampleRate ?? 16_000;
     this.diarize = options.diarize ?? false;
   }
@@ -329,6 +357,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
     // the active session lifetime.
     this.attachSessionHandlers(ws);
     this.resetInactivityTimer();
+    this.startKeepaliveTimer();
 
     log.info("Deepgram realtime session opened");
   }
@@ -644,6 +673,34 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
       clearTimeout(this.closeGraceTimer);
       this.closeGraceTimer = null;
     }
+    if (this.keepaliveTimer !== null) {
+      clearInterval(this.keepaliveTimer);
+      this.keepaliveTimer = null;
+    }
+  }
+
+  /**
+   * Start the periodic keepalive timer. Sends a Deepgram `KeepAlive`
+   * control frame every {@link keepaliveIntervalMs}; this is the only
+   * thing that resets Deepgram's server-side inactivity timer when the
+   * stream is carrying silence (raw silence PCM frames do not count).
+   *
+   * Skipped when {@link keepaliveIntervalMs} is 0 (test override) or the
+   * adapter is already closed/stopping.
+   */
+  private startKeepaliveTimer(): void {
+    if (this.closed || this.stopping) return;
+    if (this.keepaliveIntervalMs <= 0) return;
+    this.keepaliveTimer = setInterval(() => {
+      if (this.closed || this.stopping) return;
+      const ws = this.ws;
+      if (!ws || ws.readyState !== WS_OPEN) return;
+      try {
+        ws.send(JSON.stringify({ type: "KeepAlive" }));
+      } catch (err) {
+        log.warn({ err }, "Deepgram KeepAlive send failed");
+      }
+    }, this.keepaliveIntervalMs);
   }
 
   /**

package/src/providers/speech-to-text/xai-realtime.test.ts

@@ -497,14 +497,79 @@ describe("XAIRealtimeTranscriber", () => {
     const transcriber = new XAIRealtimeTranscriber(TEST_API_KEY, {
       connectTimeoutMs: 1_000,
     });
-    const { onEvent } = createEventCollector();
+    const { events, onEvent } = createEventCollector();
 
-    // First attempt: transport-level error before open.
+    // First attempt: transport-level error before open. Real WebSocket
+    // implementations commonly chain `error` → `close` on the abandoned
+    // socket, so simulate both to catch the regression where the old
+    // socket's stray close event corrupts `this.closed` and silently
+    // breaks the retry.
+    const firstSocket = mockWs;
     const firstAttempt = transcriber.start(onEvent);
-    mockWs.simulateError(new Error("ECONNREFUSED"));
+    firstSocket.simulateError(new Error("ECONNREFUSED"));
+    firstSocket.simulateClose(1006, "abnormal closure");
     await expect(firstAttempt).rejects.toThrow(/xAI realtime connect error/);
 
-    // Second attempt — instance must be reusable.
+    // The stray close on the abandoned socket must NOT have emitted a
+    // `closed` event or marked the transcriber closed — otherwise the
+    // retry below would silently no-op on sendAudio.
+    expect(events.filter((e) => e.type === "closed")).toHaveLength(0);
+
+    // Second attempt — instance must be reusable AND fully functional.
+    mockWs = new MockWebSocket();
+    (globalThis as Record<string, unknown>).WebSocket = class {
+      constructor(
+        _url: string,
+        _options?: { headers?: Record<string, string> },
+      ) {
+        return mockWs;
+      }
+    };
+
+    const secondAttempt = transcriber.start(onEvent);
+    mockWs.simulateOpen();
+    mockWs.simulateMessage(CREATED_FRAME);
+    await expect(secondAttempt).resolves.toBeUndefined();
+
+    // Confirm the retry session is actually live — sendAudio must reach
+    // the new socket (proves `this.closed` wasn't sticky-corrupted).
+    transcriber.sendAudio(Buffer.from([0x01, 0x02, 0x03]), "audio/pcm");
+    expect(mockWs.sentData).toHaveLength(1);
+  });
+
+  // ─────────────────────────────────────────────────────────────────
+  // Regression: a close event on the abandoned socket fired AFTER a
+  // handshake-phase rejection must not corrupt `this.closed`. Real
+  // WebSocket impls emit `close` asynchronously after `ws.close()` is
+  // called, and commonly chain `error` → `close`; if the handshake
+  // listeners aren't detached before `forceClose()`, the stray event
+  // routes through `handleProviderClose` and flips `this.closed`,
+  // silently breaking subsequent retries.
+  // ─────────────────────────────────────────────────────────────────
+
+  test("stray close on abandoned socket after handshake rejection does not break retry", async () => {
+    const transcriber = new XAIRealtimeTranscriber(TEST_API_KEY, {
+      connectTimeoutMs: 1_000,
+    });
+    const { events, onEvent } = createEventCollector();
+
+    // First attempt: close rejects the handshake.
+    const firstSocket = mockWs;
+    const firstAttempt = transcriber.start(onEvent);
+    firstSocket.simulateClose(4001, "unauthorized");
+    await expect(firstAttempt).rejects.toThrow(
+      /xAI WebSocket closed before handshake/,
+    );
+
+    // Simulate a second close event arriving on the abandoned socket
+    // (as `forceClose()` → `ws.close()` would trigger in a real impl).
+    firstSocket.simulateClose(1006, "abnormal closure");
+
+    // The stray event must be a no-op: no `closed` event emitted, and
+    // no internal state corruption.
+    expect(events.filter((e) => e.type === "closed")).toHaveLength(0);
+
+    // Retry and confirm the session is fully functional.
     mockWs = new MockWebSocket();
     (globalThis as Record<string, unknown>).WebSocket = class {
       constructor(
@@ -519,6 +584,9 @@ describe("XAIRealtimeTranscriber", () => {
     mockWs.simulateOpen();
     mockWs.simulateMessage(CREATED_FRAME);
     await expect(secondAttempt).resolves.toBeUndefined();
+
+    transcriber.sendAudio(Buffer.from([0x01, 0x02, 0x03]), "audio/pcm");
+    expect(mockWs.sentData).toHaveLength(1);
   });
 
   // ─────────────────────────────────────────────────────────────────

package/src/providers/speech-to-text/xai-realtime.ts

@@ -255,13 +255,17 @@ export class XAIRealtimeTranscriber implements StreamingTranscriber {
     await new Promise<void>((resolve, reject) => {
       let settled = false;
 
-      const handshakeTimer = setTimeout(() => {
-        if (settled) return;
-        settled = true;
-        this.forceClose();
-        reject(new Error("xAI realtime connect timeout"));
-      }, this.connectTimeoutMs);
-
+      // Listener references, captured so we can detach them from the
+      // abandoned socket on every reject/timeout path. Without this,
+      // `forceClose()` → `ws.close()` triggers an asynchronous `close`
+      // event (and real WebSocket impls commonly chain `error` → `close`)
+      // on the old socket. With the listeners still attached, that stray
+      // event routes through the `settled === true` branch into
+      // `handleProviderClose`, which calls `emitClosedAndCleanup()` and
+      // sets `this.closed = true`. A subsequent `start()` then resolves
+      // but `sendAudio` / `stop` / timers all no-op because `this.closed`
+      // is sticky — retry is silently dead. Detaching the handlers
+      // before `forceClose()` closes that window.
       const settleResolve = () => {
         if (settled) return;
         settled = true;
@@ -273,6 +277,9 @@ export class XAIRealtimeTranscriber implements StreamingTranscriber {
         if (settled) return;
         settled = true;
         clearTimeout(handshakeTimer);
+        // Detach listeners BEFORE `forceClose()` so stray close/error
+        // events on the abandoned socket can't flip `this.closed`.
+        detachHandshakeListeners();
         // Null out this.ws (via forceClose) so the instance can be
         // reused for a retry. Without this, a subsequent start() call
         // would throw "start() called twice" even though no session
@@ -288,9 +295,8 @@ export class XAIRealtimeTranscriber implements StreamingTranscriber {
       const onOpen = () => {
         ws.removeEventListener("open", onOpen);
       };
-      ws.addEventListener("open", onOpen);
 
-      ws.addEventListener("message", (ev: { data: unknown }) => {
+      const onMessage = (ev: { data: unknown }) => {
         if (!settled) {
           if (tryParseHandshakeFrame(ev.data)?.type === "transcript.created") {
             settleResolve();
@@ -303,9 +309,9 @@ export class XAIRealtimeTranscriber implements StreamingTranscriber {
           return;
         }
         this.handleProviderMessage(ev.data);
-      });
+      };
 
-      ws.addEventListener("close", (ev: { code: number; reason: string }) => {
+      const onClose = (ev: { code: number; reason: string }) => {
         if (!settled) {
           // 401 / 403 on connect arrive as WebSocket close codes 4001 /
           // 4003 in most runtimes (or 1008 policy-violation in others).
@@ -320,9 +326,9 @@ export class XAIRealtimeTranscriber implements StreamingTranscriber {
           return;
         }
         this.handleProviderClose(ev.code, ev.reason);
-      });
+      };
 
-      ws.addEventListener("error", (ev: unknown) => {
+      const onError = (ev: unknown) => {
         if (!settled) {
           const msg =
             ev instanceof Error
@@ -334,7 +340,26 @@ export class XAIRealtimeTranscriber implements StreamingTranscriber {
           return;
         }
         this.handleProviderError(ev);
-      });
+      };
+
+      const detachHandshakeListeners = () => {
+        ws.removeEventListener("message", onMessage);
+        ws.removeEventListener("close", onClose);
+        ws.removeEventListener("error", onError);
+      };
+
+      const handshakeTimer = setTimeout(() => {
+        if (settled) return;
+        settled = true;
+        detachHandshakeListeners();
+        this.forceClose();
+        reject(new Error("xAI realtime connect timeout"));
+      }, this.connectTimeoutMs);
+
+      ws.addEventListener("open", onOpen);
+      ws.addEventListener("message", onMessage);
+      ws.addEventListener("close", onClose);
+      ws.addEventListener("error", onError);
     });
 
     this.resetInactivityTimer();

package/src/runtime/AGENTS.md

@@ -65,7 +65,7 @@ Host browser allows the assistant to proxy CDP (Chrome DevTools Protocol) JSON-R
 
 The `chrome-extension` interface in `INTERFACE_IDS` is a non-interactive transport that supports only the `host_browser` capability — it does NOT support `host_bash`, `host_file`, or `host_cu`. This is encoded in `supportsHostProxy(id, capability)`: passing a capability argument returns `true` for `chrome-extension` only when the capability is `host_browser`; the no-arg form returns `false` for `chrome-extension` (so legacy desktop-only call sites that assume full-desktop proxy availability continue to gate correctly).
 
-Unlike the SSE-based host proxies used by the macOS client, `host_browser_request` frames for the chrome-extension interface do NOT travel through `assistantEventHub`. Instead they are routed through the `ChromeExtensionRegistry` singleton (`runtime/chrome-extension-registry.ts`), which tracks active chrome-extension WebSocket connections keyed by `(guardianId, clientInstanceId)`. The registry is populated on WebSocket `open` and drained on `close` inside `http-server.ts`'s `/v1/browser-relay` handlers — see the `wsType === "browser-relay"` branches.
+For the chrome-extension interface, `host_browser_request` frames do NOT travel through `assistantEventHub`. Instead they are routed through the `ChromeExtensionRegistry` singleton (`runtime/chrome-extension-registry.ts`), which tracks active chrome-extension WebSocket connections keyed by `(guardianId, clientInstanceId)`. The registry is populated on WebSocket `open` and drained on `close` inside `http-server.ts`'s `/v1/browser-relay` handlers — see the `wsType === "browser-relay"` branches. For macOS, `host_browser_request` frames travel through `assistantEventHub` (SSE) by default; when the guardian also has an active extension connection, the registry-routed WebSocket sender takes precedence.
 
 A single guardian may have multiple parallel extension installs connected at once (two Chrome profiles, two desktops sharing a sync identity). Each install generates a stable `clientInstanceId` on first run, persists it in `chrome.storage.local`, and sends it on every WebSocket handshake as a query param (`clientInstanceId=...`) or header (`x-client-instance-id`). The registry keys inner entries by that id so sibling installs don't evict each other on register/unregister. The default `send(guardianId, msg)` path routes to whichever instance has the most recent activity (`lastActiveAt`); `sendToInstance(guardianId, clientInstanceId, msg)` pins a specific install. Older extension builds that omit the id get a connection-scoped `legacy:<connectionId>` fallback key so they degrade gracefully to single-instance semantics.
 
@@ -79,11 +79,20 @@ See `docs/browser-use-architecture-phase2.md` for the full wire diagram and comp
 
 On macOS-originated turns, the CDP factory (`tools/browser/cdp-client/factory.ts`) evaluates three browser backends in strict priority order. Each candidate is tried lazily; if the first command fails with a transport-level error, the factory falls over to the next candidate. CDP protocol errors (the browser understood the command but rejected it) do NOT trigger failover.
 
-| Priority | Backend         | Condition                                                                                                                                                                                           | Source                                                                                 |
-| -------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
-| 1        | **Extension**   | `hostBrowserProxy` present AND `isAvailable()` returns `true` (registry-routed WebSocket is connected)                                                                                              | `ChromeExtensionRegistry` via `resolveHostBrowserSender()` in `conversation-routes.ts` |
-| 2        | **cdp-inspect** | (a) `hostBrowser.cdpInspect.enabled` is `true` in config, OR (b) `transportInterface === "macos"` AND `desktopAuto.enabled` is `true` (default) AND the cooldown from a prior failure is not active | Config + `desktopAuto` policy in factory                                               |
-| 3        | **Local**       | Always present as the final fallback                                                                                                                                                                | Playwright sacrificial-profile browser managed by `browserManager`                     |
+| Priority | Backend                    | Condition                                                                                                                                                                                           | Transport                                                                                              |
+| -------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ |
+| 1        | **Extension / host proxy** | `hostBrowserProxy` present AND `isAvailable()` returns `true`. On macOS, the proxy is always provisioned (SSE sender when no extension is present, registry-routed when extension is connected)     | WS via `ChromeExtensionRegistry` (extension present) or SSE via `assistantEventHub` (macOS host proxy) |
+| 2        | **cdp-inspect**            | (a) `hostBrowser.cdpInspect.enabled` is `true` in config, OR (b) `transportInterface === "macos"` AND `desktopAuto.enabled` is `true` (default) AND the cooldown from a prior failure is not active | Direct CDP WebSocket to `localhost:9222`                                                               |
+| 3        | **Local**                  | Always present as the final fallback                                                                                                                                                                | In-process Playwright CDP via `browserManager`                                                         |
+
+**Transport selection for the extension/host-proxy backend:**
+
+The "extension" backend label is a misnomer inherited from the original Phase 2 design where only the Chrome Extension provided host-browser access. In the current architecture, two transports can power this backend:
+
+- **Extension WebSocket**: When the `ChromeExtensionRegistry` has an active entry for the guardian, `resolveHostBrowserSender()` returns a registry-routed sender. `host_browser_request` frames travel over the `/v1/browser-relay` WebSocket to the Chrome extension, which executes CDP commands via `chrome.debugger`.
+- **macOS SSE bridge**: When the macOS desktop client is connected but no extension is present, `resolveHostBrowserSender()` returns the `onEvent` SSE hub sender. `host_browser_request` frames travel through `assistantEventHub` to the desktop client's SSE connection. The desktop client executes CDP commands against the local Chrome and POSTs results back to `/v1/host-browser-result`.
+
+Both transports use the same `HostBrowserProxy` → `ExtensionCdpClient` pipeline. The `browser_status` output distinguishes the transport via the `details.transport` field: `"extension-ws"` or `"macos-sse"`.
 
 **Fallback criteria for cdp-inspect (desktop-auto):**
 
@@ -98,24 +107,24 @@ On macOS-originated turns, the CDP factory (`tools/browser/cdp-client/factory.ts
 
 All CDP-backed browser tools (`browser_navigate`, `browser_snapshot`, `browser_screenshot`, `browser_click`, `browser_type`, `browser_hover`, `browser_scroll`, `browser_press_key`, `browser_select_option`, `browser_wait_for`, `browser_extract`, `browser_fill_credential`, `browser_attach`, `browser_detach`, `browser_close`, `browser_status`) accept an optional `browser_mode` input parameter that overrides the automatic backend selection for that invocation.
 
-| Value            | Behavior                                                                 |
-| ---------------- | ------------------------------------------------------------------------ |
-| `auto` (default) | Existing priority-ordered fallback: extension -> cdp-inspect -> local    |
-| `extension`      | Pin to chrome-extension backend. Fails immediately if proxy unavailable. |
-| `cdp-inspect`    | Pin to CDP inspect/debugger backend. Fails if endpoint unreachable.      |
-| `local`          | Pin to local Playwright-managed browser. No fallback.                    |
-| `cdp-debugger`   | Alias for `cdp-inspect`.                                                 |
-| `playwright`     | Alias for `local`.                                                       |
+| Value            | Behavior                                                                     |
+| ---------------- | ---------------------------------------------------------------------------- |
+| `auto` (default) | Existing priority-ordered fallback: extension -> cdp-inspect -> local        |
+| `extension`      | Pin to extension/host-proxy backend. Fails immediately if proxy unavailable. |
+| `cdp-inspect`    | Pin to CDP inspect/debugger backend. Fails if endpoint unreachable.          |
+| `local`          | Pin to local Playwright-managed browser. No fallback.                        |
+| `cdp-debugger`   | Alias for `cdp-inspect`.                                                     |
+| `playwright`     | Alias for `local`.                                                           |
 
 **Strict pinned-mode semantics**: When `browser_mode` is set to a specific backend (not `auto`), the factory builds exactly one candidate and disables failover. If the pinned backend is unavailable, the tool returns a detailed error including:
 
 - The requested mode
 - An ordered list of attempted backends with exact failure reasons
-- A remediation checklist tailored by backend and failure code (e.g. "Ensure Chrome is running with --remote-debugging-port=9222")
+- A remediation checklist tailored by backend, failure code, and transport (e.g. for macOS SSE: "Verify the Vellum desktop app is running"; for extension: "Ensure Chrome is running with the extension paired")
 
 **Auto-mode fallback logging**: In auto mode, fallback transitions are logged at `warn` level with structured metadata including the full candidate sequence and per-candidate failure reasons. This ensures fallback events are always observable in production logs.
 
-**Test coverage:** Regression tests for `browser_mode` wiring live in `__tests__/headless-browser-mode.test.ts`. E2E regression tests for backend precedence live in `__tests__/host-browser-e2e-cloud.test.ts` (extension path) and `__tests__/conversation-routes-disk-view.test.ts` (macOS fallback path). Unit tests for pinned candidate construction and failover live in `tools/browser/cdp-client/__tests__/factory.test.ts`.
+**Test coverage:** Regression tests for `browser_mode` wiring live in `__tests__/headless-browser-mode.test.ts`. E2E regression tests for backend precedence live in `__tests__/host-browser-e2e-cloud.test.ts` (extension path and macOS SSE bridge path) and `__tests__/conversation-routes-disk-view.test.ts` (macOS fallback path). Unit tests for pinned candidate construction and failover live in `tools/browser/cdp-client/__tests__/factory.test.ts`. Browser status tests covering macOS host-browser diagnostics live in `tools/browser/__tests__/browser-status.test.ts`.
 
 ### Channel approvals (Telegram, Slack)
 

package/src/runtime/__tests__/browser-extension-pair-routes.test.ts

@@ -26,7 +26,7 @@ import {
   verifyHostBrowserCapability,
 } from "../capability-tokens.js";
 import {
-  ALLOWED_EXTENSION_ORIGINS,
+  getAllowedExtensionOrigins,
   handleBrowserExtensionPair,
   NATIVE_HOST_MARKER_HEADER,
   NATIVE_HOST_MARKER_VALUE,
@@ -55,10 +55,10 @@ const lanPeerServer = mockServer("192.168.1.10");
 const publicPeerServer = mockServer("203.0.113.50");
 
 const ALLOWED_ORIGIN = (() => {
-  const first = Array.from(ALLOWED_EXTENSION_ORIGINS)[0];
+  const first = Array.from(getAllowedExtensionOrigins())[0];
   if (!first) {
     throw new Error(
-      "ALLOWED_EXTENSION_ORIGINS must contain at least one extension origin for tests",
+      "getAllowedExtensionOrigins() must contain at least one extension origin for tests",
     );
   }
   return first;