@voxevolv/client 0.3.8 → 0.3.10

package/dist/index.js

@@ -1,4 +1,4 @@
-import { ConnectionState as LiveKitConnectionState, DataPacket_Kind, DisconnectReason, Room, RoomEvent, RpcError, Track, } from "livekit-client";
+import { ConnectionState as LiveKitConnectionState, DataPacket_Kind, DisconnectReason, LocalAudioTrack, Room, RoomEvent, RpcError, Track, } from "livekit-client";
 import { createClientMessage, CommandError, EventCursor, isCommandAckPayload, isCommandErrorPayload, isContextRejectedPayload, isContextUpdatePayload, isContextUpdatedPayload, isRealtimeEvent, } from "./protocol.js";
 import { ContextDeliveryError, contextRetryDelay, ReplacementContextQueue } from "./context-queue.js";
 import { ClientDiagnosticBuffer, DiagnosticDeliveryError, } from "./diagnostics.js";
@@ -6,7 +6,7 @@ import { TranscriptStore } from "./transcripts.js";
 export { TranscriptStore } from "./transcripts.js";
 export { CommandError, EventCursor, createClientMessage, isCommandAckPayload, isCommandErrorPayload, isContextRejectedPayload, isContextUpdatePayload, isContextUpdatedPayload, isRealtimeEvent, } from "./protocol.js";
 export { decodeAgentSnapshotPayload, decodeClientHelloPayload, decodeClientMessage, decodeCommandAckPayload, decodeCommandErrorPayload, decodeContextRejectedPayload, decodeContextUpdatePayload, decodeContextUpdatedPayload, decodeRealtimeEvent, } from "./generated/protocol-codecs.js";
-export const SDK_VERSION = "0.3.8";
+export const SDK_VERSION = "0.3.10";
 const CLIENT_HELLO_TIMEOUT_MS = 30_000;
 const DIAGNOSTIC_REPORT_TIMEOUT_MS = 5_000;
 const DEFAULT_STARTUP_ATTEMPTS = [
@@ -99,9 +99,12 @@ export class RealtimeSession {
     startupConnectController = null;
     startupAttemptController = null;
     useManagedMicrophone = true;
-    managedMicrophoneEnabled = true;
+    microphoneEnabled = true;
     applicationAudioTrack = null;
+    localApplicationAudioTrack = null;
     connectDefaults;
+    recoveryReadiness;
+    credentialRefreshGeneration = 0;
     fetcher;
     endEndpoint;
     refreshCredentialsCallback;
@@ -109,6 +112,11 @@ export class RealtimeSession {
     startupRetryOptions;
     lifecycleCredentials;
     diagnostics = new ClientDiagnosticBuffer({ sdkVersion: SDK_VERSION });
+    activeContextTransfers = 0;
+    activeContextBytes = 0;
+    diagnosticLoopTimer;
+    diagnosticLoopExpectedAt = 0;
+    diagnosticLoopDelayMs;
     diagnosticReportTimeoutMs = DIAGNOSTIC_REPORT_TIMEOUT_MS;
     onDiagnosticVisibilityChange = () => {
         if (globalThis.document?.visibilityState === "hidden")
@@ -123,6 +131,10 @@ export class RealtimeSession {
             ? (options.connect ?? {})
             : { ...options.connect, apiBase: normalizeApiBase(options.connect.apiBase) };
         this.apiBase = this.connectDefaults.apiBase ?? this.apiBase;
+        this.recoveryReadiness = {
+            readyTimeoutMs: this.connectDefaults.readyTimeoutMs ?? 15_000,
+            waitForRemoteAudio: this.connectDefaults.waitForRemoteAudio ?? true,
+        };
         this.fetcher = options.fetch ?? globalThis.fetch.bind(globalThis);
         this.endEndpoint = options.endEndpoint ?? null;
         this.refreshCredentialsCallback = options.refreshCredentials ?? null;
@@ -133,7 +145,34 @@ export class RealtimeSession {
         for (const [name, handler] of Object.entries(options.tools ?? {})) {
             this.registerTool(name, handler);
         }
-        this.room.registerRpcMethod("voxevolv.tool.invoke", async (invocation) => {
+        this.room.registerRpcMethod("voxevolv.tool.invoke", (invocation) => this.invokeBrowserTool(invocation));
+        this.room.registerRpcMethod("voxevolv.diagnostics.flush", async (invocation) => {
+            const payload = JSON.parse(invocation.payload);
+            if (typeof payload?.call_id !== "string" || payload.call_id.length > 200) {
+                throw new RpcError(1500, "Invalid diagnostic request");
+            }
+            this.recordDiagnostic({
+                operation: "tool_rpc", status: "started", stage: "flush_requested",
+                call_id: payload.call_id, rpc_request_id: invocation.requestId,
+            });
+            // HTTP delivery has its own retry/receipt handling; never await it on the RPC channel.
+            this.flushDiagnostics("tool_rpc_worker_request");
+            return "{}";
+        });
+        this.bindRoomEvents();
+    }
+    async invokeBrowserTool(invocation) {
+        const receivedAt = performance.now();
+        let callId;
+        let toolName;
+        let slowTimer;
+        const record = (entry) => this.recordDiagnostic({
+            operation: "tool_rpc", status: "started", rpc_request_id: invocation.requestId,
+            call_id: callId, tool_name: toolName,
+            duration_ms: performance.now() - receivedAt, ...entry,
+        });
+        record({ stage: "received" });
+        try {
             let payload;
             try {
                 payload = JSON.parse(invocation.payload);
@@ -141,32 +180,47 @@ export class RealtimeSession {
             catch (error) {
                 throw new RpcError(1500, "Tool invocation payload was not valid JSON", undefined, { cause: error });
             }
-            if (typeof payload.name !== "string") {
+            if (typeof payload?.name !== "string") {
                 throw new RpcError(1500, "Tool invocation did not include a valid name");
             }
-            const handler = this.toolHandlers.get(payload.name);
+            toolName = payload.name;
+            callId = typeof payload.call_id === "string" ? payload.call_id : invocation.requestId;
+            const handler = this.toolHandlers.get(toolName);
             if (!handler)
-                throw new RpcError(1500, `No browser handler is registered for ${payload.name}`);
+                throw new RpcError(1500, `No browser handler is registered for ${toolName}`);
             if (!payload.arguments || typeof payload.arguments !== "object" || Array.isArray(payload.arguments)) {
                 throw new RpcError(1500, "Tool invocation arguments must be an object");
             }
-            const callId = typeof payload.call_id === "string" ? payload.call_id : invocation.requestId;
             const arguments_ = payload.arguments;
-            this.notifyListeners(this.toolCallListeners, {
-                name: payload.name,
-                callId,
-                arguments: arguments_,
-            });
+            this.notifyListeners(this.toolCallListeners, { name: toolName, callId, arguments: arguments_ });
+            record({ stage: "handler_started" });
+            this.flushDiagnostics("tool_rpc_received");
+            slowTimer = globalThis.setTimeout(() => {
+                record({ stage: "handler_started", reason: "handler_slow" });
+                this.flushDiagnostics("tool_rpc_slow");
+            }, 2_000);
+            let response;
             try {
                 const result = await handler(arguments_, { callId });
-                return JSON.stringify(result ?? null);
+                response = JSON.stringify(result ?? null);
             }
             catch (error) {
                 const message = error instanceof Error ? error.message : "Browser tool handler failed";
                 throw new RpcError(1500, message, undefined, { cause: error });
             }
-        });
-        this.bindRoomEvents();
+            // LiveKit sends the response after this callback returns. This is readiness,
+            // not proof of a network send or acknowledgement by the worker.
+            record({ stage: "response_ready", status: "succeeded" });
+            return response;
+        }
+        catch (error) {
+            record({ status: "failed", rpc_error_code: error instanceof RpcError ? error.code : undefined });
+            throw error;
+        }
+        finally {
+            globalThis.clearTimeout(slowTimer);
+            this.flushDiagnostics("tool_rpc_finished");
+        }
     }
     static async create(options) {
         if ("connect" in options) {
@@ -334,6 +388,9 @@ export class RealtimeSession {
         };
     }
     async connect(options = {}) {
+        if (this.startupConnectController || this.manualRejoinController) {
+            throw new Error("A session connection attempt is already in progress");
+        }
         if (["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"]
             .includes(this.stateValue)) {
             throw new Error(`Cannot connect while session is ${this.stateValue}`);
@@ -341,6 +398,17 @@ export class RealtimeSession {
         this.cancelRecovery();
         const controller = new AbortController();
         this.startupConnectController = controller;
+        this.credentialRefreshGeneration += 1;
+        const resolved = { ...this.connectDefaults, ...options };
+        // Keep caller preferences separate from temporary startup-attempt budgets.
+        this.recoveryReadiness = {
+            readyTimeoutMs: resolved.readyTimeoutMs ?? 15_000,
+            waitForRemoteAudio: resolved.waitForRemoteAudio ?? true,
+        };
+        this.microphoneEnabled = resolved.audioTrack
+            ? resolved.audioTrack.enabled !== false
+            : resolved.microphone ?? true;
+        this.setState("connecting");
         try {
             if (this.startupRetryOptions) {
                 await this.connectWithStartupRetry(options, controller);
@@ -397,6 +465,7 @@ export class RealtimeSession {
                         await this.registerStartupRecovery(remainingMs, attemptController.signal);
                         startupRegistered = true;
                     }
+                    throwIfAborted(attemptController.signal);
                     if (index === 0) {
                         await this.connectWithMode({ ...options, readyTimeoutMs: attempt.readyTimeoutMs }, false, attemptController.signal, false);
                     }
@@ -439,7 +508,7 @@ export class RealtimeSession {
         const controller = createLinkedAbortController(signal);
         const requestTimeout = globalThis.setTimeout(() => {
             controller.abort(new Error("Startup registration timed out"));
-        }, Math.max(1, Math.min(3_000, reserveMs)));
+        }, Math.max(1, reserveMs));
         try {
             // Leave room for registration latency and final readiness before the
             // browser's existing overall deadline. Rejoins never renew this window.
@@ -452,7 +521,7 @@ export class RealtimeSession {
             }), controller.signal);
             let response = await send();
             if (response.status === 401 && this.refreshCredentialsCallback) {
-                await raceWithAbort(this.refreshCredentials(), controller.signal);
+                await this.refreshCredentials(controller.signal);
                 response = await send();
             }
             if (!response.ok)
@@ -477,6 +546,7 @@ export class RealtimeSession {
         });
     }
     async connectWithMode(options, recovering, signal, markFailureTerminal = true) {
+        throwIfAborted(signal);
         const blockedStates = recovering
             ? ["connecting", "connected_not_ready", "connected", "reconnecting"]
             : ["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"];
@@ -495,8 +565,6 @@ export class RealtimeSession {
         const apiBase = normalizeApiBase(resolved.apiBase ?? this.apiBase);
         this.validateAudioOptions(microphone, audioTrack);
         this.useManagedMicrophone = microphone;
-        if (!recovering)
-            this.managedMicrophoneEnabled = microphone;
         this.applicationAudioTrack = audioTrack;
         if (!recovering) {
             this.sessionEnded = false;
@@ -520,7 +588,7 @@ export class RealtimeSession {
                 await raceWithAbort(this.publishApplicationAudioTrack(audioTrack), signal);
             }
             else if (microphone) {
-                await raceWithAbort(this.room.localParticipant.setMicrophoneEnabled(this.managedMicrophoneEnabled), signal);
+                await raceWithAbort(this.room.localParticipant.setMicrophoneEnabled(this.microphoneEnabled), signal);
             }
             throwIfAborted(signal);
             this.setState(recovering ? "resuming" : "connected_not_ready");
@@ -595,10 +663,13 @@ export class RealtimeSession {
         });
     }
     async rejoin(apiBase = this.apiBase) {
+        if (this.startupConnectController || this.manualRejoinController) {
+            throw new Error("A session connection attempt is already in progress");
+        }
         if (this.sessionEnded) {
             throw this.terminalSessionError ?? new Error("Cannot rejoin a session that has ended");
         }
-        if (["connecting", "reconnecting", "resuming"].includes(this.stateValue)) {
+        if (["connecting", "connected_not_ready", "reconnecting", "resuming"].includes(this.stateValue)) {
             throw new Error(`Cannot rejoin while session is ${this.stateValue}`);
         }
         this.cancelRecovery();
@@ -633,13 +704,13 @@ export class RealtimeSession {
         }
     }
     async performRejoin(apiBase, options = {}) {
-        const { signal, recovering = true, connect = {}, markFailureTerminal = true, } = options;
+        const { signal, recovering = true, connect, markFailureTerminal = true, } = options;
         this.contextGeneration += 1;
         this.replacementContext.pause();
         this.rejectPendingCommands(new Error("Restoring the session"));
         let response = await this.fetchRoomToken(apiBase, signal);
         if (response.status === 401 && this.refreshCredentialsCallback) {
-            await raceWithAbort(this.refreshCredentials(), signal);
+            await this.refreshCredentials(signal);
             throwIfAborted(signal);
             response = await this.fetchRoomToken(apiBase, signal);
         }
@@ -655,6 +726,7 @@ export class RealtimeSession {
             throw await raceWithAbort(httpError(response, "Could not renew room token"), signal);
         }
         const payload = (await raceWithAbort(response.json(), signal));
+        throwIfAborted(signal);
         this.credentials = payload.connection;
         if (this.room.state !== LiveKitConnectionState.Disconnected) {
             this.disconnectMode = "rejoin";
@@ -673,16 +745,28 @@ export class RealtimeSession {
                 duration_ms: Date.now() - this.reconnectStartedAt,
             };
         }
-        await this.connectWithMode({
-            ...connect,
-            apiBase,
+        // Startup retries resolve the original caller options with constructor defaults.
+        // A later rejoin restores the audio settings saved by the previous connection.
+        const connectionOptions = connect ?? {
+            ...this.recoveryReadiness,
             microphone: this.useManagedMicrophone,
             ...(this.applicationAudioTrack ? { audioTrack: this.applicationAudioTrack } : {}),
-        }, recovering, signal, markFailureTerminal);
+        };
+        await this.connectWithMode({ ...connectionOptions, apiBase }, recovering, signal, markFailureTerminal);
     }
     async setMicrophoneEnabled(enabled) {
+        // Recovery must see the user's latest intent while the media operation waits.
+        this.microphoneEnabled = enabled;
+        if (this.applicationAudioTrack) {
+            // The room may not know this track yet while publication is pending.
+            const track = this.localApplicationAudioTrack;
+            if (enabled)
+                await track?.unmute();
+            else
+                await track?.mute();
+            return;
+        }
         await this.room.localParticipant.setMicrophoneEnabled(enabled);
-        this.managedMicrophoneEnabled = enabled;
     }
     async sendCommand(type, payload, timeoutMs = 10_000) {
         const message = createClientMessage(this.connectionId, ++this.clientSeq, type, payload);
@@ -709,7 +793,13 @@ export class RealtimeSession {
             context_key: update.key,
             payload_bytes: payloadBytes,
         });
-        const uploading = "image" in update ? this.uploadImageContext(update) : this.uploadTextContext(update);
+        this.activeContextTransfers += 1;
+        this.activeContextBytes += payloadBytes;
+        const uploading = ("image" in update ? this.uploadImageContext(update) : this.uploadTextContext(update))
+            .finally(() => {
+            this.activeContextTransfers -= 1;
+            this.activeContextBytes -= payloadBytes;
+        });
         const transferSettled = uploading.then(() => { }, () => { });
         let transferTimeout;
         const transferExpired = new Promise((_resolve, reject) => {
@@ -866,6 +956,7 @@ export class RealtimeSession {
         this.failRecovery(new Error("Session disconnected during recovery"));
         this.cancelRecovery();
         this.contextGeneration += 1;
+        this.localApplicationAudioTrack = null;
         this.replacementContext.clear(new Error("Session disconnected"));
         if (this.stateValue === "disconnected")
             return;
@@ -881,6 +972,7 @@ export class RealtimeSession {
         }
     }
     async end(options = {}) {
+        this.credentialRefreshGeneration += 1;
         this.abortStartupConnect(new Error("Session ended"));
         this.startupConnectController = null;
         this.failRecovery(new Error("Session ended during recovery"));
@@ -900,7 +992,7 @@ export class RealtimeSession {
             credentials: request.credentials ?? this.lifecycleCredentials,
         });
         if (response.status === 401 && usesScopedSessionEndpoint && this.refreshCredentialsCallback) {
-            await this.refreshCredentials();
+            await this.refreshCredentials(request.signal ?? undefined);
             headers.set("Authorization", `Bearer ${this.accessToken}`);
             response = await this.fetcher(endpoint, {
                 ...request,
@@ -928,9 +1020,13 @@ export class RealtimeSession {
             return;
         this.stateValue = state;
         if (state === "connecting" || state === "resuming") {
+            this.startDiagnosticLoopMonitor();
             globalThis.document?.addEventListener?.("visibilitychange", this.onDiagnosticVisibilityChange);
         }
         if (state === "disconnected" || state === "failed") {
+            globalThis.clearInterval(this.diagnosticLoopTimer);
+            this.diagnosticLoopTimer = undefined;
+            this.diagnosticLoopDelayMs = undefined;
             globalThis.document?.removeEventListener?.("visibilitychange", this.onDiagnosticVisibilityChange);
             this.flushDiagnostics("session_finished");
             const error = new Error(`Session ${state}`);
@@ -1086,9 +1182,9 @@ export class RealtimeSession {
         this.setState(this.hasConnectedValue ? "resuming" : "connected_not_ready");
         try {
             await this.sendHello();
-            const deadline = Date.now() + (this.connectDefaults.readyTimeoutMs ?? 15_000);
+            const deadline = Date.now() + this.recoveryReadiness.readyTimeoutMs;
             await this.waitForAgentReady(Math.max(deadline - Date.now(), 0));
-            if (this.connectDefaults.waitForRemoteAudio !== false)
+            if (this.recoveryReadiness.waitForRemoteAudio)
                 await this.waitForRemoteAudioTrack(deadline);
             if (this.sessionEnded || generation !== this.contextGeneration)
                 return;
@@ -1371,10 +1467,18 @@ export class RealtimeSession {
             signal?.removeEventListener("abort", cancel);
         }
     }
-    async refreshCredentials() {
+    async refreshCredentials(signal) {
         if (!this.refreshCredentialsCallback)
             return;
-        const refreshed = await this.refreshCredentialsCallback();
+        throwIfAborted(signal);
+        const connectionGeneration = this.contextGeneration;
+        const refreshGeneration = ++this.credentialRefreshGeneration;
+        const refreshed = await raceWithAbort(this.refreshCredentialsCallback(), signal);
+        throwIfAborted(signal);
+        if (connectionGeneration !== this.contextGeneration
+            || refreshGeneration !== this.credentialRefreshGeneration) {
+            throw new Error("Credential refresh was superseded");
+        }
         validateSessionCredentials(refreshed);
         if (refreshed.id !== this.id) {
             throw new Error("Refreshed credentials belong to a different session");
@@ -1469,15 +1573,26 @@ export class RealtimeSession {
         }
     }
     async publishApplicationAudioTrack(audioTrack) {
-        const publishedTrack = audioTrack.clone();
+        const clonedTrack = audioTrack.clone();
+        let publishedTrack = null;
         try {
+            publishedTrack = new LocalAudioTrack(clonedTrack, undefined, true);
+            this.localApplicationAudioTrack = publishedTrack;
+            if (!this.microphoneEnabled) {
+                // LiveKit can re-enable a raw track when wrapping it. Mute the wrapper
+                // before publication so recovery cannot briefly transmit muted input.
+                await publishedTrack.mute();
+            }
             await this.room.localParticipant.publishTrack(publishedTrack, {
                 source: Track.Source.Microphone,
                 name: "voxevolv-application-audio",
             });
         }
         catch (error) {
-            publishedTrack.stop();
+            if (this.localApplicationAudioTrack === publishedTrack) {
+                this.localApplicationAudioTrack = null;
+            }
+            clonedTrack.stop();
             throw error;
         }
     }
@@ -1685,12 +1800,32 @@ export class RealtimeSession {
     async publishCommand(message) {
         await this.room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { reliable: true, topic: "voxevolv.commands" });
     }
+    startDiagnosticLoopMonitor() {
+        if (this.diagnosticLoopTimer !== undefined)
+            return;
+        this.diagnosticLoopExpectedAt = performance.now() + 1_000;
+        this.diagnosticLoopTimer = globalThis.setInterval(() => {
+            const now = performance.now();
+            this.diagnosticLoopDelayMs = Math.max(0, now - this.diagnosticLoopExpectedAt);
+            this.diagnosticLoopExpectedAt = now + 1_000;
+        }, 1_000);
+        this.diagnosticLoopTimer.unref?.();
+    }
     recordDiagnostic(entry) {
-        this.diagnostics.record({
+        const snapshot = {
             ...entry,
             connection_state: this.stateValue,
             buffered_bytes: this.reliableDataChannelBufferedBytes(),
-        });
+        };
+        if (entry.operation === "tool_rpc") {
+            snapshot.page_visibility = globalThis.document?.visibilityState;
+            snapshot.active_context_transfers = this.activeContextTransfers;
+            snapshot.active_context_bytes = this.activeContextBytes;
+            if (this.diagnosticLoopTimer !== undefined) {
+                snapshot.event_loop_delay_ms = Math.max(this.diagnosticLoopDelayMs ?? 0, performance.now() - this.diagnosticLoopExpectedAt);
+            }
+        }
+        this.diagnostics.record(snapshot);
     }
     reliableDataChannelBufferedBytes() {
         try {
@@ -1911,7 +2046,7 @@ export class RealtimeSession {
                 signal,
             }), signal);
             if (response.status === 401 && this.refreshCredentialsCallback) {
-                await raceWithAbort(this.refreshCredentials(), signal);
+                await this.refreshCredentials(signal);
                 response = await raceWithAbort(this.fetcher(url, {
                     headers: { Authorization: `Bearer ${this.accessToken}` },
                     credentials: this.lifecycleCredentials,