@voxevolv/client 0.3.2 → 0.3.4

package/dist/index.js

@@ -1,10 +1,14 @@
 import { ConnectionState as LiveKitConnectionState, DataPacket_Kind, DisconnectReason, Room, RoomEvent, RpcError, Track, } from "livekit-client";
 import { createClientMessage, CommandError, EventCursor, isCommandAckPayload, isCommandErrorPayload, isContextRejectedPayload, isContextUpdatePayload, isContextUpdatedPayload, isRealtimeEvent, } from "./protocol.js";
 import { ReplacementContextQueue } from "./context-queue.js";
+import { ClientDiagnosticBuffer, } from "./diagnostics.js";
+import { TranscriptStore } from "./transcripts.js";
+export { TranscriptStore } from "./transcripts.js";
 export { CommandError, EventCursor, createClientMessage, isCommandAckPayload, isCommandErrorPayload, isContextRejectedPayload, isContextUpdatePayload, isContextUpdatedPayload, isRealtimeEvent, } from "./protocol.js";
 export { decodeAgentSnapshotPayload, decodeClientHelloPayload, decodeClientMessage, decodeCommandAckPayload, decodeCommandErrorPayload, decodeContextRejectedPayload, decodeContextUpdatePayload, decodeContextUpdatedPayload, decodeRealtimeEvent, } from "./generated/protocol-codecs.js";
-export const SDK_VERSION = "0.3.2";
+export const SDK_VERSION = "0.3.4";
 const CLIENT_HELLO_TIMEOUT_MS = 30_000;
+const DIAGNOSTIC_REPORT_TIMEOUT_MS = 5_000;
 export class RealtimeSessionHttpError extends Error {
     status;
     code;
@@ -35,6 +39,8 @@ export class RealtimeSession {
     readyListeners = new Set();
     userTranscriptListeners = new Set();
     agentTranscriptListeners = new Set();
+    transcriptListeners = new Set();
+    transcriptStore = new TranscriptStore();
     errorListeners = new Set();
     endedListeners = new Set();
     toolCallListeners = new Set();
@@ -74,6 +80,8 @@ export class RealtimeSession {
     refreshCredentialsCallback;
     autoRejoinOptions;
     lifecycleCredentials;
+    diagnostics = new ClientDiagnosticBuffer({ sdkVersion: SDK_VERSION });
+    diagnosticReportTimeoutMs = DIAGNOSTIC_REPORT_TIMEOUT_MS;
     constructor(credentials, options = {}) {
         validateSessionCredentials(credentials);
         this.id = credentials.id;
@@ -248,6 +256,14 @@ export class RealtimeSession {
         this.agentTranscriptListeners.add(listener);
         return () => this.agentTranscriptListeners.delete(listener);
     }
+    /** Latest message snapshots, retained across transport recovery and replay. */
+    get transcript() {
+        return this.transcriptStore.snapshot;
+    }
+    onTranscriptUpdated(listener) {
+        this.transcriptListeners.add(listener);
+        return () => this.transcriptListeners.delete(listener);
+    }
     onError(listener) {
         this.errorListeners.add(listener);
         return () => this.errorListeners.delete(listener);
@@ -372,7 +388,7 @@ export class RealtimeSession {
         const normalizedApiBase = normalizeApiBase(apiBase);
         const recovering = this.hasConnectedValue;
         if (recovering) {
-            this.beginRecovery("rejoin", 1);
+            this.beginRecovery("rejoin", 1, { trigger: "manual_rejoin" });
             this.setState("resuming");
         }
         try {
@@ -445,18 +461,73 @@ export class RealtimeSession {
     }
     async sendContextUpdate(update, timeoutMs) {
         const generation = this.contextGeneration;
+        const payloadBytes = contextUpdateBytes(update);
+        const transferStartedAt = Date.now();
+        let transferTimedOut = false;
+        this.recordDiagnostic({
+            operation: "context_transfer",
+            status: "started",
+            context_key: update.key,
+            payload_bytes: payloadBytes,
+        });
         let transferTimeout;
         const transferExpired = new Promise((_resolve, reject) => {
             transferTimeout = globalThis.setTimeout(() => {
+                transferTimedOut = true;
                 const error = new Error(`Context transfer timed out after ${timeoutMs}ms`);
-                this.startAutoRejoin(error);
+                this.recordDiagnostic({
+                    operation: "context_transfer",
+                    status: "timed_out",
+                    reason: "context_transfer_timeout",
+                    context_key: update.key,
+                    payload_bytes: payloadBytes,
+                    duration_ms: Date.now() - transferStartedAt,
+                    timeout_ms: timeoutMs,
+                });
+                this.flushDiagnostics("context_transfer_timeout");
+                this.startAutoRejoin(error, {
+                    trigger: "context_transfer_timeout",
+                    operation: "context_transfer",
+                    contextKey: update.key,
+                    timeoutMs,
+                });
                 reject(error);
             }, timeoutMs);
         });
-        const transfer = await Promise.race([
-            "image" in update ? this.uploadImageContext(update) : this.uploadTextContext(update),
-            transferExpired,
-        ]).finally(() => globalThis.clearTimeout(transferTimeout));
+        let transfer;
+        try {
+            transfer = await Promise.race([
+                "image" in update ? this.uploadImageContext(update) : this.uploadTextContext(update),
+                transferExpired,
+            ]);
+        }
+        catch (error) {
+            if (!transferTimedOut) {
+                this.recordDiagnostic({
+                    operation: "context_transfer",
+                    status: "failed",
+                    reason: "context_transfer_failed",
+                    context_key: update.key,
+                    payload_bytes: payloadBytes,
+                    duration_ms: Date.now() - transferStartedAt,
+                });
+                this.flushDiagnostics("command_failed");
+            }
+            throw error;
+        }
+        finally {
+            globalThis.clearTimeout(transferTimeout);
+        }
+        const transferDurationMs = Date.now() - transferStartedAt;
+        this.recordDiagnostic({
+            operation: "context_transfer",
+            status: "succeeded",
+            context_key: update.key,
+            payload_bytes: payloadBytes,
+            duration_ms: transferDurationMs,
+        });
+        if (transferDurationMs > 2_000)
+            this.flushDiagnostics("slow_operation");
         if (generation !== this.contextGeneration)
             throw new Error("Context transfer interrupted by recovery");
         const { payload } = transfer;
@@ -611,7 +682,7 @@ export class RealtimeSession {
     bindRoomEvents() {
         this.room.on(RoomEvent.ParticipantDisconnected, (participant) => {
             if (participant.isAgent && !this.disconnectMode) {
-                this.startAutoRejoin(new Error("Agent disconnected from the room"));
+                this.startAutoRejoin(new Error("Agent disconnected from the room"), { trigger: "agent_disconnected" });
             }
         });
         this.room.on(RoomEvent.SignalReconnecting, () => this.handleLiveKitReconnecting());
@@ -720,7 +791,7 @@ export class RealtimeSession {
         this.reconnectStartedAt ??= Date.now();
         this.rejectPendingCommands(new Error("Connection interrupted before command completion"), true);
         if (this.activeRecovery === null) {
-            this.beginRecovery("livekit", 1);
+            this.beginRecovery("livekit", 1, { trigger: "livekit_reconnecting" });
         }
         this.setState("reconnecting");
     }
@@ -738,7 +809,7 @@ export class RealtimeSession {
             this.reconnectStartedAt = null;
         }
         if (this.hasConnectedValue && this.activeRecovery === null) {
-            this.beginRecovery("livekit", 1);
+            this.beginRecovery("livekit", 1, { trigger: "livekit_reconnecting" });
         }
         this.connectionId = `conn_${Math.random().toString(36).slice(2)}_${Date.now().toString(36)}`;
         this.clientSeq = 0;
@@ -762,7 +833,7 @@ export class RealtimeSession {
                 return;
             const normalizedError = toError(error);
             if (this.shouldAutoRejoin()) {
-                this.startAutoRejoin(normalizedError);
+                this.startAutoRejoin(normalizedError, { trigger: "livekit_resume_failed" });
             }
             else {
                 this.failRecovery(normalizedError);
@@ -792,7 +863,7 @@ export class RealtimeSession {
             return;
         }
         if (this.shouldAutoRejoin(reason)) {
-            this.startAutoRejoin(new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`));
+            this.startAutoRejoin(new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`), { trigger: "livekit_disconnected" });
             return;
         }
         if (this.activeRecovery !== null) {
@@ -816,12 +887,22 @@ export class RealtimeSession {
         this.endNotificationDelivered = true;
         this.notifyListeners(this.endedListeners, event);
     }
-    beginRecovery(strategy, attempt) {
-        this.activeRecovery = { strategy, attempt };
+    beginRecovery(strategy, attempt, details) {
+        this.activeRecovery = { strategy, attempt, details };
+        this.recordDiagnostic({
+            operation: "connection_recovery",
+            status: "started",
+            reason: details.trigger,
+            message_id: details.messageId,
+            context_key: details.contextKey,
+            timeout_ms: details.timeoutMs,
+        });
+        this.flushDiagnostics("recovery_started");
         this.notifyListeners(this.recoveryListeners, {
             status: "started",
             strategy,
             attempt,
+            ...details,
         });
     }
     completeRecovery() {
@@ -833,6 +914,7 @@ export class RealtimeSession {
             status: "succeeded",
             strategy: recovery.strategy,
             attempt: recovery.attempt,
+            ...recovery.details,
         });
     }
     failRecovery(error) {
@@ -844,6 +926,7 @@ export class RealtimeSession {
             status: "failed",
             strategy: recovery.strategy,
             attempt: recovery.attempt,
+            ...recovery.details,
             error,
         });
     }
@@ -854,7 +937,7 @@ export class RealtimeSession {
             && !["disconnected", "failed"].includes(this.stateValue)
             && !isTerminalDisconnectReason(reason));
     }
-    startAutoRejoin(initialError) {
+    startAutoRejoin(initialError, details) {
         if (this.autoRejoinPromise || !this.shouldAutoRejoin())
             return;
         this.agentReady = false;
@@ -875,7 +958,7 @@ export class RealtimeSession {
             this.failRecovery(error);
             this.setState("failed");
         }, this.autoRejoinOptions.timeoutMs);
-        const promise = this.runAutoRejoin(initialError, controller.signal)
+        const promise = this.runAutoRejoin(initialError, controller.signal, details)
             .finally(() => {
             globalThis.clearTimeout(timeout);
             if (this.autoRejoinController === controller) {
@@ -886,7 +969,7 @@ export class RealtimeSession {
         this.autoRejoinPromise = promise;
         void promise;
     }
-    async runAutoRejoin(initialError, signal) {
+    async runAutoRejoin(initialError, signal, details) {
         const options = this.autoRejoinOptions;
         if (!options)
             return;
@@ -904,7 +987,7 @@ export class RealtimeSession {
                     await abortableDelay(delayMs + Math.random() * Math.min(delayMs, 500), signal);
                 }
                 throwIfAborted(signal);
-                this.beginRecovery("rejoin", attempt);
+                this.beginRecovery("rejoin", attempt, details);
                 await this.performRejoin(this.apiBase, signal);
                 throwIfAborted(signal);
                 this.reconnectStartedAt = null;
@@ -1023,6 +1106,9 @@ export class RealtimeSession {
         }
         if (event.type.endsWith(".error"))
             this.notifyListeners(this.errorListeners, event);
+        const message = this.transcriptStore.accept(event);
+        if (message)
+            this.notifyListeners(this.transcriptListeners, message);
         const transcript = transcriptEvent(event);
         if (!transcript)
             return;
@@ -1055,6 +1141,21 @@ export class RealtimeSession {
             this.pendingReconnect = null;
     }
     async sendTrackedCommand(message, timeoutMs, retryTransfer) {
+        const contextKey = contextKeyFromMessage(message);
+        const payloadBytes = contextPayloadBytes(message.payload);
+        const startedAt = Date.now();
+        this.recordDiagnostic({
+            operation: "command_send",
+            status: "started",
+            message_id: message.message_id,
+            connection_id: message.connection_id,
+            client_seq: message.client_seq,
+            context_key: contextKey,
+            payload_bytes: payloadBytes,
+            retry_count: 0,
+            timeout_ms: timeoutMs,
+            event_type: diagnosticEventType(message.type),
+        });
         const response = new Promise((resolve, reject) => {
             const timeout = globalThis.setTimeout(() => {
                 const pending = this.pendingCommands.get(message.message_id);
@@ -1062,17 +1163,46 @@ export class RealtimeSession {
                     globalThis.clearInterval(pending.retry);
                 this.pendingCommands.delete(message.message_id);
                 const error = new Error(`Command ${message.type} timed out after ${timeoutMs}ms`);
+                this.recordDiagnostic({
+                    operation: "command_send",
+                    status: "timed_out",
+                    reason: message.type === "context.update"
+                        ? "context_command_timeout"
+                        : "command_timeout",
+                    message_id: message.message_id,
+                    connection_id: message.connection_id,
+                    client_seq: message.client_seq,
+                    context_key: contextKey,
+                    payload_bytes: payloadBytes,
+                    duration_ms: Date.now() - startedAt,
+                    retry_count: pending?.retryCount ?? 0,
+                    timeout_ms: timeoutMs,
+                    event_type: diagnosticEventType(message.type),
+                });
+                this.flushDiagnostics("command_timeout");
                 reject(error);
-                if (message.type === "context.update")
-                    this.startAutoRejoin(error);
+                if (message.type === "context.update") {
+                    this.startAutoRejoin(error, {
+                        trigger: "context_command_timeout",
+                        operation: "command_send",
+                        messageId: message.message_id,
+                        contextKey: contextKeyFromMessage(message),
+                        timeoutMs,
+                    });
+                }
             }, timeoutMs);
             const retry = globalThis.setInterval(() => {
-                void this.publishCommand(message).catch(() => {
+                void this.publishCommandAttempt(message, true).catch(() => {
                     // A reconnect may temporarily reject writes; retry until timeout.
                 });
             }, 500);
             this.pendingCommands.set(message.message_id, {
                 commandType: message.type,
+                message,
+                startedAt,
+                contextKey,
+                payloadBytes,
+                retryCount: 0,
                 resolve,
                 reject,
                 timeout,
@@ -1080,14 +1210,14 @@ export class RealtimeSession {
                 retryTransfer: retryTransfer
                     ? async () => {
                         await retryTransfer();
-                        await this.publishCommand(message);
+                        await this.publishCommandAttempt(message, true);
                     }
                     : undefined,
                 transferRetrying: false,
             });
         });
         try {
-            await this.publishCommand(message);
+            await this.publishCommandAttempt(message, false);
         }
         catch {
             // The reliable data channel may still be warming immediately after a
@@ -1096,9 +1226,114 @@ export class RealtimeSession {
         }
         return response;
     }
+    async publishCommandAttempt(message, retrying) {
+        const pending = this.pendingCommands.get(message.message_id);
+        if (!pending)
+            return;
+        if (retrying) {
+            pending.retryCount += 1;
+            this.recordDiagnostic({
+                operation: "command_send",
+                status: "retrying",
+                message_id: message.message_id,
+                connection_id: message.connection_id,
+                client_seq: message.client_seq,
+                context_key: pending.contextKey,
+                payload_bytes: pending.payloadBytes,
+                retry_count: pending.retryCount,
+                event_type: diagnosticEventType(message.type),
+            });
+        }
+        const attemptStartedAt = Date.now();
+        try {
+            await this.publishCommand(message);
+        }
+        catch (error) {
+            this.recordDiagnostic({
+                operation: "command_send",
+                status: "failed",
+                reason: "command_publish_failed",
+                message_id: message.message_id,
+                connection_id: message.connection_id,
+                client_seq: message.client_seq,
+                context_key: pending.contextKey,
+                payload_bytes: pending.payloadBytes,
+                duration_ms: Date.now() - attemptStartedAt,
+                retry_count: pending.retryCount,
+                event_type: diagnosticEventType(message.type),
+            });
+            this.flushDiagnostics("command_failed");
+            throw error;
+        }
+        const durationMs = Date.now() - attemptStartedAt;
+        this.recordDiagnostic({
+            operation: "command_send",
+            status: "succeeded",
+            message_id: message.message_id,
+            connection_id: message.connection_id,
+            client_seq: message.client_seq,
+            context_key: pending.contextKey,
+            payload_bytes: pending.payloadBytes,
+            duration_ms: durationMs,
+            retry_count: pending.retryCount,
+            event_type: diagnosticEventType(message.type),
+        });
+        if (durationMs > 2_000)
+            this.flushDiagnostics("slow_operation");
+    }
     async publishCommand(message) {
         await this.room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { reliable: true, topic: "voxevolv.commands" });
     }
+    recordDiagnostic(entry) {
+        this.diagnostics.record({
+            ...entry,
+            connection_state: this.stateValue,
+            buffered_bytes: this.reliableDataChannelBufferedBytes(),
+        });
+    }
+    reliableDataChannelBufferedBytes() {
+        try {
+            const bufferedAmount = this.room.engine.dataChannelForKind(0)?.bufferedAmount;
+            return typeof bufferedAmount === "number" ? bufferedAmount : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    flushDiagnostics(trigger) {
+        void this.diagnostics.flush(trigger, (report) => this.postDiagnostics(report));
+    }
+    async postDiagnostics(report) {
+        const controller = new AbortController();
+        let timeout;
+        const timeoutExpired = new Promise((_resolve, reject) => {
+            timeout = globalThis.setTimeout(() => {
+                controller.abort();
+                reject(new Error(`Diagnostic report timed out after ${this.diagnosticReportTimeoutMs}ms`));
+            }, this.diagnosticReportTimeoutMs);
+        });
+        try {
+            const response = await Promise.race([
+                this.fetcher(`${this.apiBase}/sessions/${this.id}/diagnostics`, {
+                    method: "POST",
+                    headers: {
+                        Authorization: `Bearer ${this.accessToken}`,
+                        "Content-Type": "application/json",
+                    },
+                    credentials: this.lifecycleCredentials,
+                    body: JSON.stringify(report),
+                    signal: controller.signal,
+                }),
+                timeoutExpired,
+            ]);
+            if (!response.ok)
+                throw new Error(`Diagnostic report failed (${response.status})`);
+        }
+        finally {
+            if (timeout !== undefined)
+                globalThis.clearTimeout(timeout);
+        }
+    }
     waitForRemoteAudioTrack(deadline, signal) {
         if ((this.outputMediaStream?.getAudioTracks().length ?? 0) > 0)
             return Promise.resolve();
@@ -1162,7 +1397,7 @@ export class RealtimeSession {
         if (event.type !== "agent.ready" && !snapshotReady)
             return;
         if (this.agentStreamId && this.agentStreamId !== event.stream_id && !this.activeRecovery) {
-            this.startAutoRejoin(new Error("Agent worker was replaced"));
+            this.startAutoRejoin(new Error("Agent worker was replaced"), { trigger: "worker_replaced" });
             return;
         }
         this.agentStreamId = event.stream_id;
@@ -1184,6 +1419,7 @@ export class RealtimeSession {
         if (!pending)
             return;
         if (event.type === "command.ack" && isCommandAckPayload(event.payload)) {
+            this.recordCommandResult(pending, event.type, "acknowledged");
             globalThis.clearTimeout(pending.timeout);
             globalThis.clearInterval(pending.retry);
             this.pendingCommands.delete(event.payload.message_id);
@@ -1191,6 +1427,7 @@ export class RealtimeSession {
             return;
         }
         if (event.type === "context.updated" && isContextUpdatedPayload(event.payload)) {
+            this.recordCommandResult(pending, event.type, "acknowledged");
             globalThis.clearTimeout(pending.timeout);
             globalThis.clearInterval(pending.retry);
             this.pendingCommands.delete(event.payload.message_id);
@@ -1199,6 +1436,8 @@ export class RealtimeSession {
         }
         if ((event.type === "command.error" && isCommandErrorPayload(event.payload))
             || (event.type === "context.rejected" && isContextRejectedPayload(event.payload))) {
+            this.recordCommandResult(pending, event.type, "rejected", event.payload.code);
+            this.flushDiagnostics("command_rejected");
             if (event.type === "context.rejected" && event.payload.retryable) {
                 if (pending.retryTransfer && !pending.transferRetrying) {
                     pending.transferRetrying = true;
@@ -1220,12 +1459,31 @@ export class RealtimeSession {
         for (const [messageId, pending] of this.pendingCommands) {
             if (preserveContext && pending.commandType === "context.update")
                 continue;
+            this.recordCommandResult(pending, pending.commandType === "context.update" ? "context.rejected" : "command.error", "failed", "pending_command_cancelled");
             globalThis.clearTimeout(pending.timeout);
             globalThis.clearInterval(pending.retry);
             pending.reject(error);
             this.pendingCommands.delete(messageId);
         }
     }
+    recordCommandResult(pending, eventType, status, reason) {
+        const durationMs = Date.now() - pending.startedAt;
+        this.recordDiagnostic({
+            operation: "command_result",
+            status,
+            reason,
+            message_id: pending.message.message_id,
+            connection_id: pending.message.connection_id,
+            client_seq: pending.message.client_seq,
+            context_key: pending.contextKey,
+            payload_bytes: pending.payloadBytes,
+            duration_ms: durationMs,
+            retry_count: pending.retryCount,
+            event_type: eventType,
+        });
+        if (durationMs > 2_000)
+            this.flushDiagnostics("slow_operation");
+    }
     async fetchReplay(apiBase, afterSeq) {
         let nextAfter = afterSeq;
         while (true) {
@@ -1374,6 +1632,43 @@ function contextStreamId() {
         return globalThis.crypto.randomUUID();
     return `ctx_${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`;
 }
+function contextUpdateBytes(update) {
+    return "image" in update
+        ? update.image.size
+        : new TextEncoder().encode(update.text).byteLength;
+}
+function contextPayloadBytes(payload) {
+    const image = payload.image;
+    if (image && typeof image === "object" && "bytes" in image) {
+        const bytes = image.bytes;
+        if (typeof bytes === "number")
+            return bytes;
+    }
+    const textStream = payload.text_stream;
+    if (textStream && typeof textStream === "object" && "bytes" in textStream) {
+        const bytes = textStream.bytes;
+        if (typeof bytes === "number")
+            return bytes;
+    }
+    if (typeof payload.text === "string") {
+        return new TextEncoder().encode(payload.text).byteLength;
+    }
+    try {
+        return new TextEncoder().encode(JSON.stringify(payload)).byteLength;
+    }
+    catch {
+        return undefined;
+    }
+}
+function contextKeyFromMessage(message) {
+    if (message.type !== "context.update")
+        return undefined;
+    const key = message.payload.key;
+    return typeof key === "string" ? key : undefined;
+}
+function diagnosticEventType(type) {
+    return type === "client.hello" || type === "context.update" ? type : undefined;
+}
 function validateSessionCredentials(value) {
     if (!value || typeof value !== "object") {
         throw new TypeError("Session credentials must be an object");