@estuary-ai/sdk 0.1.23 → 0.1.24

package/README.md

@@ -120,6 +120,15 @@ import { parseActions } from '@estuary-ai/sdk';
 const { actions, cleanText } = parseActions(rawBotText);
 ```
 
+### Character Info
+
+Fetch character details (name, avatar, 3D model URLs):
+
+```typescript
+const character = await client.getCharacter();
+console.log(character.name, character.avatar);
+```
+
 ### Memory & Knowledge Graph
 
 ```typescript
@@ -237,9 +246,21 @@ interface EstuaryConfig {
   voiceTransport?: 'websocket' | 'livekit' | 'auto'; // Default: 'auto'
   realtimeMemory?: boolean;    // Enable real-time memory extraction events. Default: false
   suppressMicDuringPlayback?: boolean; // Mute mic while bot audio plays (software AEC). Default: false
+  autoInterruptOnSpeech?: boolean;     // Interrupt bot audio when user speaks. Default: true
 }
 ```
 
+## Runtime Properties
+
+```typescript
+client.connectionState     // ConnectionState enum (Disconnected, Connecting, Connected, ...)
+client.isConnected         // boolean shorthand
+client.isVoiceActive       // true while voice session is running
+client.isMuted             // current mute state
+client.suppressMicDuringPlayback // get/set at runtime without reconnecting
+client.session             // SessionInfo | null after connect
+```
+
 ## Exports
 
 Key exports:
@@ -261,6 +282,7 @@ import { parseActions } from '@estuary-ai/sdk';
 import type {
   EstuaryConfig,
   SessionInfo,
+  CharacterInfo,
   BotResponse,
   BotVoice,
   SttResponse,

package/dist/index.d.mts

@@ -147,6 +147,8 @@ type EstuaryEventMap = {
     livekitDisconnected: () => void;
     audioPlaybackStarted: (messageId: string) => void;
     audioPlaybackComplete: (messageId: string) => void;
+    /** Bot audio level 0.0–1.0, emitted during playback for both transports. */
+    botAudioLevel: (level: number) => void;
     memoryUpdated: (event: MemoryUpdatedEvent) => void;
 };
 interface VoiceManager {
@@ -155,6 +157,10 @@ interface VoiceManager {
     toggleMute(): void;
     /** Suppress audio sending (software AEC). No-op if not supported. */
     setSuppressed?(suppressed: boolean): void;
+    /** Set callback for speaking state from participant attributes (LiveKit only). */
+    setSpeakingStateCallback?(cb: (speaking: boolean) => void): void;
+    /** Set callback for audio level updates (0-1) during bot speech. */
+    setAudioLevelCallback?(cb: (level: number) => void): void;
     readonly isMuted: boolean;
     readonly isActive: boolean;
     dispose(): void;
@@ -368,6 +374,8 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
     private forwardSocketEvents;
     private handleBotResponse;
     private handleBotVoice;
+    /** Compute RMS audio level (0-1) from base64-encoded Int16 PCM. */
+    private computeAudioLevel;
     private maybeAutoInterrupt;
 }
 

package/dist/index.d.ts

@@ -147,6 +147,8 @@ type EstuaryEventMap = {
     livekitDisconnected: () => void;
     audioPlaybackStarted: (messageId: string) => void;
     audioPlaybackComplete: (messageId: string) => void;
+    /** Bot audio level 0.0–1.0, emitted during playback for both transports. */
+    botAudioLevel: (level: number) => void;
     memoryUpdated: (event: MemoryUpdatedEvent) => void;
 };
 interface VoiceManager {
@@ -155,6 +157,10 @@ interface VoiceManager {
     toggleMute(): void;
     /** Suppress audio sending (software AEC). No-op if not supported. */
     setSuppressed?(suppressed: boolean): void;
+    /** Set callback for speaking state from participant attributes (LiveKit only). */
+    setSpeakingStateCallback?(cb: (speaking: boolean) => void): void;
+    /** Set callback for audio level updates (0-1) during bot speech. */
+    setAudioLevelCallback?(cb: (level: number) => void): void;
     readonly isMuted: boolean;
     readonly isActive: boolean;
     dispose(): void;
@@ -368,6 +374,8 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
     private forwardSocketEvents;
     private handleBotResponse;
     private handleBotVoice;
+    /** Compute RMS audio level (0-1) from base64-encoded Int16 PCM. */
+    private computeAudioLevel;
     private maybeAutoInterrupt;
 }
 

package/dist/index.js

@@ -247,6 +247,13 @@ var init_livekit_voice = __esm({
       // livekit-client Room (dynamically imported)
       _isMuted = false;
       _isActive = false;
+      speakingStateCallback = null;
+      audioLevelCallback = null;
+      // Audio analyser (via livekit-client's createAudioAnalyser)
+      calculateVolume = null;
+      analyserCleanup = null;
+      audioLevelPollTimer = null;
+      _isBotSpeaking = false;
       constructor(socketManager, logger) {
         this.socketManager = socketManager;
         this.logger = logger;
@@ -257,6 +264,12 @@ var init_livekit_voice = __esm({
       get isActive() {
         return this._isActive;
       }
+      setSpeakingStateCallback(cb) {
+        this.speakingStateCallback = cb;
+      }
+      setAudioLevelCallback(cb) {
+        this.audioLevelCallback = cb;
+      }
       async start() {
         if (this._isActive) {
           throw new exports.EstuaryError("VOICE_ALREADY_ACTIVE" /* VOICE_ALREADY_ACTIVE */, "Voice is already active");
@@ -296,16 +309,24 @@ var init_livekit_voice = __esm({
             }
             audioElement.play().catch(() => {
             });
+            this.setupAnalyser(track);
+            if (this._isBotSpeaking) {
+              setTimeout(() => this.startAudioLevelPolling(), 50);
+            }
           }
         });
         this.room.on(RoomEvent.TrackUnsubscribed, (track) => {
           if (track.kind === Track.Kind.Audio) {
+            this.teardownAnalyser();
             track.detach().forEach((el) => el.remove());
           }
         });
         this.room.on(RoomEvent.Disconnected, () => {
           this.logger.debug("LiveKit room disconnected");
           this._isActive = false;
+          this._isBotSpeaking = false;
+          this.teardownAnalyser();
+          this.speakingStateCallback?.(false);
         });
         try {
           await this.room.connect(tokenData.url, tokenData.token);
@@ -319,6 +340,23 @@ var init_livekit_voice = __esm({
             err
           );
         }
+        this.room.on(
+          RoomEvent.ParticipantAttributesChanged,
+          (changedAttributes, participant) => {
+            if (participant === this.room?.localParticipant) return;
+            const state = changedAttributes["estuary.state"];
+            if (state === "speaking") {
+              this._isBotSpeaking = true;
+              this.speakingStateCallback?.(true);
+              this.startAudioLevelPolling();
+            } else if (state === "idle") {
+              this._isBotSpeaking = false;
+              this.stopAudioLevelPolling();
+              this.speakingStateCallback?.(false);
+              this.audioLevelCallback?.(0);
+            }
+          }
+        );
         try {
           await this.room.localParticipant.setMicrophoneEnabled(true);
           this.logger.debug("Microphone enabled");
@@ -342,6 +380,9 @@ var init_livekit_voice = __esm({
           this.socketManager.emitEvent("livekit_leave");
         } catch {
         }
+        this._isBotSpeaking = false;
+        this.teardownAnalyser();
+        this.speakingStateCallback?.(false);
         if (this.room) {
           for (const [, publication] of this.room.localParticipant.trackPublications) {
             if (publication.track) {
@@ -362,6 +403,10 @@ var init_livekit_voice = __esm({
         this.logger.debug("Mute toggled:", this._isMuted);
       }
       dispose() {
+        this.speakingStateCallback = null;
+        this.audioLevelCallback = null;
+        this._isBotSpeaking = false;
+        this.teardownAnalyser();
         if (this.room) {
           this.room.disconnect();
           this.room = null;
@@ -369,6 +414,53 @@ var init_livekit_voice = __esm({
         this._isActive = false;
         this._isMuted = false;
       }
+      // ─── Audio Analyser (livekit-client built-in) ───────────────────
+      async setupAnalyser(track) {
+        this.teardownAnalyser();
+        try {
+          const { createAudioAnalyser } = await import('livekit-client');
+          const { analyser, calculateVolume, cleanup } = createAudioAnalyser(track, {
+            fftSize: 256,
+            smoothingTimeConstant: 0.3
+          });
+          if (analyser.context.state === "suspended") {
+            await analyser.context.resume();
+          }
+          this.calculateVolume = calculateVolume;
+          this.analyserCleanup = cleanup;
+          this.logger.debug("Audio analyser created for bot track");
+        } catch (err) {
+          this.logger.debug("Failed to create audio analyser:", err);
+        }
+      }
+      teardownAnalyser() {
+        this.stopAudioLevelPolling();
+        if (this.analyserCleanup) {
+          this.analyserCleanup().catch(() => {
+          });
+          this.analyserCleanup = null;
+        }
+        this.calculateVolume = null;
+      }
+      startAudioLevelPolling() {
+        if (this.audioLevelPollTimer !== null) return;
+        if (!this.calculateVolume) return;
+        this.audioLevelPollTimer = setInterval(() => {
+          if (!this.calculateVolume) {
+            this.stopAudioLevelPolling();
+            return;
+          }
+          const vol = this.calculateVolume();
+          this.audioLevelCallback?.(vol);
+        }, 33);
+      }
+      stopAudioLevelPolling() {
+        if (this.audioLevelPollTimer !== null) {
+          clearInterval(this.audioLevelPollTimer);
+          this.audioLevelPollTimer = null;
+        }
+      }
+      // ─── Private ────────────────────────────────────────────────────
       requestToken() {
         return new Promise((resolve, reject) => {
           const timeout = setTimeout(() => {
@@ -1277,6 +1369,7 @@ var EstuaryClient = class extends TypedEventEmitter {
           }
         } else if (event.type === "complete") {
           this.emit("audioPlaybackComplete", event.messageId);
+          this.emit("botAudioLevel", 0);
           this.notifyAudioPlaybackComplete(event.messageId);
           if (this.config.suppressMicDuringPlayback) {
             this.voiceManager?.setSuppressed?.(false);
@@ -1285,6 +1378,24 @@ var EstuaryClient = class extends TypedEventEmitter {
       });
     }
     await this.voiceManager.start();
+    this.voiceManager.setSpeakingStateCallback?.((speaking) => {
+      if (speaking) {
+        this.emit("audioPlaybackStarted", "livekit-audio");
+        if (this.config.suppressMicDuringPlayback) {
+          this.voiceManager?.setSuppressed?.(true);
+        }
+      } else {
+        this.emit("audioPlaybackComplete", "livekit-audio");
+        this.emit("botAudioLevel", 0);
+        this.notifyAudioPlaybackComplete("livekit-audio");
+        if (this.config.suppressMicDuringPlayback) {
+          this.voiceManager?.setSuppressed?.(false);
+        }
+      }
+    });
+    this.voiceManager.setAudioLevelCallback?.((level) => {
+      this.emit("botAudioLevel", level);
+    });
     this.emit("voiceStarted");
   }
   /** Stop voice input */
@@ -1383,7 +1494,31 @@ var EstuaryClient = class extends TypedEventEmitter {
   }
   handleBotVoice(voice) {
     this.emit("botVoice", voice);
-    this.audioPlayer?.enqueue(voice);
+    if (voice.audio) {
+      this.audioPlayer?.enqueue(voice);
+      this.emit("botAudioLevel", this.computeAudioLevel(voice.audio));
+    }
+  }
+  /** Compute RMS audio level (0-1) from base64-encoded Int16 PCM. */
+  computeAudioLevel(base64Audio) {
+    try {
+      const binaryStr = atob(base64Audio);
+      const len = binaryStr.length;
+      const step = 16;
+      let sum = 0;
+      let count = 0;
+      for (let i = 0; i + 1 < len; i += step) {
+        const sample = binaryStr.charCodeAt(i) | binaryStr.charCodeAt(i + 1) << 8;
+        const signed = sample > 32767 ? sample - 65536 : sample;
+        const normalized = signed / 32768;
+        sum += normalized * normalized;
+        count++;
+      }
+      if (count === 0) return 0;
+      return Math.min(1, Math.sqrt(sum / count) * 5);
+    } catch {
+      return 0;
+    }
   }
   maybeAutoInterrupt(stt) {
     if ((this.config.autoInterruptOnSpeech ?? true) === false) return;