@estuary-ai/sdk 0.1.4 → 0.1.6

package/README.md

@@ -74,6 +74,50 @@ await client.startVoice();
 client.toggleMute();
 ```
 
+### Interrupts
+
+Interrupt the bot's current response (stops audio playback and generation):
+
+```typescript
+client.interrupt();                // interrupt current response
+client.interrupt('msg_abc123');    // interrupt a specific message
+```
+
+### Vision / Camera
+
+Send images for vision processing. The server may also request captures via the `cameraCaptureRequest` event.
+
+```typescript
+// Send a camera image proactively
+client.sendCameraImage(base64Image, 'image/jpeg');
+
+// Respond to a server-initiated capture request
+client.on('cameraCaptureRequest', (request) => {
+  const image = captureFrame(); // your capture logic
+  client.sendCameraImage(image, 'image/jpeg', request.requestId, request.text);
+});
+```
+
+### Character Actions
+
+Bot responses can include inline action tags (e.g., `<action name="wave" target="user"/>`). The SDK automatically parses these, strips them from `botResponse.text`, and emits `characterAction` events:
+
+```typescript
+client.on('characterAction', (action) => {
+  console.log(action.name);      // e.g., "wave"
+  console.log(action.params);    // e.g., { target: "user" }
+  console.log(action.messageId); // originating message
+});
+```
+
+For non-streaming contexts, use the `parseActions` utility:
+
+```typescript
+import { parseActions } from '@estuary-ai/sdk';
+
+const { actions, cleanText } = parseActions(rawBotText);
+```
+
 ### Memory & Knowledge Graph
 
 ```typescript
@@ -81,6 +125,9 @@ const memories = await client.memory.getMemories({ status: 'active', limit: 50 }
 const facts = await client.memory.getCoreFacts();
 const graph = await client.memory.getGraph({ includeEntities: true });
 const results = await client.memory.search('favorite food');
+const timeline = await client.memory.getTimeline({ groupBy: 'week' });
+const stats = await client.memory.getStats();
+await client.memory.deleteAll(true); // pass true to confirm
 ```
 
 ### Real-Time Memory Extraction
@@ -109,17 +156,69 @@ await client.connect();
 ## Events
 
 ```typescript
+// Connection
 client.on('connected', (session) => { /* authenticated */ });
 client.on('disconnected', (reason) => { /* lost connection */ });
-client.on('botResponse', (response) => { /* streaming text */ });
+client.on('reconnecting', (attempt) => { /* reconnect attempt number */ });
+client.on('connectionStateChanged', (state) => { /* ConnectionState enum */ });
+client.on('authError', (error) => { /* authentication failed */ });
+
+// Conversation
+client.on('botResponse', (response) => { /* streaming text (actions auto-stripped) */ });
 client.on('botVoice', (voice) => { /* audio chunk */ });
-client.on('sttResponse', (stt) => { /* speech-to-text */ });
+client.on('sttResponse', (stt) => { /* speech-to-text transcript */ });
 client.on('interrupt', (data) => { /* response interrupted */ });
+client.on('characterAction', (action) => { /* parsed action from bot response */ });
+client.on('cameraCaptureRequest', (request) => { /* server requests a camera image */ });
+
+// Voice
+client.on('voiceStarted', () => { /* voice session began */ });
+client.on('voiceStopped', () => { /* voice session ended */ });
+client.on('livekitConnected', (room) => { /* joined LiveKit room */ });
+client.on('livekitDisconnected', () => { /* left LiveKit room */ });
+
+// Audio playback
+client.on('audioPlaybackStarted', (messageId) => { /* bot audio started playing */ });
+client.on('audioPlaybackComplete', (messageId) => { /* bot audio finished playing */ });
+
+// Memory
 client.on('memoryUpdated', (event) => { /* real-time memory extraction */ });
+
+// Errors & limits
 client.on('error', (error) => { /* EstuaryError */ });
 client.on('quotaExceeded', (data) => { /* rate limited */ });
 ```
 
+## Error Handling
+
+Errors are instances of `EstuaryError` with a typed `code` field:
+
+```typescript
+import { EstuaryError, ErrorCode } from '@estuary-ai/sdk';
+
+client.on('error', (error) => {
+  if (error instanceof EstuaryError) {
+    switch (error.code) {
+      case ErrorCode.NOT_CONNECTED:
+      case ErrorCode.CONNECTION_FAILED:
+      case ErrorCode.CONNECTION_TIMEOUT:
+        // connection issues
+        break;
+      case ErrorCode.AUTH_FAILED:
+        // bad API key or character ID
+        break;
+      case ErrorCode.MICROPHONE_DENIED:
+        // user denied mic permission
+        break;
+    }
+  }
+});
+
+client.on('authError', (message) => {
+  console.error('Authentication failed:', message);
+});
+```
+
 ## Configuration
 
 ```typescript
@@ -135,9 +234,46 @@ interface EstuaryConfig {
   debug?: boolean;             // Default: false
   voiceTransport?: 'websocket' | 'livekit' | 'auto'; // Default: 'auto'
   realtimeMemory?: boolean;    // Enable real-time memory extraction events. Default: false
+  suppressMicDuringPlayback?: boolean; // Mute mic while bot audio plays (software AEC). Default: false
 }
 ```
 
+## Exports
+
+Key exports for TypeScript users:
+
+```typescript
+// Client
+import { EstuaryClient } from '@estuary-ai/sdk';
+
+// Errors
+import { EstuaryError, ErrorCode } from '@estuary-ai/sdk';
+
+// Enums
+import { ConnectionState } from '@estuary-ai/sdk';
+
+// Utilities
+import { parseActions } from '@estuary-ai/sdk';
+
+// Types (import type)
+import type {
+  EstuaryConfig,
+  SessionInfo,
+  BotResponse,
+  BotVoice,
+  SttResponse,
+  InterruptData,
+  CameraCaptureRequest,
+  CharacterAction,
+  QuotaExceededData,
+  MemoryData,
+  MemoryUpdatedEvent,
+  EstuaryEventMap,
+  ParsedAction,
+  MemoryClient,
+} from '@estuary-ai/sdk';
+```
+
 ## Requirements
 
 - Node.js 18+ or modern browser

package/dist/index.d.mts

@@ -21,6 +21,8 @@ interface EstuaryConfig {
     voiceTransport?: VoiceTransport;
     /** Enable real-time memory extraction after each response (default: false) */
     realtimeMemory?: boolean;
+    /** Suppress mic during TTS playback (software AEC fallback, disables barge-in). Default: false */
+    suppressMicDuringPlayback?: boolean;
 }
 type VoiceTransport = 'websocket' | 'livekit' | 'auto';
 declare enum ConnectionState {
@@ -130,6 +132,8 @@ interface VoiceManager {
     start(): Promise<void>;
     stop(): Promise<void>;
     toggleMute(): void;
+    /** Suppress audio sending (software AEC). No-op if not supported. */
+    setSuppressed?(suppressed: boolean): void;
     readonly isMuted: boolean;
     readonly isActive: boolean;
     dispose(): void;

package/dist/index.js

@@ -4889,6 +4889,7 @@ var init_websocket_voice = __esm({
       scriptProcessor = null;
       sourceNode = null;
       _isMuted = false;
+      _isSuppressed = false;
       _isActive = false;
       constructor(socketManager, sampleRate, logger) {
         this.socketManager = socketManager;
@@ -4911,7 +4912,13 @@ var init_websocket_voice = __esm({
         let stream;
         try {
           stream = await navigator.mediaDevices.getUserMedia({
-            audio: { sampleRate: this.sampleRate, channelCount: 1 }
+            audio: {
+              sampleRate: this.sampleRate,
+              channelCount: 1,
+              echoCancellation: true,
+              noiseSuppression: true,
+              autoGainControl: true
+            }
           });
         } catch (err) {
           throw new exports.EstuaryError(
@@ -4928,7 +4935,7 @@ var init_websocket_voice = __esm({
         const nativeRate = this.audioContext.sampleRate;
         const targetRate = this.sampleRate;
         this.scriptProcessor.onaudioprocess = (event) => {
-          if (this._isMuted) return;
+          if (this._isMuted || this._isSuppressed) return;
           const inputData = event.inputBuffer.getChannelData(0);
           let pcmFloat;
           if (nativeRate !== targetRate) {
@@ -4958,6 +4965,7 @@ var init_websocket_voice = __esm({
         this.cleanup();
         this._isActive = false;
         this._isMuted = false;
+        this._isSuppressed = false;
         this.logger.debug("WebSocket voice stopped");
       }
       toggleMute() {
@@ -4968,10 +4976,15 @@ var init_websocket_voice = __esm({
         }
         this.logger.debug("Mute toggled:", this._isMuted);
       }
+      setSuppressed(suppressed) {
+        this._isSuppressed = suppressed;
+        this.logger.debug("Audio suppression:", suppressed ? "on" : "off");
+      }
       dispose() {
         this.cleanup();
         this._isActive = false;
         this._isMuted = false;
+        this._isSuppressed = false;
       }
       cleanup() {
         if (this.scriptProcessor) {
@@ -9184,6 +9197,8 @@ var AudioPlayer = class {
   sampleRate;
   onEvent;
   audioContext = null;
+  mediaStreamDest = null;
+  audioElement = null;
   queue = [];
   currentSource = null;
   currentMessageId = null;
@@ -9219,6 +9234,16 @@ var AudioPlayer = class {
   }
   dispose() {
     this.clear();
+    if (this.audioElement) {
+      this.audioElement.pause();
+      this.audioElement.srcObject = null;
+      this.audioElement.remove();
+      this.audioElement = null;
+    }
+    if (this.mediaStreamDest) {
+      this.mediaStreamDest.disconnect();
+      this.mediaStreamDest = null;
+    }
     if (this.audioContext) {
       this.audioContext.close().catch(() => {
       });
@@ -9231,8 +9256,20 @@ var AudioPlayer = class {
       return null;
     }
     const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
-    this.audioContext = new AudioCtx({ sampleRate: this.sampleRate });
-    return this.audioContext;
+    const ctx = new AudioCtx({ sampleRate: this.sampleRate });
+    this.audioContext = ctx;
+    if (typeof document !== "undefined") {
+      this.mediaStreamDest = ctx.createMediaStreamDestination();
+      const el = document.createElement("audio");
+      el.srcObject = this.mediaStreamDest.stream;
+      el.autoplay = true;
+      el.style.display = "none";
+      document.body.appendChild(el);
+      el.play().catch(() => {
+      });
+      this.audioElement = el;
+    }
+    return ctx;
   }
   playNext() {
     const ctx = this.getAudioContext();
@@ -9255,7 +9292,7 @@ var AudioPlayer = class {
     this.isPlaying = true;
     const source = ctx.createBufferSource();
     source.buffer = buffer;
-    source.connect(ctx.destination);
+    source.connect(this.mediaStreamDest ?? ctx.destination);
     this.currentSource = source;
     source.onended = () => {
       this.currentSource = null;
@@ -9418,6 +9455,9 @@ var EstuaryClient = class extends TypedEventEmitter {
     this.ensureConnected();
     this.socketManager.emitEvent("client_interrupt", { message_id: messageId });
     this.audioPlayer?.clear();
+    if (this.config.suppressMicDuringPlayback) {
+      this.voiceManager?.setSuppressed?.(false);
+    }
   }
   /** Send a camera image for vision processing */
   sendCameraImage(imageBase64, mimeType, requestId, text) {
@@ -9456,9 +9496,15 @@ var EstuaryClient = class extends TypedEventEmitter {
       this.audioPlayer = new AudioPlayer(sampleRate, (event) => {
         if (event.type === "started") {
           this.emit("audioPlaybackStarted", event.messageId);
+          if (this.config.suppressMicDuringPlayback) {
+            this.voiceManager?.setSuppressed?.(true);
+          }
         } else if (event.type === "complete") {
           this.emit("audioPlaybackComplete", event.messageId);
           this.notifyAudioPlaybackComplete(event.messageId);
+          if (this.config.suppressMicDuringPlayback) {
+            this.voiceManager?.setSuppressed?.(false);
+          }
         }
       });
     }
@@ -9513,6 +9559,9 @@ var EstuaryClient = class extends TypedEventEmitter {
     this.socketManager.on("interrupt", (data) => {
       this.audioPlayer?.clear();
       this.actionParsers.clear();
+      if (this.config.suppressMicDuringPlayback) {
+        this.voiceManager?.setSuppressed?.(false);
+      }
       this.emit("interrupt", data);
     });
     this.socketManager.on("error", (error) => this.emit("error", error));