@animated-waffle/client 0.2.4 → 0.2.6

package/README.md

@@ -64,6 +64,12 @@ const waffle = new WaffleClient({
   text input, Agent audio, transcripts, and reconnects continue to work. Push-
   to-talk controls fail closed with `input_audio_disabled`. The default is
   `true`, enabled only after the Agent is ready.
+- `muted` — silences the Agent's voice while it keeps performing: the session,
+  the transcript and the Avatar's cues carry on, only playback goes quiet.
+- `audioPlayback: false` — the client builds no audio element of its own. Play
+  `remoteAudioTrack` yourself; an application that routes the voice through its
+  own element, mixer or Web Audio graph must turn this off, or the voice is
+  heard twice. `muted` then has nothing to do.
 - `disconnect()`, `sendText(text)`.
 - `startPushToTalk()` / `commitPushToTalk()` / `cancelPushToTalk()`.
 - `on('status' | 'session' | 'transcript' | 'agentSpeaking' | 'error', fn)` —

package/dist/index.d.ts

@@ -57,6 +57,13 @@ interface WaffleClientBaseOptions {
     agentId: string;
     /** Disable browser microphone access for text-triggered, output-only sessions. */
     inputAudioEnabled?: boolean;
+    /**
+     * Play the Agent's voice through an audio element the client owns. Default
+     * true. Pass false to play `remoteAudioTrack` yourself — an application that
+     * routes the voice through its own element, mixer or Web Audio graph must
+     * turn this off, or the voice is heard twice.
+     */
+    audioPlayback?: boolean;
 }
 interface WaffleTokenClientOptions extends WaffleClientBaseOptions {
     /** A short-lived token minted for this Agent by the Narya API. */
@@ -72,6 +79,18 @@ interface WaffleManagedClientOptions extends WaffleClientBaseOptions {
     apiOrigin?: never;
 }
 type WaffleClientOptions = WaffleTokenClientOptions | WaffleManagedClientOptions;
+interface WaffleSendTextOptions {
+    /**
+     * Whether the Agent speaks its reply.
+     *
+     * Defaults to true, which is the only behavior this method had. Passing
+     * false makes one turn text-only: the reply still streams as transcript and
+     * still lands in the conversation context, but no audio is synthesized —
+     * what a typed exchange inside an otherwise spoken session wants, and what a
+     * message thread wants for every turn.
+     */
+    readonly audioResponse?: boolean;
+}
 interface WaffleSessionStartOptions {
     inputMode?: WaffleInputMode;
     /** Existing application session to associate with this realtime run. */
@@ -94,10 +113,17 @@ interface WaffleSessionClient {
     readonly status: WaffleStatus;
     readonly session: WaffleSession | null;
     readonly remoteAudioTrack: MediaStreamTrack | null;
+    /**
+     * Silence the Agent's voice while it keeps performing: the session, the
+     * transcript and the Avatar's cues carry on, only playback goes quiet. Has
+     * no effect when `audioPlayback` is false, where the application owns the
+     * voice and mutes it its own way.
+     */
+    muted: boolean;
     on<K extends keyof WaffleClientEvents>(event: K, listener: (payload: WaffleClientEvents[K]) => void): () => void;
     connect(options?: WaffleSessionStartOptions): Promise<WaffleSession>;
     disconnect(): Promise<void>;
-    sendText(text: string): Promise<void>;
+    sendText(text: string, options?: WaffleSendTextOptions): Promise<void>;
     startPushToTalk(): void;
     commitPushToTalk(): void;
     cancelPushToTalk(): void;
@@ -121,6 +147,7 @@ declare class WaffleClient implements WaffleSessionClient {
     private requestAbort;
     private readyWaiter;
     private audio;
+    private audioMuted;
     private currentRemoteAudioTrack;
     private activeInputMode;
     private turnId;
@@ -133,10 +160,12 @@ declare class WaffleClient implements WaffleSessionClient {
     get connectionState(): WaffleConnectionState;
     get session(): WaffleSession | null;
     get remoteAudioTrack(): MediaStreamTrack | null;
+    get muted(): boolean;
+    set muted(muted: boolean);
     on<K extends keyof WaffleClientEvents>(event: K, listener: Listener<K>): () => void;
     connect(options?: WaffleConnectOptions): Promise<WaffleSession>;
     disconnect(): Promise<void>;
-    sendText(text: string): Promise<void>;
+    sendText(text: string, options?: WaffleSendTextOptions): Promise<void>;
     startPushToTalk(): void;
     commitPushToTalk(): void;
     /** @deprecated Use commitPushToTalk(). */
@@ -159,4 +188,4 @@ declare class WaffleClient implements WaffleSessionClient {
     private emit;
 }
 
-export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript };
+export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSendTextOptions, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript };

package/dist/index.js

@@ -1 +1 @@
-import{b as m}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,I="https://animated-waffle.narya.ai",b=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,A=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,r){super(t);this.status=n;this.code=r;this.name="WaffleError"}status;code};function p(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function C(s=I){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function P(s){let e=u(s,"avatar.asset_hash");if(!A.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function f(s){return typeof s=="string"&&s.length>0?s:void 0}function _(s,e,t){let n=p(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let r=p(n.connection);if(!r||r.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let h=u(r.url,"connection.url");try{if(new URL(h).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=f(n.avatar_id),l=p(n.avatar),v=null;if(d||l){if(!d||!l||u(l.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(l.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let a=u(l.model_url,"avatar.model_url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(l.name,"avatar.name"),engine:o,modelUrl:a,assetHash:P(l.asset_hash),runtimeProfile:u(l.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:h,token:u(r.token,"connection.token")}}function E(s){let e=p(s);if(!e||typeof e.type!="string")return null;let t={contextId:f(e.contextId)??null,segmentId:f(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime)};if(e.type==="avatar-emotion"){let n=f(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=f(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function O(s){let e=p(s),t=f(e?.messageId),n=e?.role;return e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||e.final!==!0?null:{id:t,role:n,text:e.text,final:!0}}function S(s){let{avatar:e,room:t,token:n,...r}=s;return r}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=C(e.apiOrigin),!b.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onServerMessage:t=>{let n=O(t);n&&this.emit("transcript",n);let r=E(t);r&&m(this,"avatar",r)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(p(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:this.options.inputAudioEnabled===!1?"none":"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}))}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let r;try{r=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let a=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let h=await r.json().catch(()=>null);if(!r.ok){let o=p(h),a=new i(f(o?.detail)??"Could not create session.",r.status,f(o?.code));throw this.fail(a),a}let c;try{c=_(h,this.activeInputMode,this.options.agentId)}catch(o){let a=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(a),a}this.currentSession=c,this.emit("session",S(c)),m(this,"presentation",c.avatar),this.setStatus("connecting");let d,l,v=new Promise((o,a)=>{d=o,l=a});this.readyWaiter={reject:l,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let a=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.options.inputAudioEnabled!==!1&&this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),m(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),m(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),m(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,r){return new Promise((h,c)=>{let d=setTimeout(()=>{r?.(),c(new i(n))},t);e.then(l=>{clearTimeout(d),h(l)},l=>{clearTimeout(d),c(l)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};
+import{b as m}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,b="https://animated-waffle.narya.ai",I=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,P=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,r){super(t);this.status=n;this.code=r;this.name="WaffleError"}status;code};function h(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function A(s=b){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function C(s){let e=u(s,"avatar.asset_hash");if(!P.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function f(s){return typeof s=="string"&&s.length>0?s:void 0}function _(s,e,t){let n=h(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let r=h(n.connection);if(!r||r.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let p=u(r.url,"connection.url");try{if(new URL(p).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=f(n.avatar_id),l=h(n.avatar),v=null;if(d||l){if(!d||!l||u(l.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(l.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let a=u(l.model_url,"avatar.model_url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(l.name,"avatar.name"),engine:o,modelUrl:a,assetHash:C(l.asset_hash),runtimeProfile:u(l.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:p,token:u(r.token,"connection.token")}}function O(s){let e=h(s);if(!e||typeof e.type!="string")return null;let t={contextId:f(e.contextId)??null,segmentId:f(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime)};if(e.type==="avatar-emotion"){let n=f(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=f(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function E(s){let e=h(s),t=f(e?.messageId),n=e?.role;return e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||e.final!==!0?null:{id:t,role:n,text:e.text,final:!0}}function S(s){let{avatar:e,room:t,token:n,...r}=s;return r}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=A(e.apiOrigin),!I.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onServerMessage:t=>{let n=E(t);n&&this.emit("transcript",n);let r=O(t);r&&m(this,"avatar",r)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(h(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:this.options.inputAudioEnabled===!1?"none":"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(e){this.audioMuted=e,this.audio&&(this.audio.muted=e)}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e,t){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:t?.audioResponse??!0,run_immediately:!0}))}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let r;try{r=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let a=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let p=await r.json().catch(()=>null);if(!r.ok){let o=h(p),a=new i(f(o?.detail)??"Could not create session.",r.status,f(o?.code));throw this.fail(a),a}let c;try{c=_(p,this.activeInputMode,this.options.agentId)}catch(o){let a=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(a),a}this.currentSession=c,this.emit("session",S(c)),m(this,"presentation",c.avatar),this.setStatus("connecting");let d,l,v=new Promise((o,a)=>{d=o,l=a});this.readyWaiter={reject:l,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let a=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.options.inputAudioEnabled!==!1&&this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),m(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u"||this.options.audioPlayback===!1)return;let t=document.createElement("audio");t.autoplay=!0,t.muted=this.audioMuted,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),m(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),m(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,r){return new Promise((p,c)=>{let d=setTimeout(()=>{r?.(),c(new i(n))},t);e.then(l=>{clearTimeout(d),p(l)},l=>{clearTimeout(d),c(l)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,