@animated-waffle/client 0.1.2 → 0.1.4

package/dist/index.d.ts

@@ -146,19 +146,22 @@ declare class WaffleClient implements WaffleSessionClient {
     /**
      * The runtime announces one spoken segment three times with the same text: as
      * it is queued for synthesis (`spoken_status: 'new'`), on word-level progress
-     * (`'in-progress'`), and once synthesis completes (`'completed'`).
+     * (`'in-progress'`), and once synthesis finishes (`'completed'`). Only one of
+     * them can be a transcript entry, and it has to be the first:
      *
-     * Only the last one is reported. The other two cannot be paired with it:
-     * `segment_id` is the *frame* id on this path, so the queued and completed
-     * events for one sentence carry different ids, and they are not even
-     * adjacent — a segment queued at the end of one turn can complete after the
-     * next user message. Forwarding all three duplicated every sentence in a
-     * caller's transcript with no key to reconcile them.
+     * - `'new'` fires as the agent commits to saying that sentence, so the text
+     *   lands with the voice and in the order the agent produced it;
+     * - `'completed'` fires after synthesis, so text appears only once a sentence
+     *   has finished being spoken, and a segment queued at the end of a turn
+     *   completes *after* the next user message — attributing the tail of one
+     *   reply to the turn that follows it;
+     * - `'in-progress'` is word-level playback progress, not a sentence.
      *
-     * Reporting only the completed event costs the caption the few hundred
-     * milliseconds of synthesis, and it arrives as the agent starts saying it.
-     * Text the runtime will not speak carries no lifecycle at all and is
-     * reported as it arrives.
+     * `segment_id` cannot reconcile them: on this path it is the frame id, so the
+     * three events for one sentence carry three different ids. Text the runtime
+     * will not speak has no lifecycle at all and is reported as it arrives.
+     *
+     * One entry per spoken sentence, in order, exactly once.
      */
     private handleBotOutput;
     private handleTransportState;

package/dist/index.js

@@ -1 +1 @@
-import{b as h}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,I="https://animated-waffle.narya.ai",C=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,b=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,l){super(t);this.status=n;this.code=l;this.name="WaffleError"}status;code};function m(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function P(s=I){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function A(s){let e=u(s,"avatar.asset_hash");if(!b.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function p(s){return typeof s=="string"&&s.length>0?s:void 0}function O(s,e,t){let n=m(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let l=m(n.connection);if(!l||l.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let f=u(l.url,"connection.url");try{if(new URL(f).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=p(n.avatar_id),a=m(n.avatar),v=null;if(d||a){if(!d||!a||u(a.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(a.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let r=u(a.model_url,"avatar.model_url");try{if(new URL(r).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(a.name,"avatar.name"),engine:o,modelUrl:r,assetHash:A(a.asset_hash),runtimeProfile:u(a.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:f,token:u(l.token,"connection.token")}}function _(s){let e=m(s);if(!e||typeof e.type!="string")return null;let t={contextId:p(e.contextId)??null,segmentId:p(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime),offset:g(e.offset)};if(e.type==="avatar-emotion"){let n=p(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=p(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function S(s){let{avatar:e,room:t,token:n,...l}=s;return l}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=P(e.apiOrigin),!C.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onUserTranscript:t=>this.handleUserTranscript(t),onBotOutput:t=>this.handleBotOutput(t),onServerMessage:t=>{let n=_(t);n&&h(this,"avatar",n)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(m(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}),this.emit("transcript",{role:"user",text:e,final:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let l;try{l=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let r=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let f=await l.json().catch(()=>null);if(!l.ok){let o=m(f),r=new i(p(o?.detail)??"Could not create session.",l.status,p(o?.code));throw this.fail(r),r}let c;try{c=O(f,this.activeInputMode,this.options.agentId)}catch(o){let r=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(r),r}this.currentSession=c,this.emit("session",S(c)),h(this,"presentation",c.avatar),this.setStatus("connecting");let d,a,v=new Promise((o,r)=>{d=o,a=r});this.readyWaiter={reject:a,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let r=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleUserTranscript(e){this.emit("transcript",{role:"user",text:e.text,final:e.final,timestamp:e.timestamp})}handleBotOutput(e){e.spoken_status==="new"||e.spoken_status==="in-progress"||this.emit("transcript",{role:"assistant",text:e.text,final:!0})}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),h(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),h(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),h(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,l){return new Promise((f,c)=>{let d=setTimeout(()=>{l?.(),c(new i(n))},t);e.then(a=>{clearTimeout(d),f(a)},a=>{clearTimeout(d),c(a)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};
+import{b as p}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,I="https://animated-waffle.narya.ai",C=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,b=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,l){super(t);this.status=n;this.code=l;this.name="WaffleError"}status;code};function m(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function P(s=I){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function A(s){let e=u(s,"avatar.asset_hash");if(!b.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function h(s){return typeof s=="string"&&s.length>0?s:void 0}function O(s,e,t){let n=m(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let l=m(n.connection);if(!l||l.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let f=u(l.url,"connection.url");try{if(new URL(f).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=h(n.avatar_id),a=m(n.avatar),v=null;if(d||a){if(!d||!a||u(a.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(a.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let r=u(a.model_url,"avatar.model_url");try{if(new URL(r).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(a.name,"avatar.name"),engine:o,modelUrl:r,assetHash:A(a.asset_hash),runtimeProfile:u(a.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:f,token:u(l.token,"connection.token")}}function _(s){let e=m(s);if(!e||typeof e.type!="string")return null;let t={contextId:h(e.contextId)??null,segmentId:h(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime),offset:g(e.offset)};if(e.type==="avatar-emotion"){let n=h(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=h(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function S(s){let{avatar:e,room:t,token:n,...l}=s;return l}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=P(e.apiOrigin),!C.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onUserTranscript:t=>this.handleUserTranscript(t),onBotOutput:t=>this.handleBotOutput(t),onBotTtsText:t=>{t.text&&p(this,"spokenWord",{text:t.text})},onServerMessage:t=>{let n=_(t);n&&p(this,"avatar",n)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(m(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let l;try{l=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let r=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let f=await l.json().catch(()=>null);if(!l.ok){let o=m(f),r=new i(h(o?.detail)??"Could not create session.",l.status,h(o?.code));throw this.fail(r),r}let c;try{c=O(f,this.activeInputMode,this.options.agentId)}catch(o){let r=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(r),r}this.currentSession=c,this.emit("session",S(c)),p(this,"presentation",c.avatar),this.setStatus("connecting");let d,a,v=new Promise((o,r)=>{d=o,a=r});this.readyWaiter={reject:a,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let r=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleUserTranscript(e){this.emit("transcript",{role:"user",text:e.text,final:e.final,timestamp:e.timestamp})}handleBotOutput(e){e.spoken_status==="in-progress"||e.spoken_status==="completed"||this.emit("transcript",{role:"assistant",text:e.text,final:!0})}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),p(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),p(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),p(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,l){return new Promise((f,c)=>{let d=setTimeout(()=>{l?.(),c(new i(n))},t);e.then(a=>{clearTimeout(d),f(a)},a=>{clearTimeout(d),c(a)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};

package/dist/internal.d.ts

@@ -20,6 +20,14 @@ interface InternalWaffleEvents {
     avatar: InternalAvatarEvent;
     audioTrack: MediaStreamTrack | null;
     presentation: InternalAvatarPresentation | null;
+    /**
+     * One spoken word at the moment it becomes audible. Present only when the
+     * runtime's TTS provider supplies word timestamps; drives client-side
+     * performance choreography.
+     */
+    spokenWord: {
+        text: string;
+    };
 }
 type InternalListener<K extends keyof InternalWaffleEvents> = (payload: InternalWaffleEvents[K]) => void;
 declare function subscribeInternal<K extends keyof InternalWaffleEvents>(owner: object, event: K, listener: InternalListener<K>): () => void;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,