@animated-waffle/client 0.1.3 → 0.1.5

package/README.md

@@ -34,8 +34,11 @@ const waffle = new WaffleClient({
   },
 })
 
-waffle.on('transcript', ({ role, text, final }) => {
-  if (final) console.log(role, text)
+waffle.on('transcript', ({ id, role, text, final }) => {
+  // Rows use the same id as persisted history. Assistant rows stream: the
+  // same id is re-sent with accumulated text while the agent speaks, then
+  // once more with final: true. Render by replacing the row with that id.
+  console.log(id, role, text, final)
 })
 
 await waffle.connect({ inputMode: 'push-to-talk' })
@@ -61,6 +64,10 @@ const waffle = new WaffleClient({
 - `startPushToTalk()` / `commitPushToTalk()` / `cancelPushToTalk()`.
 - `on('status' | 'session' | 'transcript' | 'agentSpeaking' | 'error', fn)` —
   returns the unsubscribe function.
+- Conversation transcripts carry the same required `id` as persisted history.
+  Assistant rows stream in place — accumulated `text` plus tag-preserving
+  `annotatedText`, then `final: true` — so replace by `id`, never append.
+  Provider sentence/progress framing never defines a public conversation row.
 - `status`, `session` getters; failures throw `WaffleError` with `code`.
 
 `./internal` is a private contract consumed by `@animated-waffle/react`; it is

package/dist/{chunk-BXRNROI7.js → chunk-LJ4KJFLQ.js}

@@ -1 +1 @@
-var o=new WeakMap;function s(n,t,a){let e=o.get(n)??new Map,r=e.get(t)??new Set;return r.add(a),e.set(t,r),o.set(n,e),()=>r.delete(a)}function i(n,t,a){o.get(n)?.get(t)?.forEach(e=>{e(a)})}export{s as a,i as b};
+var o=new WeakMap;function s(n,t,a){let e=o.get(n)??new Map,r=e.get(t)??new Set;return r.add(a),e.set(t,r),o.set(n,e),()=>r.delete(a)}function l(n,t,a){o.get(n)?.get(t)?.forEach(e=>{e(a)})}export{s as a,l as b};

package/dist/index.d.ts

@@ -9,12 +9,21 @@ interface WaffleSession {
     inputMode: WaffleInputMode;
 }
 interface WaffleTranscript {
+    /** The persisted conversation message id. */
+    id: string;
     role: 'user' | 'assistant';
+    /** User-facing text with voice-control tags stripped. */
     text: string;
     /**
-     * False while this text is still provisional. Only interim speech recognition
-     * is provisional: a caller replaces the previous non-final entry from that
-     * speaker rather than appending. Agent text is always final.
+     * The same text with inline voice tags (e.g. `[happy]`) preserved, present on
+     * assistant entries. Each update strictly extends the previous one for the
+     * same id, so performance layers can act on the appended suffix alone.
+     */
+    annotatedText?: string;
+    /**
+     * False while the agent is still speaking this entry. The runtime re-sends
+     * the same id with accumulated text as each sentence is spoken, then once
+     * more with final: true when the turn completes. Replace by id, not append.
      */
     final: boolean;
     timestamp?: string;
@@ -142,28 +151,6 @@ declare class WaffleClient implements WaffleSessionClient {
     private stopInternal;
     private sendInputControl;
     private assertConnected;
-    private handleUserTranscript;
-    /**
-     * The runtime announces one spoken segment three times with the same text: as
-     * it is queued for synthesis (`spoken_status: 'new'`), on word-level progress
-     * (`'in-progress'`), and once synthesis finishes (`'completed'`). Only one of
-     * them can be a transcript entry, and it has to be the first:
-     *
-     * - `'new'` fires as the agent commits to saying that sentence, so the text
-     *   lands with the voice and in the order the agent produced it;
-     * - `'completed'` fires after synthesis, so text appears only once a sentence
-     *   has finished being spoken, and a segment queued at the end of a turn
-     *   completes *after* the next user message — attributing the tail of one
-     *   reply to the turn that follows it;
-     * - `'in-progress'` is word-level playback progress, not a sentence.
-     *
-     * `segment_id` cannot reconcile them: on this path it is the frame id, so the
-     * three events for one sentence carry three different ids. Text the runtime
-     * will not speak has no lifecycle at all and is reported as it arrives.
-     *
-     * One entry per spoken sentence, in order, exactly once.
-     */
-    private handleBotOutput;
     private handleTransportState;
     private handleReady;
     private handleDisconnected;

package/dist/index.js

@@ -1 +1 @@
-import{b as h}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,I="https://animated-waffle.narya.ai",C=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,b=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,l){super(t);this.status=n;this.code=l;this.name="WaffleError"}status;code};function m(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function P(s=I){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function A(s){let e=u(s,"avatar.asset_hash");if(!b.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function p(s){return typeof s=="string"&&s.length>0?s:void 0}function O(s,e,t){let n=m(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let l=m(n.connection);if(!l||l.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let f=u(l.url,"connection.url");try{if(new URL(f).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=p(n.avatar_id),a=m(n.avatar),v=null;if(d||a){if(!d||!a||u(a.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(a.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let r=u(a.model_url,"avatar.model_url");try{if(new URL(r).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(a.name,"avatar.name"),engine:o,modelUrl:r,assetHash:A(a.asset_hash),runtimeProfile:u(a.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:f,token:u(l.token,"connection.token")}}function _(s){let e=m(s);if(!e||typeof e.type!="string")return null;let t={contextId:p(e.contextId)??null,segmentId:p(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime),offset:g(e.offset)};if(e.type==="avatar-emotion"){let n=p(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=p(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function S(s){let{avatar:e,room:t,token:n,...l}=s;return l}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=P(e.apiOrigin),!C.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onUserTranscript:t=>this.handleUserTranscript(t),onBotOutput:t=>this.handleBotOutput(t),onServerMessage:t=>{let n=_(t);n&&h(this,"avatar",n)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(m(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let l;try{l=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let r=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let f=await l.json().catch(()=>null);if(!l.ok){let o=m(f),r=new i(p(o?.detail)??"Could not create session.",l.status,p(o?.code));throw this.fail(r),r}let c;try{c=O(f,this.activeInputMode,this.options.agentId)}catch(o){let r=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(r),r}this.currentSession=c,this.emit("session",S(c)),h(this,"presentation",c.avatar),this.setStatus("connecting");let d,a,v=new Promise((o,r)=>{d=o,a=r});this.readyWaiter={reject:a,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let r=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleUserTranscript(e){this.emit("transcript",{role:"user",text:e.text,final:e.final,timestamp:e.timestamp})}handleBotOutput(e){e.spoken_status==="in-progress"||e.spoken_status==="completed"||this.emit("transcript",{role:"assistant",text:e.text,final:!0})}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),h(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),h(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),h(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,l){return new Promise((f,c)=>{let d=setTimeout(()=>{l?.(),c(new i(n))},t);e.then(a=>{clearTimeout(d),f(a)},a=>{clearTimeout(d),c(a)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};
+import{b as p}from"./chunk-LJ4KJFLQ.js";import{PipecatClient as T}from"@pipecat-ai/client-js";import{DailyTransport as y}from"@pipecat-ai/daily-transport";var w=15e3,k=3e4,W="https://animated-waffle.narya.ai",I=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,C=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,o){super(t);this.status=n;this.code=o;this.name="WaffleError"}status;code};function h(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function P(s=W){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function f(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function b(s){let e=f(s,"avatar.asset_hash");if(!C.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function m(s){return typeof s=="string"&&s.length>0?s:void 0}function A(s,e,t){let n=h(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let o=h(n.connection);if(!o||o.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let u=f(o.url,"connection.url");try{if(new URL(u).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=f(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=m(n.avatar_id),l=h(n.avatar),v=null;if(d||l){if(!d||!l||f(l.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let r=f(l.engine,"avatar.engine");if(r!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let a=f(l.model_url,"avatar.model_url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:f(l.name,"avatar.name"),engine:r,modelUrl:a,assetHash:b(l.asset_hash),runtimeProfile:f(l.runtime_profile,"avatar.runtime_profile")}}return{id:f(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:u,token:f(o.token,"connection.token")}}function _(s){let e=h(s),t=m(e?.tag),n=e?.kind;return e?.type!=="avatar-performance"||!t||n!=="emotion"&&n!=="vocal-action"?null:{phase:n,tag:t}}function O(s){let e=h(s),t=m(e?.messageId),n=e?.role;if(e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||typeof e.final!="boolean")return null;let o={id:t,role:n,text:e.text,final:e.final},u=m(e.annotatedText);return u!==void 0&&(o.annotatedText=u),o}function S(s){let{avatar:e,room:t,token:n,...o}=s;return o}var g=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=P(e.apiOrigin),!I.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onBotTtsText:t=>{t.text&&p(this,"spokenWord",{text:t.text})},onServerMessage:t=>{let n=O(t);n&&this.emit("transcript",n);let o=_(t);o&&p(this,"avatarPerformance",o)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(h(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new y({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new T({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let o;try{o=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(r=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${r}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),w,"Timed out creating the Waffle session.",()=>n.abort())}catch(r){let a=r instanceof Error?r:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let u=await o.json().catch(()=>null);if(!o.ok){let r=h(u),a=new i(m(r?.detail)??"Could not create session.",o.status,m(r?.code));throw this.fail(a),a}let c;try{c=A(u,this.activeInputMode,this.options.agentId)}catch(r){let a=r instanceof Error?r:new i("Could not parse the Waffle session.");throw this.fail(a),a}this.currentSession=c,this.emit("session",S(c)),p(this,"presentation",c.avatar),this.setStatus("connecting");let d,l,v=new Promise((r,a)=>{d=r,l=a});this.readyWaiter={reject:l,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),k,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(r){await this.realtime.disconnect().catch(()=>{});let a=r instanceof Error?r:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),p(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),p(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),p(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,o){return new Promise((u,c)=>{let d=setTimeout(()=>{o?.(),c(new i(n))},t);e.then(l=>{clearTimeout(d),u(l)},l=>{clearTimeout(d),c(l)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{g as WaffleClient,i as WaffleError};

package/dist/internal.d.ts

@@ -1,12 +1,6 @@
-interface InternalAvatarEvent {
+interface InternalAvatarPerformance {
     phase: 'emotion' | 'vocal-action';
-    contextId?: string | null;
-    emotion?: string;
-    action?: string;
-    segmentId?: string;
-    occurrence?: number;
-    audioTime?: number;
-    offset?: number;
+    tag: string;
 }
 interface InternalAvatarPresentation {
     id: string;
@@ -17,12 +11,25 @@ interface InternalAvatarPresentation {
     runtimeProfile: string;
 }
 interface InternalWaffleEvents {
-    avatar: InternalAvatarEvent;
+    /**
+     * One performance cue, delivered by the runtime at the exact playout moment
+     * it should be performed (the server schedules it on the audio clock, with
+     * the emotion pre-entry lead already applied).
+     */
+    avatarPerformance: InternalAvatarPerformance;
     audioTrack: MediaStreamTrack | null;
     presentation: InternalAvatarPresentation | null;
+    /**
+     * One spoken word at the moment it becomes audible. Present only when the
+     * runtime's TTS provider supplies word timestamps; drives client-side
+     * performance choreography.
+     */
+    spokenWord: {
+        text: string;
+    };
 }
 type InternalListener<K extends keyof InternalWaffleEvents> = (payload: InternalWaffleEvents[K]) => void;
 declare function subscribeInternal<K extends keyof InternalWaffleEvents>(owner: object, event: K, listener: InternalListener<K>): () => void;
 declare function emitInternal<K extends keyof InternalWaffleEvents>(owner: object, event: K, payload: InternalWaffleEvents[K]): void;
 
-export { type InternalAvatarEvent, type InternalAvatarPresentation, type InternalWaffleEvents, emitInternal, subscribeInternal };
+export { type InternalAvatarPerformance, type InternalAvatarPresentation, type InternalWaffleEvents, emitInternal, subscribeInternal };

package/dist/internal.js

@@ -1 +1 @@
-import{a,b}from"./chunk-BXRNROI7.js";export{b as emitInternal,a as subscribeInternal};
+import{a,b}from"./chunk-LJ4KJFLQ.js";export{b as emitInternal,a as subscribeInternal};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,