@animated-waffle/client 0.1.6 → 0.2.0

package/README.md

@@ -39,6 +39,18 @@ waffle.on('transcript', ({ id, role, text, final }) => {
   if (final) console.log(id, role, text)
 })
 
+waffle.on('speechSegment', ({ id, messageId, sequence, progressId, text, annotatedText }) => {
+  // Published only at this segment's queued audio start—not when text merely
+  // enters TTS or a transport-wide speaking callback fires.
+  console.log(id, messageId, sequence, progressId, text, annotatedText)
+})
+
+waffle.on('speechProgress', ({ progressId, status, accumulatedText, remainingText }) => {
+  // Factual progress associated with the segment above. The browser owns any
+  // semantic interpretation of annotatedText.
+  console.log(progressId, status, accumulatedText, remainingText)
+})
+
 await waffle.connect({ inputMode: 'push-to-talk' })
 ```
 
@@ -55,15 +67,25 @@ const waffle = new WaffleClient({
 })
 ```
 
+At an input interruption the client immediately resets and temporarily blocks
+speech facts. The runtime orders `conversation-speech-playout-cutoff` after all
+old source identities and begins cancelling the old media queue; the client
+then clears again and reopens the fact stream. An already-in-flight old marker
+may arrive later, but it is an orphan and cannot revive cancelled performance.
+
 ## API surface
 
 - `connect({ inputMode? })` — `'push-to-talk'` (default) or `'automatic'`.
 - `disconnect()`, `sendText(text)`.
 - `startPushToTalk()` / `commitPushToTalk()` / `cancelPushToTalk()`.
-- `on('status' | 'session' | 'transcript' | 'agentSpeaking' | 'error', fn)` —
-  returns the unsubscribe function.
+- `on('status' | 'session' | 'transcript' | 'speechSegment' | 'speechProgress' |
+  'agentSpeaking' | 'error', fn)` — returns the unsubscribe function.
 - Conversation transcripts carry the same required `id` as persisted history.
   Provider sentence/progress framing never defines a public conversation row.
+- Speech segments become public at their associated audio start. The SDK uses
+  raw `new` output only to correlate identity, never as a timing signal. Their
+  `annotatedText` drives local Avatar choreography while `text` remains clean
+  spoken text; `speechProgress` carries provider-paced accumulated facts.
 - `status`, `session` getters; failures throw `WaffleError` with `code`.
 
 `./internal` is a private contract consumed by `@animated-waffle/react`; it is

package/dist/chunk-M7Z7AFVA.js

@@ -0,0 +1 @@
+var s=new WeakMap;function o(n,t,a){let e=s.get(n)??new Map,r=e.get(t)??new Set;return r.add(a),e.set(t,r),s.set(n,e),()=>r.delete(a)}function l(n,t,a){s.get(n)?.get(t)?.forEach(e=>{e(a)})}export{o as a,l as b};

package/dist/index.d.ts

@@ -21,6 +21,35 @@ interface WaffleTranscript {
     final: boolean;
     timestamp?: string;
 }
+/**
+ * One Agent speech segment, published at its associated audio start. `text` is
+ * the clean spoken text while `annotatedText` retains inline performance tags
+ * for the browser-side Avatar choreographer. It is deliberately separate from
+ * the persisted conversation transcript.
+ */
+interface WaffleSpeechSegment {
+    /** Stable speech-segment id. */
+    id: string;
+    /** Persisted assistant message this segment belongs to. */
+    messageId: string;
+    /** Zero-based order within the assistant message. */
+    sequence: number;
+    /** Numeric identity shared with Pipecat's factual playout progress. */
+    progressId: number;
+    text: string;
+    annotatedText: string;
+}
+type WaffleSpeechProgressStatus = 'new' | 'in-progress' | 'completed';
+/** Factual playout progress associated with one public speech segment. */
+interface WaffleSpeechProgress {
+    segmentId: string;
+    messageId: string;
+    sequence: number;
+    progressId: number;
+    status: WaffleSpeechProgressStatus;
+    accumulatedText: string;
+    remainingText: string;
+}
 interface WaffleClientEvents {
     connectionState: WaffleConnectionState;
     /** @deprecated Subscribe to connectionState. */
@@ -30,6 +59,10 @@ interface WaffleClientEvents {
     agentSpeaking: boolean;
     remoteAudioTrack: MediaStreamTrack | null;
     transcript: WaffleTranscript;
+    speechSegment: WaffleSpeechSegment;
+    speechProgress: WaffleSpeechProgress;
+    /** A user input boundary cancelled any pending Agent speech/performance. */
+    interrupted: undefined;
 }
 type WaffleSessionToken = () => string | Promise<string>;
 interface WaffleSessionOverrides {
@@ -125,6 +158,14 @@ declare class WaffleClient implements WaffleSessionClient {
     private nextTurnId;
     private lifecycleVersion;
     private recreateRealtimeOnConnect;
+    private readonly speechSegmentsByProgressId;
+    private readonly pendingSpeechProgress;
+    private readonly seenSpeechProgress;
+    private readonly releasedSpeechSegments;
+    private readonly pendingSpeechPlayoutStarts;
+    private botAudioActive;
+    private speechFactsBlocked;
+    private speechPlayoutCutoffVersion;
     constructor(options: WaffleClientOptions);
     private createRealtime;
     get status(): WaffleStatus;
@@ -147,6 +188,16 @@ declare class WaffleClient implements WaffleSessionClient {
     private handleTransportState;
     private handleReady;
     private handleDisconnected;
+    private handleSpeechSegment;
+    private handleSpeechPlayoutStart;
+    private handleBotOutput;
+    private handleBotStartedSpeaking;
+    private handleBotStoppedSpeaking;
+    private releaseSpeechSegment;
+    private emitSpeechProgress;
+    private resetSpeechProgress;
+    private interruptSpeech;
+    private handleSpeechPlayoutCutoff;
     private setAudioTrack;
     private stopAudio;
     private setStatus;
@@ -156,4 +207,4 @@ declare class WaffleClient implements WaffleSessionClient {
     private emit;
 }
 
-export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript };
+export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleSpeechProgress, type WaffleSpeechProgressStatus, type WaffleSpeechSegment, type WaffleStatus, type WaffleTranscript };

package/dist/index.js

@@ -1 +1 @@
-import{b as p}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,I="https://animated-waffle.narya.ai",C=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,b=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,r){super(t);this.status=n;this.code=r;this.name="WaffleError"}status;code};function m(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function P(s=I){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function A(s){let e=u(s,"avatar.asset_hash");if(!b.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function f(s){return typeof s=="string"&&s.length>0?s:void 0}function _(s,e,t){let n=m(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let r=m(n.connection);if(!r||r.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let h=u(r.url,"connection.url");try{if(new URL(h).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=f(n.avatar_id),l=m(n.avatar),v=null;if(d||l){if(!d||!l||u(l.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(l.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let a=u(l.model_url,"avatar.model_url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(l.name,"avatar.name"),engine:o,modelUrl:a,assetHash:A(l.asset_hash),runtimeProfile:u(l.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:h,token:u(r.token,"connection.token")}}function O(s){let e=m(s);if(!e||typeof e.type!="string")return null;let t={contextId:f(e.contextId)??null,segmentId:f(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime),offset:g(e.offset)};if(e.type==="avatar-emotion"){let n=f(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=f(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function E(s){let e=m(s),t=f(e?.messageId),n=e?.role;return e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||e.final!==!0?null:{id:t,role:n,text:e.text,final:!0}}function S(s){let{avatar:e,room:t,token:n,...r}=s;return r}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=P(e.apiOrigin),!C.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onBotTtsText:t=>{t.text&&p(this,"spokenWord",{text:t.text})},onServerMessage:t=>{let n=E(t);n&&this.emit("transcript",n);let r=O(t);r&&p(this,"avatar",r)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(m(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let r;try{r=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let a=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let h=await r.json().catch(()=>null);if(!r.ok){let o=m(h),a=new i(f(o?.detail)??"Could not create session.",r.status,f(o?.code));throw this.fail(a),a}let c;try{c=_(h,this.activeInputMode,this.options.agentId)}catch(o){let a=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(a),a}this.currentSession=c,this.emit("session",S(c)),p(this,"presentation",c.avatar),this.setStatus("connecting");let d,l,v=new Promise((o,a)=>{d=o,l=a});this.readyWaiter={reject:l,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let a=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),p(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),p(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),p(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,r){return new Promise((h,c)=>{let d=setTimeout(()=>{r?.(),c(new i(n))},t);e.then(l=>{clearTimeout(d),h(l)},l=>{clearTimeout(d),c(l)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};
+import{b as g}from"./chunk-M7Z7AFVA.js";import{PipecatClient as k}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var T=15e3,P=3e4,I="https://animated-waffle.narya.ai",W=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,b=/^sha256:[a-f0-9]{64}$/,r=class extends Error{constructor(t,n,i){super(t);this.status=n;this.code=i;this.name="WaffleError"}status;code};function h(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function C(s=I){let e;try{e=new URL(s)}catch{throw new r("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new r("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function p(s,e){if(typeof s!="string"||s.trim().length===0)throw new r(`Session response is missing ${e}.`);return s}function A(s){let e=p(s,"avatar.asset_hash");if(!b.test(e))throw new r("Session response contains an invalid Avatar asset hash.");return e}function v(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function f(s){return typeof s=="string"&&s.length>0?s:void 0}function x(s,e,t){let n=h(s);if(!n)throw new r("Session response must be a JSON object.");if(n.protocol_version!==1)throw new r("Session response has an unsupported protocol version.");let i=h(n.connection);if(!i||i.transport!=="webrtc")throw new r("Session response is missing a WebRTC connection.");let a=p(i.url,"connection.url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new r("Session response contains an invalid connection URL.")}let o=p(n.agent_id,"agent_id");if(o!==t)throw new r("Session response contains a mismatched Agent id.");let u=f(n.avatar_id),d=h(n.avatar),m=null;if(u||d){if(!u||!d||p(d.id,"avatar.id")!==u)throw new r("Session response contains mismatched Avatar metadata.");let l=p(d.engine,"avatar.engine");if(l!=="vrm")throw new r("Session response contains an unsupported Avatar engine.");let c=p(d.model_url,"avatar.model_url");try{if(new URL(c).protocol!=="https:")throw new Error}catch{throw new r("Session response contains an invalid Avatar model URL.")}m={id:u,name:p(d.name,"avatar.name"),engine:l,modelUrl:c,assetHash:A(d.asset_hash),runtimeProfile:p(d.runtime_profile,"avatar.runtime_profile")}}return{id:p(n.session_id,"session_id"),agentId:o,...u?{avatarId:u}:{},avatar:m,inputMode:e,room:a,token:p(i.token,"connection.token")}}function _(s){let e=h(s),t=f(e?.messageId),n=e?.role;return e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||e.final!==!0?null:{id:t,role:n,text:e.text,final:!0}}function O(s){let e=h(s),t=f(e?.segmentId),n=f(e?.messageId),i=v(e?.sequence),a=v(e?.progressId);return e?.type!=="conversation-speech-segment"||!t||!n||i===void 0||!Number.isInteger(i)||i<0||a===void 0||!Number.isInteger(a)||a<0||typeof e.text!="string"||typeof e.annotatedText!="string"?null:{id:t,messageId:n,sequence:i,progressId:a,text:e.text,annotatedText:e.annotatedText}}function R(s){let e=h(s),t=v(e?.progressId);return e?.type!=="conversation-speech-playout-start"||t===void 0||!Number.isInteger(t)||t<0?null:t}function M(s){return h(s)?.type==="conversation-speech-playout-cutoff"}function B(s){let e=h(s),t=h(e?.spoken_progress),n=v(e?.segment_id),i=e?.spoken_status;return n===void 0||!Number.isInteger(n)||n<0||i!=="new"&&i!=="in-progress"&&i!=="completed"||typeof t?.accumulated_text!="string"||typeof t.remaining_text!="string"?null:{progressId:n,status:i,accumulatedText:t.accumulated_text,remainingText:t.remaining_text}}function E(s){return JSON.stringify([s.status,s.accumulatedText,s.remainingText])}function S(s){let{avatar:e,room:t,token:n,...i}=s;return i}var y=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;speechSegmentsByProgressId=new Map;pendingSpeechProgress=new Map;seenSpeechProgress=new Map;releasedSpeechSegments=new Set;pendingSpeechPlayoutStarts=new Set;botAudioActive=!1;speechFactsBlocked=!1;speechPlayoutCutoffVersion=0;constructor(e){if(this.options=e,this.apiOrigin=C(e.apiOrigin),!W.test(e.agentId))throw new r("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new r("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.handleBotStartedSpeaking(),onBotStoppedSpeaking:()=>this.handleBotStoppedSpeaking(),onUserStartedSpeaking:()=>{this.interruptSpeech()},onBotOutput:t=>this.handleBotOutput(t),onServerMessage:t=>{if(M(t)){this.handleSpeechPlayoutCutoff();return}let n=_(t);n&&this.emit("transcript",n);let i=O(t);i&&this.handleSpeechSegment(i);let a=R(t);a!==null&&this.handleSpeechPlayoutStart(a)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new r(h(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new k({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){if(!e.trim())return;this.assertConnected();let t=this.speechFactsBlocked,n=this.speechPlayoutCutoffVersion;this.speechFactsBlocked=!0;try{await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0})}catch(i){throw this.speechFactsBlocked=t,i}this.interruptSpeech(this.speechPlayoutCutoffVersion===n)}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId),this.interruptSpeech()}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.resetSpeechProgress(),this.speechFactsBlocked=!1,this.speechPlayoutCutoffVersion=0,this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let i;try{i=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(l=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${l}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),T,"Timed out creating the Waffle session.",()=>n.abort())}catch(l){let c=l instanceof Error?l:new r("Could not create session.");throw t!==this.lifecycleVersion?new r("The connection attempt was cancelled."):(this.fail(c),c)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new r("The connection attempt was cancelled.");let a=await i.json().catch(()=>null);if(!i.ok){let l=h(a),c=new r(f(l?.detail)??"Could not create session.",i.status,f(l?.code));throw this.fail(c),c}let o;try{o=x(a,this.activeInputMode,this.options.agentId)}catch(l){let c=l instanceof Error?l:new r("Could not parse the Waffle session.");throw this.fail(c),c}this.currentSession=o,this.emit("session",S(o)),g(this,"presentation",o.avatar),this.setStatus("connecting");let u,d,m=new Promise((l,c)=>{u=l,d=c});this.readyWaiter={reject:d,resolve:u,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:o.room,token:o.token}),m]).then(()=>{}),P,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(l){await this.realtime.disconnect().catch(()=>{});let c=l instanceof Error?l:new r("Could not connect.");throw t!==this.lifecycleVersion?new r("The connection attempt was cancelled."):(this.fail(c),c)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(o)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new r("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new r('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new r("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new r("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.resetSpeechProgress(),this.speechFactsBlocked=!1,this.emit("agentSpeaking",!1),this.setStatus("disconnected")}handleSpeechSegment(e){this.speechFactsBlocked||this.speechSegmentsByProgressId.has(e.progressId)||(this.speechSegmentsByProgressId.set(e.progressId,e),this.pendingSpeechPlayoutStarts.has(e.progressId)&&this.releaseSpeechSegment(e.progressId,!0))}handleSpeechPlayoutStart(e){this.speechFactsBlocked||this.releasedSpeechSegments.has(e)||(this.pendingSpeechPlayoutStarts.add(e),this.releaseSpeechSegment(e,!0))}handleBotOutput(e){if(this.speechFactsBlocked)return;let t=B(e);if(!t)return;let n=E(t),i=this.seenSpeechProgress.get(t.progressId)??new Set;if(i.has(n))return;i.add(n),this.seenSpeechProgress.set(t.progressId,i);let a=this.speechSegmentsByProgressId.get(t.progressId);if(a&&this.releasedSpeechSegments.has(t.progressId)){this.emitSpeechProgress(a,t);return}let o=this.pendingSpeechProgress.get(t.progressId)??[];o.push(t),this.pendingSpeechProgress.set(t.progressId,o),this.pendingSpeechPlayoutStarts.has(t.progressId)&&this.releaseSpeechSegment(t.progressId,!0)}handleBotStartedSpeaking(){this.speechFactsBlocked||(this.botAudioActive=!0,this.emit("agentSpeaking",!0))}handleBotStoppedSpeaking(){this.speechFactsBlocked||(this.botAudioActive=!1,this.emit("agentSpeaking",!1))}releaseSpeechSegment(e,t=!1){if(this.releasedSpeechSegments.has(e))return!1;let n=this.speechSegmentsByProgressId.get(e),i=this.pendingSpeechProgress.get(e)??[];if(!n||!i.some(o=>o.status==="new"))return!1;t&&!this.botAudioActive&&(this.botAudioActive=!0,this.emit("agentSpeaking",!0)),this.releasedSpeechSegments.add(e),this.pendingSpeechPlayoutStarts.delete(e),this.emit("speechSegment",n),this.pendingSpeechProgress.delete(e);let a=i.find(o=>o.status==="new");a&&this.emitSpeechProgress(n,a);for(let o of i)o!==a&&this.emitSpeechProgress(n,o);return!0}emitSpeechProgress(e,t){this.emit("speechProgress",{segmentId:e.id,messageId:e.messageId,sequence:e.sequence,progressId:t.progressId,status:t.status,accumulatedText:t.accumulatedText,remainingText:t.remainingText})}resetSpeechProgress(){this.speechSegmentsByProgressId.clear(),this.pendingSpeechProgress.clear(),this.seenSpeechProgress.clear(),this.releasedSpeechSegments.clear(),this.pendingSpeechPlayoutStarts.clear(),this.botAudioActive=!1}interruptSpeech(e=!0){this.resetSpeechProgress(),this.speechFactsBlocked=e,this.emit("interrupted",void 0)}handleSpeechPlayoutCutoff(){let e=this.speechFactsBlocked;this.resetSpeechProgress(),this.speechFactsBlocked=!1,this.speechPlayoutCutoffVersion+=1,e||this.emit("interrupted",void 0)}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),g(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new r("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),g(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0,this.resetSpeechProgress(),this.speechFactsBlocked=!1),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),g(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new r("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new r("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,i){return new Promise((a,o)=>{let u=setTimeout(()=>{i?.(),o(new r(n))},t);e.then(d=>{clearTimeout(u),a(d)},d=>{clearTimeout(u),o(d)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{y as WaffleClient,r as WaffleError};

package/dist/internal.d.ts

@@ -1,13 +1,3 @@
-interface InternalAvatarEvent {
-    phase: 'emotion' | 'vocal-action';
-    contextId?: string | null;
-    emotion?: string;
-    action?: string;
-    segmentId?: string;
-    occurrence?: number;
-    audioTime?: number;
-    offset?: number;
-}
 interface InternalAvatarPresentation {
     id: string;
     name: string;
@@ -17,20 +7,11 @@ interface InternalAvatarPresentation {
     runtimeProfile: string;
 }
 interface InternalWaffleEvents {
-    avatar: InternalAvatarEvent;
     audioTrack: MediaStreamTrack | null;
     presentation: InternalAvatarPresentation | null;
-    /**
-     * One spoken word at the moment it becomes audible. Present only when the
-     * runtime's TTS provider supplies word timestamps; drives client-side
-     * performance choreography.
-     */
-    spokenWord: {
-        text: string;
-    };
 }
 type InternalListener<K extends keyof InternalWaffleEvents> = (payload: InternalWaffleEvents[K]) => void;
 declare function subscribeInternal<K extends keyof InternalWaffleEvents>(owner: object, event: K, listener: InternalListener<K>): () => void;
 declare function emitInternal<K extends keyof InternalWaffleEvents>(owner: object, event: K, payload: InternalWaffleEvents[K]): void;
 
-export { type InternalAvatarEvent, type InternalAvatarPresentation, type InternalWaffleEvents, emitInternal, subscribeInternal };
+export { type InternalAvatarPresentation, type InternalWaffleEvents, emitInternal, subscribeInternal };

package/dist/internal.js

@@ -1 +1 @@
-import{a,b}from"./chunk-BXRNROI7.js";export{b as emitInternal,a as subscribeInternal};
+import{a,b}from"./chunk-M7Z7AFVA.js";export{b as emitInternal,a as subscribeInternal};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.1.6",
+  "version": "0.2.0",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,

package/dist/chunk-BXRNROI7.js

@@ -1 +0,0 @@
-var o=new WeakMap;function s(n,t,a){let e=o.get(n)??new Map,r=e.get(t)??new Set;return r.add(a),e.set(t,r),o.set(n,e),()=>r.delete(a)}function i(n,t,a){o.get(n)?.get(t)?.forEach(e=>{e(a)})}export{s as a,i as b};