@animated-waffle/client 0.1.5 → 0.2.0

package/README.md

@@ -35,10 +35,20 @@ const waffle = new WaffleClient({
 })
 
 waffle.on('transcript', ({ id, role, text, final }) => {
-  // Rows use the same id as persisted history. Assistant rows stream: the
-  // same id is re-sent with accumulated text while the agent speaks, then
-  // once more with final: true. Render by replacing the row with that id.
-  console.log(id, role, text, final)
+  // Completed Narya conversation rows use the same id as persisted history.
+  if (final) console.log(id, role, text)
+})
+
+waffle.on('speechSegment', ({ id, messageId, sequence, progressId, text, annotatedText }) => {
+  // Published only at this segment's queued audio start—not when text merely
+  // enters TTS or a transport-wide speaking callback fires.
+  console.log(id, messageId, sequence, progressId, text, annotatedText)
+})
+
+waffle.on('speechProgress', ({ progressId, status, accumulatedText, remainingText }) => {
+  // Factual progress associated with the segment above. The browser owns any
+  // semantic interpretation of annotatedText.
+  console.log(progressId, status, accumulatedText, remainingText)
 })
 
 await waffle.connect({ inputMode: 'push-to-talk' })
@@ -57,17 +67,25 @@ const waffle = new WaffleClient({
 })
 ```
 
+At an input interruption the client immediately resets and temporarily blocks
+speech facts. The runtime orders `conversation-speech-playout-cutoff` after all
+old source identities and begins cancelling the old media queue; the client
+then clears again and reopens the fact stream. An already-in-flight old marker
+may arrive later, but it is an orphan and cannot revive cancelled performance.
+
 ## API surface
 
 - `connect({ inputMode? })` — `'push-to-talk'` (default) or `'automatic'`.
 - `disconnect()`, `sendText(text)`.
 - `startPushToTalk()` / `commitPushToTalk()` / `cancelPushToTalk()`.
-- `on('status' | 'session' | 'transcript' | 'agentSpeaking' | 'error', fn)` —
-  returns the unsubscribe function.
+- `on('status' | 'session' | 'transcript' | 'speechSegment' | 'speechProgress' |
+  'agentSpeaking' | 'error', fn)` — returns the unsubscribe function.
 - Conversation transcripts carry the same required `id` as persisted history.
-  Assistant rows stream in place — accumulated `text` plus tag-preserving
-  `annotatedText`, then `final: true` — so replace by `id`, never append.
   Provider sentence/progress framing never defines a public conversation row.
+- Speech segments become public at their associated audio start. The SDK uses
+  raw `new` output only to correlate identity, never as a timing signal. Their
+  `annotatedText` drives local Avatar choreography while `text` remains clean
+  spoken text; `speechProgress` carries provider-paced accumulated facts.
 - `status`, `session` getters; failures throw `WaffleError` with `code`.
 
 `./internal` is a private contract consumed by `@animated-waffle/react`; it is

package/dist/chunk-M7Z7AFVA.js

@@ -0,0 +1 @@
+var s=new WeakMap;function o(n,t,a){let e=s.get(n)??new Map,r=e.get(t)??new Set;return r.add(a),e.set(t,r),s.set(n,e),()=>r.delete(a)}function l(n,t,a){s.get(n)?.get(t)?.forEach(e=>{e(a)})}export{o as a,l as b};

package/dist/index.d.ts

@@ -12,22 +12,44 @@ interface WaffleTranscript {
     /** The persisted conversation message id. */
     id: string;
     role: 'user' | 'assistant';
-    /** User-facing text with voice-control tags stripped. */
     text: string;
     /**
-     * The same text with inline voice tags (e.g. `[happy]`) preserved, present on
-     * assistant entries. Each update strictly extends the previous one for the
-     * same id, so performance layers can act on the appended suffix alone.
-     */
-    annotatedText?: string;
-    /**
-     * False while the agent is still speaking this entry. The runtime re-sends
-     * the same id with accumulated text as each sentence is spoken, then once
-     * more with final: true when the turn completes. Replace by id, not append.
+     * False while this text is still provisional. Only interim speech recognition
+     * is provisional: a caller replaces the previous non-final entry from that
+     * speaker rather than appending. Agent text is always final.
      */
     final: boolean;
     timestamp?: string;
 }
+/**
+ * One Agent speech segment, published at its associated audio start. `text` is
+ * the clean spoken text while `annotatedText` retains inline performance tags
+ * for the browser-side Avatar choreographer. It is deliberately separate from
+ * the persisted conversation transcript.
+ */
+interface WaffleSpeechSegment {
+    /** Stable speech-segment id. */
+    id: string;
+    /** Persisted assistant message this segment belongs to. */
+    messageId: string;
+    /** Zero-based order within the assistant message. */
+    sequence: number;
+    /** Numeric identity shared with Pipecat's factual playout progress. */
+    progressId: number;
+    text: string;
+    annotatedText: string;
+}
+type WaffleSpeechProgressStatus = 'new' | 'in-progress' | 'completed';
+/** Factual playout progress associated with one public speech segment. */
+interface WaffleSpeechProgress {
+    segmentId: string;
+    messageId: string;
+    sequence: number;
+    progressId: number;
+    status: WaffleSpeechProgressStatus;
+    accumulatedText: string;
+    remainingText: string;
+}
 interface WaffleClientEvents {
     connectionState: WaffleConnectionState;
     /** @deprecated Subscribe to connectionState. */
@@ -37,6 +59,10 @@ interface WaffleClientEvents {
     agentSpeaking: boolean;
     remoteAudioTrack: MediaStreamTrack | null;
     transcript: WaffleTranscript;
+    speechSegment: WaffleSpeechSegment;
+    speechProgress: WaffleSpeechProgress;
+    /** A user input boundary cancelled any pending Agent speech/performance. */
+    interrupted: undefined;
 }
 type WaffleSessionToken = () => string | Promise<string>;
 interface WaffleSessionOverrides {
@@ -132,6 +158,14 @@ declare class WaffleClient implements WaffleSessionClient {
     private nextTurnId;
     private lifecycleVersion;
     private recreateRealtimeOnConnect;
+    private readonly speechSegmentsByProgressId;
+    private readonly pendingSpeechProgress;
+    private readonly seenSpeechProgress;
+    private readonly releasedSpeechSegments;
+    private readonly pendingSpeechPlayoutStarts;
+    private botAudioActive;
+    private speechFactsBlocked;
+    private speechPlayoutCutoffVersion;
     constructor(options: WaffleClientOptions);
     private createRealtime;
     get status(): WaffleStatus;
@@ -154,6 +188,16 @@ declare class WaffleClient implements WaffleSessionClient {
     private handleTransportState;
     private handleReady;
     private handleDisconnected;
+    private handleSpeechSegment;
+    private handleSpeechPlayoutStart;
+    private handleBotOutput;
+    private handleBotStartedSpeaking;
+    private handleBotStoppedSpeaking;
+    private releaseSpeechSegment;
+    private emitSpeechProgress;
+    private resetSpeechProgress;
+    private interruptSpeech;
+    private handleSpeechPlayoutCutoff;
     private setAudioTrack;
     private stopAudio;
     private setStatus;
@@ -163,4 +207,4 @@ declare class WaffleClient implements WaffleSessionClient {
     private emit;
 }
 
-export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript };
+export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleSpeechProgress, type WaffleSpeechProgressStatus, type WaffleSpeechSegment, type WaffleStatus, type WaffleTranscript };

package/dist/index.js

@@ -1 +1 @@
-import{b as p}from"./chunk-LJ4KJFLQ.js";import{PipecatClient as T}from"@pipecat-ai/client-js";import{DailyTransport as y}from"@pipecat-ai/daily-transport";var w=15e3,k=3e4,W="https://animated-waffle.narya.ai",I=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,C=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,o){super(t);this.status=n;this.code=o;this.name="WaffleError"}status;code};function h(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function P(s=W){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function f(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function b(s){let e=f(s,"avatar.asset_hash");if(!C.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function m(s){return typeof s=="string"&&s.length>0?s:void 0}function A(s,e,t){let n=h(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let o=h(n.connection);if(!o||o.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let u=f(o.url,"connection.url");try{if(new URL(u).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=f(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=m(n.avatar_id),l=h(n.avatar),v=null;if(d||l){if(!d||!l||f(l.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let r=f(l.engine,"avatar.engine");if(r!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let a=f(l.model_url,"avatar.model_url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:f(l.name,"avatar.name"),engine:r,modelUrl:a,assetHash:b(l.asset_hash),runtimeProfile:f(l.runtime_profile,"avatar.runtime_profile")}}return{id:f(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:u,token:f(o.token,"connection.token")}}function _(s){let e=h(s),t=m(e?.tag),n=e?.kind;return e?.type!=="avatar-performance"||!t||n!=="emotion"&&n!=="vocal-action"?null:{phase:n,tag:t}}function O(s){let e=h(s),t=m(e?.messageId),n=e?.role;if(e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||typeof e.final!="boolean")return null;let o={id:t,role:n,text:e.text,final:e.final},u=m(e.annotatedText);return u!==void 0&&(o.annotatedText=u),o}function S(s){let{avatar:e,room:t,token:n,...o}=s;return o}var g=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=P(e.apiOrigin),!I.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onBotTtsText:t=>{t.text&&p(this,"spokenWord",{text:t.text})},onServerMessage:t=>{let n=O(t);n&&this.emit("transcript",n);let o=_(t);o&&p(this,"avatarPerformance",o)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(h(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new y({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new T({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let o;try{o=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(r=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${r}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),w,"Timed out creating the Waffle session.",()=>n.abort())}catch(r){let a=r instanceof Error?r:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let u=await o.json().catch(()=>null);if(!o.ok){let r=h(u),a=new i(m(r?.detail)??"Could not create session.",o.status,m(r?.code));throw this.fail(a),a}let c;try{c=A(u,this.activeInputMode,this.options.agentId)}catch(r){let a=r instanceof Error?r:new i("Could not parse the Waffle session.");throw this.fail(a),a}this.currentSession=c,this.emit("session",S(c)),p(this,"presentation",c.avatar),this.setStatus("connecting");let d,l,v=new Promise((r,a)=>{d=r,l=a});this.readyWaiter={reject:l,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),k,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(r){await this.realtime.disconnect().catch(()=>{});let a=r instanceof Error?r:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),p(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),p(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),p(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,o){return new Promise((u,c)=>{let d=setTimeout(()=>{o?.(),c(new i(n))},t);e.then(l=>{clearTimeout(d),u(l)},l=>{clearTimeout(d),c(l)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{g as WaffleClient,i as WaffleError};
+import{b as g}from"./chunk-M7Z7AFVA.js";import{PipecatClient as k}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var T=15e3,P=3e4,I="https://animated-waffle.narya.ai",W=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,b=/^sha256:[a-f0-9]{64}$/,r=class extends Error{constructor(t,n,i){super(t);this.status=n;this.code=i;this.name="WaffleError"}status;code};function h(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function C(s=I){let e;try{e=new URL(s)}catch{throw new r("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new r("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function p(s,e){if(typeof s!="string"||s.trim().length===0)throw new r(`Session response is missing ${e}.`);return s}function A(s){let e=p(s,"avatar.asset_hash");if(!b.test(e))throw new r("Session response contains an invalid Avatar asset hash.");return e}function v(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function f(s){return typeof s=="string"&&s.length>0?s:void 0}function x(s,e,t){let n=h(s);if(!n)throw new r("Session response must be a JSON object.");if(n.protocol_version!==1)throw new r("Session response has an unsupported protocol version.");let i=h(n.connection);if(!i||i.transport!=="webrtc")throw new r("Session response is missing a WebRTC connection.");let a=p(i.url,"connection.url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new r("Session response contains an invalid connection URL.")}let o=p(n.agent_id,"agent_id");if(o!==t)throw new r("Session response contains a mismatched Agent id.");let u=f(n.avatar_id),d=h(n.avatar),m=null;if(u||d){if(!u||!d||p(d.id,"avatar.id")!==u)throw new r("Session response contains mismatched Avatar metadata.");let l=p(d.engine,"avatar.engine");if(l!=="vrm")throw new r("Session response contains an unsupported Avatar engine.");let c=p(d.model_url,"avatar.model_url");try{if(new URL(c).protocol!=="https:")throw new Error}catch{throw new r("Session response contains an invalid Avatar model URL.")}m={id:u,name:p(d.name,"avatar.name"),engine:l,modelUrl:c,assetHash:A(d.asset_hash),runtimeProfile:p(d.runtime_profile,"avatar.runtime_profile")}}return{id:p(n.session_id,"session_id"),agentId:o,...u?{avatarId:u}:{},avatar:m,inputMode:e,room:a,token:p(i.token,"connection.token")}}function _(s){let e=h(s),t=f(e?.messageId),n=e?.role;return e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||e.final!==!0?null:{id:t,role:n,text:e.text,final:!0}}function O(s){let e=h(s),t=f(e?.segmentId),n=f(e?.messageId),i=v(e?.sequence),a=v(e?.progressId);return e?.type!=="conversation-speech-segment"||!t||!n||i===void 0||!Number.isInteger(i)||i<0||a===void 0||!Number.isInteger(a)||a<0||typeof e.text!="string"||typeof e.annotatedText!="string"?null:{id:t,messageId:n,sequence:i,progressId:a,text:e.text,annotatedText:e.annotatedText}}function R(s){let e=h(s),t=v(e?.progressId);return e?.type!=="conversation-speech-playout-start"||t===void 0||!Number.isInteger(t)||t<0?null:t}function M(s){return h(s)?.type==="conversation-speech-playout-cutoff"}function B(s){let e=h(s),t=h(e?.spoken_progress),n=v(e?.segment_id),i=e?.spoken_status;return n===void 0||!Number.isInteger(n)||n<0||i!=="new"&&i!=="in-progress"&&i!=="completed"||typeof t?.accumulated_text!="string"||typeof t.remaining_text!="string"?null:{progressId:n,status:i,accumulatedText:t.accumulated_text,remainingText:t.remaining_text}}function E(s){return JSON.stringify([s.status,s.accumulatedText,s.remainingText])}function S(s){let{avatar:e,room:t,token:n,...i}=s;return i}var y=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;speechSegmentsByProgressId=new Map;pendingSpeechProgress=new Map;seenSpeechProgress=new Map;releasedSpeechSegments=new Set;pendingSpeechPlayoutStarts=new Set;botAudioActive=!1;speechFactsBlocked=!1;speechPlayoutCutoffVersion=0;constructor(e){if(this.options=e,this.apiOrigin=C(e.apiOrigin),!W.test(e.agentId))throw new r("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new r("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.handleBotStartedSpeaking(),onBotStoppedSpeaking:()=>this.handleBotStoppedSpeaking(),onUserStartedSpeaking:()=>{this.interruptSpeech()},onBotOutput:t=>this.handleBotOutput(t),onServerMessage:t=>{if(M(t)){this.handleSpeechPlayoutCutoff();return}let n=_(t);n&&this.emit("transcript",n);let i=O(t);i&&this.handleSpeechSegment(i);let a=R(t);a!==null&&this.handleSpeechPlayoutStart(a)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new r(h(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new k({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){if(!e.trim())return;this.assertConnected();let t=this.speechFactsBlocked,n=this.speechPlayoutCutoffVersion;this.speechFactsBlocked=!0;try{await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0})}catch(i){throw this.speechFactsBlocked=t,i}this.interruptSpeech(this.speechPlayoutCutoffVersion===n)}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId),this.interruptSpeech()}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.resetSpeechProgress(),this.speechFactsBlocked=!1,this.speechPlayoutCutoffVersion=0,this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let i;try{i=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(l=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${l}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),T,"Timed out creating the Waffle session.",()=>n.abort())}catch(l){let c=l instanceof Error?l:new r("Could not create session.");throw t!==this.lifecycleVersion?new r("The connection attempt was cancelled."):(this.fail(c),c)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new r("The connection attempt was cancelled.");let a=await i.json().catch(()=>null);if(!i.ok){let l=h(a),c=new r(f(l?.detail)??"Could not create session.",i.status,f(l?.code));throw this.fail(c),c}let o;try{o=x(a,this.activeInputMode,this.options.agentId)}catch(l){let c=l instanceof Error?l:new r("Could not parse the Waffle session.");throw this.fail(c),c}this.currentSession=o,this.emit("session",S(o)),g(this,"presentation",o.avatar),this.setStatus("connecting");let u,d,m=new Promise((l,c)=>{u=l,d=c});this.readyWaiter={reject:d,resolve:u,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:o.room,token:o.token}),m]).then(()=>{}),P,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(l){await this.realtime.disconnect().catch(()=>{});let c=l instanceof Error?l:new r("Could not connect.");throw t!==this.lifecycleVersion?new r("The connection attempt was cancelled."):(this.fail(c),c)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(o)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new r("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new r('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new r("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new r("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.resetSpeechProgress(),this.speechFactsBlocked=!1,this.emit("agentSpeaking",!1),this.setStatus("disconnected")}handleSpeechSegment(e){this.speechFactsBlocked||this.speechSegmentsByProgressId.has(e.progressId)||(this.speechSegmentsByProgressId.set(e.progressId,e),this.pendingSpeechPlayoutStarts.has(e.progressId)&&this.releaseSpeechSegment(e.progressId,!0))}handleSpeechPlayoutStart(e){this.speechFactsBlocked||this.releasedSpeechSegments.has(e)||(this.pendingSpeechPlayoutStarts.add(e),this.releaseSpeechSegment(e,!0))}handleBotOutput(e){if(this.speechFactsBlocked)return;let t=B(e);if(!t)return;let n=E(t),i=this.seenSpeechProgress.get(t.progressId)??new Set;if(i.has(n))return;i.add(n),this.seenSpeechProgress.set(t.progressId,i);let a=this.speechSegmentsByProgressId.get(t.progressId);if(a&&this.releasedSpeechSegments.has(t.progressId)){this.emitSpeechProgress(a,t);return}let o=this.pendingSpeechProgress.get(t.progressId)??[];o.push(t),this.pendingSpeechProgress.set(t.progressId,o),this.pendingSpeechPlayoutStarts.has(t.progressId)&&this.releaseSpeechSegment(t.progressId,!0)}handleBotStartedSpeaking(){this.speechFactsBlocked||(this.botAudioActive=!0,this.emit("agentSpeaking",!0))}handleBotStoppedSpeaking(){this.speechFactsBlocked||(this.botAudioActive=!1,this.emit("agentSpeaking",!1))}releaseSpeechSegment(e,t=!1){if(this.releasedSpeechSegments.has(e))return!1;let n=this.speechSegmentsByProgressId.get(e),i=this.pendingSpeechProgress.get(e)??[];if(!n||!i.some(o=>o.status==="new"))return!1;t&&!this.botAudioActive&&(this.botAudioActive=!0,this.emit("agentSpeaking",!0)),this.releasedSpeechSegments.add(e),this.pendingSpeechPlayoutStarts.delete(e),this.emit("speechSegment",n),this.pendingSpeechProgress.delete(e);let a=i.find(o=>o.status==="new");a&&this.emitSpeechProgress(n,a);for(let o of i)o!==a&&this.emitSpeechProgress(n,o);return!0}emitSpeechProgress(e,t){this.emit("speechProgress",{segmentId:e.id,messageId:e.messageId,sequence:e.sequence,progressId:t.progressId,status:t.status,accumulatedText:t.accumulatedText,remainingText:t.remainingText})}resetSpeechProgress(){this.speechSegmentsByProgressId.clear(),this.pendingSpeechProgress.clear(),this.seenSpeechProgress.clear(),this.releasedSpeechSegments.clear(),this.pendingSpeechPlayoutStarts.clear(),this.botAudioActive=!1}interruptSpeech(e=!0){this.resetSpeechProgress(),this.speechFactsBlocked=e,this.emit("interrupted",void 0)}handleSpeechPlayoutCutoff(){let e=this.speechFactsBlocked;this.resetSpeechProgress(),this.speechFactsBlocked=!1,this.speechPlayoutCutoffVersion+=1,e||this.emit("interrupted",void 0)}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),g(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new r("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),g(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0,this.resetSpeechProgress(),this.speechFactsBlocked=!1),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),g(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new r("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new r("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,i){return new Promise((a,o)=>{let u=setTimeout(()=>{i?.(),o(new r(n))},t);e.then(d=>{clearTimeout(u),a(d)},d=>{clearTimeout(u),o(d)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{y as WaffleClient,r as WaffleError};

package/dist/internal.d.ts

@@ -1,7 +1,3 @@
-interface InternalAvatarPerformance {
-    phase: 'emotion' | 'vocal-action';
-    tag: string;
-}
 interface InternalAvatarPresentation {
     id: string;
     name: string;
@@ -11,25 +7,11 @@ interface InternalAvatarPresentation {
     runtimeProfile: string;
 }
 interface InternalWaffleEvents {
-    /**
-     * One performance cue, delivered by the runtime at the exact playout moment
-     * it should be performed (the server schedules it on the audio clock, with
-     * the emotion pre-entry lead already applied).
-     */
-    avatarPerformance: InternalAvatarPerformance;
     audioTrack: MediaStreamTrack | null;
     presentation: InternalAvatarPresentation | null;
-    /**
-     * One spoken word at the moment it becomes audible. Present only when the
-     * runtime's TTS provider supplies word timestamps; drives client-side
-     * performance choreography.
-     */
-    spokenWord: {
-        text: string;
-    };
 }
 type InternalListener<K extends keyof InternalWaffleEvents> = (payload: InternalWaffleEvents[K]) => void;
 declare function subscribeInternal<K extends keyof InternalWaffleEvents>(owner: object, event: K, listener: InternalListener<K>): () => void;
 declare function emitInternal<K extends keyof InternalWaffleEvents>(owner: object, event: K, payload: InternalWaffleEvents[K]): void;
 
-export { type InternalAvatarPerformance, type InternalAvatarPresentation, type InternalWaffleEvents, emitInternal, subscribeInternal };
+export { type InternalAvatarPresentation, type InternalWaffleEvents, emitInternal, subscribeInternal };

package/dist/internal.js

@@ -1 +1 @@
-import{a,b}from"./chunk-LJ4KJFLQ.js";export{b as emitInternal,a as subscribeInternal};
+import{a,b}from"./chunk-M7Z7AFVA.js";export{b as emitInternal,a as subscribeInternal};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,

package/dist/chunk-LJ4KJFLQ.js

@@ -1 +0,0 @@
-var o=new WeakMap;function s(n,t,a){let e=o.get(n)??new Map,r=e.get(t)??new Set;return r.add(a),e.set(t,r),o.set(n,e),()=>r.delete(a)}function l(n,t,a){o.get(n)?.get(t)?.forEach(e=>{e(a)})}export{s as a,l as b};