@animated-waffle/client 0.3.0 → 0.3.1

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.3.1
+
+- Pass SDK-owned request/session identities to managed starters and optional
+  end-user context through the default token start.
+
 ## 0.3.0
 
 - Require managed Agent session protocol v2 and connect to LiveKit WebRTC.

package/README.md

@@ -12,10 +12,10 @@ Install the current npm release without copying a mutable registry version:
 pnpm add @animated-waffle/client@latest
 ```
 
-The workspace [`package.json`](./package.json) defines the 0.3.0 candidate;
-npm publication is pending. That candidate requires managed Agent session
-protocol v2 and explicitly rejects the retired v1 grant. Publish the coordinated
-SDK and update consumers before cutting the managed backend over.
+The managed-session identity and `sessionContext` APIs described here were
+introduced in version 0.3.1. The workspace [`package.json`](./package.json) is
+the source of truth for this repository's version; `@latest` resolves the
+current npm release.
 
 The Agent owns its avatar, voice, and prompt. Media runtime, API routes, and
 avatar asset URLs are SDK implementation details. Production is the default
@@ -36,6 +36,10 @@ const waffle = new WaffleClient({
   agentId,
   // Set false for text-triggered, output-only applications.
   inputAudioEnabled: false,
+  sessionContext: {
+    userName: ' Space é ',
+    timeZone: 'Asia/Shanghai',
+  },
   sessionToken: async () => {
     const response = await fetch('/api/narya-session-token', {
       method: 'POST',
@@ -73,6 +77,11 @@ const waffle = new WaffleClient({
 
 ## API surface
 
+- `sessionContext?: { userName?: string; timeZone?: string }` — available only
+  with the default `sessionToken` starter. Nonblank values are sent exactly as
+  provided; blank values are omitted. It is mutually exclusive with the
+  advanced `startSession` callback, so token-path consumers do not recreate
+  the raw managed-session request.
 - `connect({ inputMode?, sessionId?, overrides? })` — `'push-to-talk'`
   (default) or `'automatic'`; resolves only when the Agent runtime is ready.
 - `inputAudioEnabled: false` — never requests or enables the browser microphone;

package/dist/index.d.ts

@@ -72,6 +72,11 @@ interface WaffleClientBaseOptions {
 interface WaffleTokenClientOptions extends WaffleClientBaseOptions {
     /** A short-lived token minted for this Agent by the Narya API. */
     sessionToken: WaffleSessionToken;
+    /** Optional end-user context for the default managed-session request. */
+    sessionContext?: {
+        readonly userName?: string;
+        readonly timeZone?: string;
+    };
     /** Override the Narya API origin for a non-production environment. */
     apiOrigin?: string;
     startSession?: never;
@@ -80,6 +85,7 @@ interface WaffleManagedClientOptions extends WaffleClientBaseOptions {
     /** Start a session through the embedding application's authenticated backend. */
     startSession: WaffleSessionStarter;
     sessionToken?: never;
+    sessionContext?: never;
     apiOrigin?: never;
 }
 type WaffleClientOptions = WaffleTokenClientOptions | WaffleManagedClientOptions;
@@ -102,7 +108,8 @@ type WaffleConnectOptions = WaffleSessionStartOptions;
 interface WaffleSessionStartInput {
     agentId: string;
     inputMode: WaffleInputMode;
-    sessionId?: string;
+    requestId: string;
+    sessionId: string;
     overrides?: Readonly<WaffleSessionOverrides>;
     signal: AbortSignal;
 }

package/dist/index.js

@@ -1 +1 @@
-import{b as k}from"./chunk-BXRNROI7.js";import{DataPacket_Kind as _,ParticipantKind as y,Room as x,RoomEvent as w,Track as b}from"livekit-client";var C=15e3,R=3e4,O="https://animated-waffle.narya.ai",M=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,N=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(e,n,s){super(e);this.status=n;this.code=s;this.name="WaffleError"}status;code};function I(o){return o!==null&&typeof o=="object"&&!Array.isArray(o)?o:null}function U(o=O){let t;try{t=new URL(o)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let e=t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="[::1]";if(t.protocol!=="https:"&&!(t.protocol==="http:"&&e)||t.username||t.password||t.pathname!=="/"&&t.pathname!==""||t.search||t.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return t.origin}function v(o,t){if(typeof o!="string"||o.trim().length===0)throw new i(`Session response is missing ${t}.`);return o}function D(o){let t=v(o,"avatar.asset_hash");if(!N.test(t))throw new i("Session response contains an invalid Avatar asset hash.");return t}function W(o){return typeof o=="number"&&Number.isFinite(o)?o:void 0}function g(o){return typeof o=="string"&&o.length>0?o:void 0}function A(o,t){let e=g(o[`${t}.speech_id`]),n=Number(o[`${t}.speech_step_index`]);return e&&Number.isSafeInteger(n)&&n>0?JSON.stringify([e,n]):null}function K(o,t,e){let n=I(o);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==2)throw new i("Session response has an unsupported protocol version.");let s=I(n.connection);if(!s||s.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let c=v(s.url,"connection.url");try{let u=new URL(c),m=u.hostname==="localhost"||u.hostname==="127.0.0.1"||u.hostname==="[::1]";if(u.protocol!=="wss:"&&!(u.protocol==="ws:"&&m))throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let d=v(n.agent_id,"agent_id");if(d!==e)throw new i("Session response contains a mismatched Agent id.");let l=g(n.avatar_id),a=I(n.avatar),p=null;if(l||a){if(!l||!a||v(a.id,"avatar.id")!==l)throw new i("Session response contains mismatched Avatar metadata.");let u=v(a.engine,"avatar.engine");if(u!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let m=v(a.model_url,"avatar.model_url");try{let h=new URL(m),T=h.hostname==="localhost"||h.hostname==="127.0.0.1"||h.hostname==="[::1]";if(h.protocol!=="https:"&&!(h.protocol==="http:"&&T))throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}p={id:l,name:v(a.name,"avatar.name"),engine:u,modelUrl:m,assetHash:D(a.asset_hash),runtimeProfile:v(a.runtime_profile,"avatar.runtime_profile")}}return{id:v(n.session_id,"session_id"),agentId:d,...l?{avatarId:l}:{},avatar:p,inputMode:t,room:c,token:v(s.token,"connection.token")}}function L(o){let t=I(o);if(!t||typeof t.type!="string")return null;let e={contextId:g(t.contextId)??null,segmentId:g(t.segmentId),occurrence:W(t.occurrence),audioTime:W(t.audioTime)};if(t.type==="avatar-emotion"){let n=g(t.emotion);return n?{phase:"emotion",emotion:n,...e}:null}if(t.type==="avatar-vocal-action"){let n=g(t.action);return n?{phase:"vocal-action",action:n,...e}:null}return null}function P(o){let{avatar:t,room:e,token:n,...s}=o;return s}var E=class{room;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";agentIdentity=null;progressExecutions=new Map;progressText=new Map;completedProgressExecutions=new Set;turnId=null;nextTurnId=0;turnControl=Promise.resolve();lifecycleVersion=0;recreateRoomOnConnect=!1;constructor(t){if(this.options=t,this.apiOrigin=U(t.apiOrigin),!M.test(t.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!t.startSession&&typeof t.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.room=this.createRoom()}createRoom(){let t=new x({adaptiveStream:!0,dynacast:!0,audioCaptureDefaults:{voiceIsolation:!1}});return t.on(w.ParticipantConnected,e=>{t===this.room&&this.handleParticipant(e)}),t.on(w.ParticipantDisconnected,e=>{t!==this.room||e.kind!==y.AGENT||(this.agentIdentity=null,this.currentStatus!=="disconnecting"&&(this.handleDisconnected("The Waffle Agent disconnected."),t.disconnect().catch(n=>{this.emit("error",n instanceof Error?n:new i("Could not close the Agent room."))})))}),t.on(w.ParticipantAttributesChanged,(e,n)=>{t===this.room&&this.handleParticipant(n)}),t.on(w.TrackSubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===b.Kind.Audio&&this.setAudioTrack(e.mediaStreamTrack)}),t.on(w.TrackUnsubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===b.Kind.Audio&&this.setAudioTrack(null)}),t.on(w.DataReceived,(e,n,s,c)=>{if(!(t!==this.room||n?.kind!==y.AGENT||s!==_.RELIABLE||c!=="avatar.performance"))try{let d=L(JSON.parse(new TextDecoder().decode(e)));d&&k(this,"avatar",d)}catch{}}),t.on(w.Disconnected,()=>{t===this.room&&this.handleDisconnected("The Agent session disconnected before it was ready.")}),t.registerTextStreamHandler("lk.transcription",(e,n)=>{(async()=>{let s=this.agentIdentity?t.remoteParticipants.get(this.agentIdentity):void 0;if(t!==this.room||this.currentStatus!=="connected"||s?.kind!==y.AGENT)return;let c=e.info.attributes??{},d=g(c["lk.segment_id"]),l=g(c["lk.transcribed_track_id"]),a=c["lk.transcription_final"];if(!d||!l||a!=="true"&&a!=="false")return;let p=t.localParticipant.audioTrackPublications.has(l),u=s.audioTrackPublications.has(l);if(p===u)return;let m=p?"user":"assistant";if(n.identity!==(p?t.localParticipant.identity:s.identity))return;let h=m==="assistant"?A(c,"lk"):null;if(m==="assistant"&&!h||h&&(this.progressExecutions.set(d,h),this.completedProgressExecutions.has(h)))return;let T="";for await(let S of e){if(T+=S,t!==this.room||this.currentStatus!=="connected"||h&&this.completedProgressExecutions.has(h))return;if(T.length>0){let r={segmentId:d,role:m,text:T,complete:!1};this.progressText.set(d,r),this.emit("transcriptProgress",r)}}a==="true"&&T.length>0&&(this.emit("transcriptProgress",{segmentId:d,role:m,text:T,complete:!0}),this.progressExecutions.delete(d),this.progressText.delete(d))})().catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t.registerTextStreamHandler("animated-waffle.transcript",(e,n)=>{e.readAll().then(s=>{if(t!==this.room||n.identity!==this.agentIdentity||!s)return;let c=e.info.attributes??{},d=g(c["animated-waffle.message_id"]),l=c["animated-waffle.role"];if(!d||l!=="user"&&l!=="assistant"||c["animated-waffle.final"]!=="true")return;let a=l==="assistant"?A(c,"animated-waffle"):null;a&&this.completeTranscriptExecution(a);let p=Number(e.info.timestamp);this.emit("transcript",{id:d,role:l,text:s,final:!0,...Number.isFinite(p)&&p>0?{timestamp:new Date(p).toISOString()}:{}})}).catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?P(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(t){this.audioMuted=t,this.audio&&(this.audio.muted=t)}on(t,e){let n=this.listeners.get(t)??new Set;return n.add(e),this.listeners.set(t,n),()=>n.delete(e)}connect(t={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(P(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRoomOnConnect&&(this.room=this.createRoom(),this.recreateRoomOnConnect=!1);let e=++this.lifecycleVersion,n=this.startInternal(t,e).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let t=this.stopInternal().finally(()=>{this.stopPromise===t&&(this.stopPromise=null)});return this.stopPromise=t,t}async sendText(t,e){if(!t.trim())return;this.assertConnected();let n=e?.audioResponse===!1?{"animated-waffle.audio_response":"false"}:void 0;await this.room.localParticipant.sendText(t,{topic:"lk.chat",...n?{attributes:n}:{}})}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start_turn")}catch(t){throw this.turnId=null,t}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("end_turn"),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel_turn"),this.turnId=null)}async startInternal(t,e){this.activeInputMode=t.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let s=t.sessionId??crypto.randomUUID(),c=crypto.randomUUID(),d=this.options.startSession,l=()=>this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...t.sessionId?{sessionId:t.sessionId}:{},...t.overrides?{overrides:t.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(r=>fetch(this.apiOrigin+"/v1/agents/"+encodeURIComponent(this.options.agentId)+"/sessions",{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:"Bearer "+r,"Idempotency-Key":c},signal:n.signal,body:JSON.stringify({chat_session_id:s,input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic",...t.overrides?{overrides:t.overrides}:{}})})),a;try{a=await this.withTimeout(d?l():l().catch(r=>{if(n.signal.aborted)throw r;return l()}),C,"Timed out creating the Waffle session.",()=>n.abort())}catch(r){let f=r instanceof Error?r:new i("Could not create session.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let p=await a.json().catch(()=>null);if(!a.ok){let r=I(p),f=new i(g(r?.detail)??"Could not create session.",a.status,g(r?.code));throw this.fail(f),f}let u;try{u=K(p,this.activeInputMode,this.options.agentId)}catch(r){let f=r instanceof Error?r:new i("Could not parse the Waffle session.");throw this.fail(f),f}this.currentSession=u,this.emit("session",P(u)),k(this,"presentation",u.avatar),this.setStatus("connecting");let m,h,T=new Promise((r,f)=>{m=r,h=f});this.readyWaiter={reject:h,resolve:m,version:e};let S=this.room;try{if(await S.connect(u.room,u.token),await S.startAudio(),this.options.inputAudioEnabled!==!1){let r=await S.localParticipant.setMicrophoneEnabled(!0),f=r?.audioTrack?.mediaStreamTrack;if(!r||r.isMuted||r.isUpstreamPaused||!f||!f.enabled||f.muted||f.readyState!=="live")throw new i("The microphone track did not publish.")}for(let r of S.remoteParticipants.values())this.handleParticipant(r);await this.withTimeout(T,R,"Timed out waiting for the Waffle Agent.",()=>{S.disconnect().catch(()=>{})})}catch(r){this.readyWaiter?.version===e&&(this.readyWaiter=null),this.currentStatus!=="disconnected"&&await S.disconnect().catch(()=>{});let f=r instanceof Error?r:new i("Could not connect.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.readyWaiter?.version===e&&(this.readyWaiter=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.setStatus("connected"),P(u)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.turnControl=Promise.resolve(),this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.room.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected();let e=this.agentIdentity;if(!e)throw new i("The Waffle Agent is not ready.");let n=this.room,s=this.turnControl.then(()=>n.localParticipant.performRpc({destinationIdentity:e,method:t,payload:""}));this.turnControl=s.then(()=>{},c=>{this.emit("error",c instanceof Error?c:new i("Push-to-talk control failed."))})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleParticipant(t){if(t.kind===y.AGENT){this.agentIdentity=t.identity,this.emit("agentSpeaking",t.attributes["lk.agent.state"]==="speaking"),t.attributes["animated-waffle.runtime.ready"]==="true"&&this.readyWaiter?.resolve();for(let e of t.audioTrackPublications.values()){let n=e.track;if(n?.kind===b.Kind.Audio){this.setAudioTrack(n.mediaStreamTrack);break}}}}handleDisconnected(t){this.readyWaiter?.reject(new i(t)),this.readyWaiter=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(t){if(t===this.currentRemoteAudioTrack||(this.stopAudio(!1),this.currentRemoteAudioTrack=t,this.emit("remoteAudioTrack",t),k(this,"audioTrack",t),!t||typeof document>"u")||this.options.audioPlayback===!1)return;let e=document.createElement("audio");e.autoplay=!0,e.muted=this.audioMuted,e.setAttribute("playsinline",""),e.srcObject=new MediaStream([t]),this.audio=e,e.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(t=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),t&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),k(this,"audioTrack",null))}completeTranscriptExecution(t){this.completedProgressExecutions.add(t);for(let[e,n]of this.progressText)this.progressExecutions.get(e)===t&&(this.emit("transcriptProgress",{...n,complete:!0}),this.progressExecutions.delete(e),this.progressText.delete(e))}clearTranscriptProgress(){this.progressExecutions.clear(),this.progressText.clear(),this.completedProgressExecutions.clear()}setStatus(t){t==="disconnected"&&(this.recreateRoomOnConnect=!0),t==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),k(this,"presentation",null)),this.currentStatus!==t&&(this.currentStatus=t,this.emit("connectionState",t),this.emit("status",t))}fail(t){this.readyWaiter?.reject(t),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected"),this.emit("error",t)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let t=await this.options.sessionToken();if(!t.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return t}withTimeout(t,e,n,s){return new Promise((c,d)=>{let l=setTimeout(()=>{s?.(),d(new i(n))},e);t.then(a=>{clearTimeout(l),c(a)},a=>{clearTimeout(l),d(a)})})}emit(t,e){this.listeners.get(t)?.forEach(n=>{n(e)})}};export{E as WaffleClient,i as WaffleError};
+import{b as k}from"./chunk-BXRNROI7.js";import{DataPacket_Kind as C,ParticipantKind as y,Room as R,RoomEvent as S,Track as A}from"livekit-client";var O=15e3,M=3e4,N="https://animated-waffle.narya.ai",U=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,D=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(e,n,s){super(e);this.status=n;this.code=s;this.name="WaffleError"}status;code};function P(o){return o!==null&&typeof o=="object"&&!Array.isArray(o)?o:null}function K(o=N){let t;try{t=new URL(o)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let e=t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="[::1]";if(t.protocol!=="https:"&&!(t.protocol==="http:"&&e)||t.username||t.password||t.pathname!=="/"&&t.pathname!==""||t.search||t.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return t.origin}function T(o,t){if(typeof o!="string"||o.trim().length===0)throw new i(`Session response is missing ${t}.`);return o}function L(o){let t=T(o,"avatar.asset_hash");if(!D.test(t))throw new i("Session response contains an invalid Avatar asset hash.");return t}function E(o){return typeof o=="number"&&Number.isFinite(o)?o:void 0}function g(o){return typeof o=="string"&&o.length>0?o:void 0}function x(o,t){let e=g(o[`${t}.speech_id`]),n=Number(o[`${t}.speech_step_index`]);return e&&Number.isSafeInteger(n)&&n>0?JSON.stringify([e,n]):null}function j(o,t,e){let n=P(o);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==2)throw new i("Session response has an unsupported protocol version.");let s=P(n.connection);if(!s||s.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let c=T(s.url,"connection.url");try{let h=new URL(c),m=h.hostname==="localhost"||h.hostname==="127.0.0.1"||h.hostname==="[::1]";if(h.protocol!=="wss:"&&!(h.protocol==="ws:"&&m))throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let a=T(n.agent_id,"agent_id");if(a!==e)throw new i("Session response contains a mismatched Agent id.");let l=g(n.avatar_id),d=P(n.avatar),p=null;if(l||d){if(!l||!d||T(d.id,"avatar.id")!==l)throw new i("Session response contains mismatched Avatar metadata.");let h=T(d.engine,"avatar.engine");if(h!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let m=T(d.model_url,"avatar.model_url");try{let u=new URL(m),v=u.hostname==="localhost"||u.hostname==="127.0.0.1"||u.hostname==="[::1]";if(u.protocol!=="https:"&&!(u.protocol==="http:"&&v))throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}p={id:l,name:T(d.name,"avatar.name"),engine:h,modelUrl:m,assetHash:L(d.asset_hash),runtimeProfile:T(d.runtime_profile,"avatar.runtime_profile")}}return{id:T(n.session_id,"session_id"),agentId:a,...l?{avatarId:l}:{},avatar:p,inputMode:t,room:c,token:T(s.token,"connection.token")}}function q(o){let t=P(o);if(!t||typeof t.type!="string")return null;let e={contextId:g(t.contextId)??null,segmentId:g(t.segmentId),occurrence:E(t.occurrence),audioTime:E(t.audioTime)};if(t.type==="avatar-emotion"){let n=g(t.emotion);return n?{phase:"emotion",emotion:n,...e}:null}if(t.type==="avatar-vocal-action"){let n=g(t.action);return n?{phase:"vocal-action",action:n,...e}:null}return null}function W(o){let{avatar:t,room:e,token:n,...s}=o;return s}var _=class{room;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";agentIdentity=null;progressExecutions=new Map;progressText=new Map;completedProgressExecutions=new Set;turnId=null;nextTurnId=0;turnControl=Promise.resolve();lifecycleVersion=0;recreateRoomOnConnect=!1;constructor(t){if(this.options=t,this.apiOrigin=K(t.apiOrigin),!U.test(t.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!t.startSession&&typeof t.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.room=this.createRoom()}createRoom(){let t=new R({adaptiveStream:!0,dynacast:!0,audioCaptureDefaults:{voiceIsolation:!1}});return t.on(S.ParticipantConnected,e=>{t===this.room&&this.handleParticipant(e)}),t.on(S.ParticipantDisconnected,e=>{t!==this.room||e.kind!==y.AGENT||(this.agentIdentity=null,this.currentStatus!=="disconnecting"&&(this.handleDisconnected("The Waffle Agent disconnected."),t.disconnect().catch(n=>{this.emit("error",n instanceof Error?n:new i("Could not close the Agent room."))})))}),t.on(S.ParticipantAttributesChanged,(e,n)=>{t===this.room&&this.handleParticipant(n)}),t.on(S.TrackSubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===A.Kind.Audio&&this.setAudioTrack(e.mediaStreamTrack)}),t.on(S.TrackUnsubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===A.Kind.Audio&&this.setAudioTrack(null)}),t.on(S.DataReceived,(e,n,s,c)=>{if(!(t!==this.room||n?.kind!==y.AGENT||s!==C.RELIABLE||c!=="avatar.performance"))try{let a=q(JSON.parse(new TextDecoder().decode(e)));a&&k(this,"avatar",a)}catch{}}),t.on(S.Disconnected,()=>{t===this.room&&this.handleDisconnected("The Agent session disconnected before it was ready.")}),t.registerTextStreamHandler("lk.transcription",(e,n)=>{(async()=>{let s=this.agentIdentity?t.remoteParticipants.get(this.agentIdentity):void 0;if(t!==this.room||this.currentStatus!=="connected"||s?.kind!==y.AGENT)return;let c=e.info.attributes??{},a=g(c["lk.segment_id"]),l=g(c["lk.transcribed_track_id"]),d=c["lk.transcription_final"];if(!a||!l||d!=="true"&&d!=="false")return;let p=t.localParticipant.audioTrackPublications.has(l),h=s.audioTrackPublications.has(l);if(p===h)return;let m=p?"user":"assistant";if(n.identity!==(p?t.localParticipant.identity:s.identity))return;let u=m==="assistant"?x(c,"lk"):null;if(m==="assistant"&&!u||u&&(this.progressExecutions.set(a,u),this.completedProgressExecutions.has(u)))return;let v="";for await(let I of e){if(v+=I,t!==this.room||this.currentStatus!=="connected"||u&&this.completedProgressExecutions.has(u))return;if(v.length>0){let b={segmentId:a,role:m,text:v,complete:!1};this.progressText.set(a,b),this.emit("transcriptProgress",b)}}d==="true"&&v.length>0&&(this.emit("transcriptProgress",{segmentId:a,role:m,text:v,complete:!0}),this.progressExecutions.delete(a),this.progressText.delete(a))})().catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t.registerTextStreamHandler("animated-waffle.transcript",(e,n)=>{e.readAll().then(s=>{if(t!==this.room||n.identity!==this.agentIdentity||!s)return;let c=e.info.attributes??{},a=g(c["animated-waffle.message_id"]),l=c["animated-waffle.role"];if(!a||l!=="user"&&l!=="assistant"||c["animated-waffle.final"]!=="true")return;let d=l==="assistant"?x(c,"animated-waffle"):null;d&&this.completeTranscriptExecution(d);let p=Number(e.info.timestamp);this.emit("transcript",{id:a,role:l,text:s,final:!0,...Number.isFinite(p)&&p>0?{timestamp:new Date(p).toISOString()}:{}})}).catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?W(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(t){this.audioMuted=t,this.audio&&(this.audio.muted=t)}on(t,e){let n=this.listeners.get(t)??new Set;return n.add(e),this.listeners.set(t,n),()=>n.delete(e)}connect(t={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(W(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRoomOnConnect&&(this.room=this.createRoom(),this.recreateRoomOnConnect=!1);let e=++this.lifecycleVersion,n=this.startInternal(t,e).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let t=this.stopInternal().finally(()=>{this.stopPromise===t&&(this.stopPromise=null)});return this.stopPromise=t,t}async sendText(t,e){if(!t.trim())return;this.assertConnected();let n=e?.audioResponse===!1?{"animated-waffle.audio_response":"false"}:void 0;await this.room.localParticipant.sendText(t,{topic:"lk.chat",...n?{attributes:n}:{}})}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start_turn")}catch(t){throw this.turnId=null,t}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("end_turn"),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel_turn"),this.turnId=null)}async startInternal(t,e){this.activeInputMode=t.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let s=t.sessionId??crypto.randomUUID(),c=crypto.randomUUID(),a=this.options.startSession,l=a?void 0:this.options.sessionContext,d=a?void 0:JSON.stringify({chat_session_id:s,input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic",...t.overrides?{overrides:t.overrides}:{},...l?.userName?.trim()?{user_name:l.userName}:{},...l?.timeZone?.trim()?{time_zone:l.timeZone}:{}}),p=()=>this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,requestId:c,sessionId:s,...t.overrides?{overrides:t.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(r=>fetch(this.apiOrigin+"/v1/agents/"+encodeURIComponent(this.options.agentId)+"/sessions",{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:"Bearer "+r,"Idempotency-Key":c},signal:n.signal,body:d})),h;try{h=await this.withTimeout(a?p():p().catch(r=>{if(n.signal.aborted)throw r;return p()}),O,"Timed out creating the Waffle session.",()=>n.abort())}catch(r){let f=r instanceof Error?r:new i("Could not create session.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let m=await h.json().catch(()=>null);if(!h.ok){let r=P(m),f=new i(g(r?.detail)??"Could not create session.",h.status,g(r?.code));throw this.fail(f),f}let u;try{u=j(m,this.activeInputMode,this.options.agentId)}catch(r){let f=r instanceof Error?r:new i("Could not parse the Waffle session.");throw this.fail(f),f}this.currentSession=u,this.emit("session",W(u)),k(this,"presentation",u.avatar),this.setStatus("connecting");let v,I,b=new Promise((r,f)=>{v=r,I=f});this.readyWaiter={reject:I,resolve:v,version:e};let w=this.room;try{if(await w.connect(u.room,u.token),await w.startAudio(),this.options.inputAudioEnabled!==!1){let r=await w.localParticipant.setMicrophoneEnabled(!0),f=r?.audioTrack?.mediaStreamTrack;if(!r||r.isMuted||r.isUpstreamPaused||!f||!f.enabled||f.muted||f.readyState!=="live")throw new i("The microphone track did not publish.")}for(let r of w.remoteParticipants.values())this.handleParticipant(r);await this.withTimeout(b,M,"Timed out waiting for the Waffle Agent.",()=>{w.disconnect().catch(()=>{})})}catch(r){this.readyWaiter?.version===e&&(this.readyWaiter=null),this.currentStatus!=="disconnected"&&await w.disconnect().catch(()=>{});let f=r instanceof Error?r:new i("Could not connect.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.readyWaiter?.version===e&&(this.readyWaiter=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.setStatus("connected"),W(u)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.turnControl=Promise.resolve(),this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.room.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected();let e=this.agentIdentity;if(!e)throw new i("The Waffle Agent is not ready.");let n=this.room,s=this.turnControl.then(()=>n.localParticipant.performRpc({destinationIdentity:e,method:t,payload:""}));this.turnControl=s.then(()=>{},c=>{this.emit("error",c instanceof Error?c:new i("Push-to-talk control failed."))})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleParticipant(t){if(t.kind===y.AGENT){this.agentIdentity=t.identity,this.emit("agentSpeaking",t.attributes["lk.agent.state"]==="speaking"),t.attributes["animated-waffle.runtime.ready"]==="true"&&this.readyWaiter?.resolve();for(let e of t.audioTrackPublications.values()){let n=e.track;if(n?.kind===A.Kind.Audio){this.setAudioTrack(n.mediaStreamTrack);break}}}}handleDisconnected(t){this.readyWaiter?.reject(new i(t)),this.readyWaiter=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(t){if(t===this.currentRemoteAudioTrack||(this.stopAudio(!1),this.currentRemoteAudioTrack=t,this.emit("remoteAudioTrack",t),k(this,"audioTrack",t),!t||typeof document>"u")||this.options.audioPlayback===!1)return;let e=document.createElement("audio");e.autoplay=!0,e.muted=this.audioMuted,e.setAttribute("playsinline",""),e.srcObject=new MediaStream([t]),this.audio=e,e.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(t=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),t&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),k(this,"audioTrack",null))}completeTranscriptExecution(t){this.completedProgressExecutions.add(t);for(let[e,n]of this.progressText)this.progressExecutions.get(e)===t&&(this.emit("transcriptProgress",{...n,complete:!0}),this.progressExecutions.delete(e),this.progressText.delete(e))}clearTranscriptProgress(){this.progressExecutions.clear(),this.progressText.clear(),this.completedProgressExecutions.clear()}setStatus(t){t==="disconnected"&&(this.recreateRoomOnConnect=!0),t==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),k(this,"presentation",null)),this.currentStatus!==t&&(this.currentStatus=t,this.emit("connectionState",t),this.emit("status",t))}fail(t){this.readyWaiter?.reject(t),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected"),this.emit("error",t)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let t=await this.options.sessionToken();if(!t.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return t}withTimeout(t,e,n,s){return new Promise((c,a)=>{let l=setTimeout(()=>{s?.(),a(new i(n))},e);t.then(d=>{clearTimeout(l),c(d)},d=>{clearTimeout(l),a(d)})})}emit(t,e){this.listeners.get(t)?.forEach(n=>{n(e)})}};export{_ as WaffleClient,i as WaffleError};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,