@animated-waffle/client 0.3.0 → 0.3.2

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.3.2
+
+- Emit `declined` when an opted-in Agent deliberately answers a live turn with silence.
+
+## 0.3.1
+
+- Pass SDK-owned request/session identities to managed starters and optional
+  end-user context through the default token start.
+
 ## 0.3.0
 
 - Require managed Agent session protocol v2 and connect to LiveKit WebRTC.

package/README.md

@@ -12,10 +12,10 @@ Install the current npm release without copying a mutable registry version:
 pnpm add @animated-waffle/client@latest
 ```
 
-The workspace [`package.json`](./package.json) defines the 0.3.0 candidate;
-npm publication is pending. That candidate requires managed Agent session
-protocol v2 and explicitly rejects the retired v1 grant. Publish the coordinated
-SDK and update consumers before cutting the managed backend over.
+The managed-session identity and `sessionContext` APIs described here were
+introduced in version 0.3.1. The workspace [`package.json`](./package.json) is
+the source of truth for this repository's version; `@latest` resolves the
+current npm release.
 
 The Agent owns its avatar, voice, and prompt. Media runtime, API routes, and
 avatar asset URLs are SDK implementation details. Production is the default
@@ -36,6 +36,10 @@ const waffle = new WaffleClient({
   agentId,
   // Set false for text-triggered, output-only applications.
   inputAudioEnabled: false,
+  sessionContext: {
+    userName: ' Space é ',
+    timeZone: 'Asia/Shanghai',
+  },
   sessionToken: async () => {
     const response = await fetch('/api/narya-session-token', {
       method: 'POST',
@@ -73,6 +77,11 @@ const waffle = new WaffleClient({
 
 ## API surface
 
+- `sessionContext?: { userName?: string; timeZone?: string }` — available only
+  with the default `sessionToken` starter. Nonblank values are sent exactly as
+  provided; blank values are omitted. It is mutually exclusive with the
+  advanced `startSession` callback, so token-path consumers do not recreate
+  the raw managed-session request.
 - `connect({ inputMode?, sessionId?, overrides? })` — `'push-to-talk'`
   (default) or `'automatic'`; resolves only when the Agent runtime is ready.
 - `inputAudioEnabled: false` — never requests or enables the browser microphone;
@@ -90,8 +99,9 @@ const waffle = new WaffleClient({
   audio for that turn.
 - `startPushToTalk()` / `commitPushToTalk()` / `cancelPushToTalk()`.
 - `on('connectionState' | 'session' | 'remoteAudioTrack' | 'transcriptProgress' | 'transcript' |
-  'agentSpeaking' | 'error', fn)` —
-  returns the unsubscribe function.
+  'agentSpeaking' | 'declined' | 'error', fn)` — `declined` carries the
+  operator-facing reason for deliberate silence. The call returns
+  the unsubscribe function.
 - Conversation transcripts are completed persisted rows, carry the same required
   `id` as history, and currently always have `final: true`. Provider
   sentence/progress framing never defines a public conversation row.

package/dist/index.d.ts

@@ -36,6 +36,18 @@ interface WaffleClientEvents {
     remoteAudioTrack: MediaStreamTrack | null;
     transcriptProgress: WaffleTranscriptProgress;
     transcript: WaffleTranscript;
+    /**
+     * The Agent answered a turn with silence, and said why.
+     *
+     * Only Agents configured to be allowed this ever emit it. Without the event
+     * a caller has no way to tell a considered silence from a turn that broke:
+     * both produce no audio, so both look like a timeout.
+     */
+    declined: WaffleDecline;
+}
+interface WaffleDecline {
+    /** Short, for the operator. Never shown to an end user by this SDK. */
+    readonly reason: string;
 }
 type WaffleSessionToken = () => string | Promise<string>;
 interface WaffleSessionOverrides {
@@ -72,6 +84,11 @@ interface WaffleClientBaseOptions {
 interface WaffleTokenClientOptions extends WaffleClientBaseOptions {
     /** A short-lived token minted for this Agent by the Narya API. */
     sessionToken: WaffleSessionToken;
+    /** Optional end-user context for the default managed-session request. */
+    sessionContext?: {
+        readonly userName?: string;
+        readonly timeZone?: string;
+    };
     /** Override the Narya API origin for a non-production environment. */
     apiOrigin?: string;
     startSession?: never;
@@ -80,6 +97,7 @@ interface WaffleManagedClientOptions extends WaffleClientBaseOptions {
     /** Start a session through the embedding application's authenticated backend. */
     startSession: WaffleSessionStarter;
     sessionToken?: never;
+    sessionContext?: never;
     apiOrigin?: never;
 }
 type WaffleClientOptions = WaffleTokenClientOptions | WaffleManagedClientOptions;
@@ -102,7 +120,8 @@ type WaffleConnectOptions = WaffleSessionStartOptions;
 interface WaffleSessionStartInput {
     agentId: string;
     inputMode: WaffleInputMode;
-    sessionId?: string;
+    requestId: string;
+    sessionId: string;
     overrides?: Readonly<WaffleSessionOverrides>;
     signal: AbortSignal;
 }
@@ -193,4 +212,4 @@ declare class WaffleClient implements WaffleSessionClient {
     private emit;
 }
 
-export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSendTextOptions, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript, type WaffleTranscriptProgress };
+export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, type WaffleDecline, WaffleError, type WaffleInputMode, type WaffleSendTextOptions, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript, type WaffleTranscriptProgress };

package/dist/index.js

@@ -1 +1 @@
-import{b as k}from"./chunk-BXRNROI7.js";import{DataPacket_Kind as _,ParticipantKind as y,Room as x,RoomEvent as w,Track as b}from"livekit-client";var C=15e3,R=3e4,O="https://animated-waffle.narya.ai",M=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,N=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(e,n,s){super(e);this.status=n;this.code=s;this.name="WaffleError"}status;code};function I(o){return o!==null&&typeof o=="object"&&!Array.isArray(o)?o:null}function U(o=O){let t;try{t=new URL(o)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let e=t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="[::1]";if(t.protocol!=="https:"&&!(t.protocol==="http:"&&e)||t.username||t.password||t.pathname!=="/"&&t.pathname!==""||t.search||t.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return t.origin}function v(o,t){if(typeof o!="string"||o.trim().length===0)throw new i(`Session response is missing ${t}.`);return o}function D(o){let t=v(o,"avatar.asset_hash");if(!N.test(t))throw new i("Session response contains an invalid Avatar asset hash.");return t}function W(o){return typeof o=="number"&&Number.isFinite(o)?o:void 0}function g(o){return typeof o=="string"&&o.length>0?o:void 0}function A(o,t){let e=g(o[`${t}.speech_id`]),n=Number(o[`${t}.speech_step_index`]);return e&&Number.isSafeInteger(n)&&n>0?JSON.stringify([e,n]):null}function K(o,t,e){let n=I(o);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==2)throw new i("Session response has an unsupported protocol version.");let s=I(n.connection);if(!s||s.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let c=v(s.url,"connection.url");try{let u=new URL(c),m=u.hostname==="localhost"||u.hostname==="127.0.0.1"||u.hostname==="[::1]";if(u.protocol!=="wss:"&&!(u.protocol==="ws:"&&m))throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let d=v(n.agent_id,"agent_id");if(d!==e)throw new i("Session response contains a mismatched Agent id.");let l=g(n.avatar_id),a=I(n.avatar),p=null;if(l||a){if(!l||!a||v(a.id,"avatar.id")!==l)throw new i("Session response contains mismatched Avatar metadata.");let u=v(a.engine,"avatar.engine");if(u!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let m=v(a.model_url,"avatar.model_url");try{let h=new URL(m),T=h.hostname==="localhost"||h.hostname==="127.0.0.1"||h.hostname==="[::1]";if(h.protocol!=="https:"&&!(h.protocol==="http:"&&T))throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}p={id:l,name:v(a.name,"avatar.name"),engine:u,modelUrl:m,assetHash:D(a.asset_hash),runtimeProfile:v(a.runtime_profile,"avatar.runtime_profile")}}return{id:v(n.session_id,"session_id"),agentId:d,...l?{avatarId:l}:{},avatar:p,inputMode:t,room:c,token:v(s.token,"connection.token")}}function L(o){let t=I(o);if(!t||typeof t.type!="string")return null;let e={contextId:g(t.contextId)??null,segmentId:g(t.segmentId),occurrence:W(t.occurrence),audioTime:W(t.audioTime)};if(t.type==="avatar-emotion"){let n=g(t.emotion);return n?{phase:"emotion",emotion:n,...e}:null}if(t.type==="avatar-vocal-action"){let n=g(t.action);return n?{phase:"vocal-action",action:n,...e}:null}return null}function P(o){let{avatar:t,room:e,token:n,...s}=o;return s}var E=class{room;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";agentIdentity=null;progressExecutions=new Map;progressText=new Map;completedProgressExecutions=new Set;turnId=null;nextTurnId=0;turnControl=Promise.resolve();lifecycleVersion=0;recreateRoomOnConnect=!1;constructor(t){if(this.options=t,this.apiOrigin=U(t.apiOrigin),!M.test(t.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!t.startSession&&typeof t.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.room=this.createRoom()}createRoom(){let t=new x({adaptiveStream:!0,dynacast:!0,audioCaptureDefaults:{voiceIsolation:!1}});return t.on(w.ParticipantConnected,e=>{t===this.room&&this.handleParticipant(e)}),t.on(w.ParticipantDisconnected,e=>{t!==this.room||e.kind!==y.AGENT||(this.agentIdentity=null,this.currentStatus!=="disconnecting"&&(this.handleDisconnected("The Waffle Agent disconnected."),t.disconnect().catch(n=>{this.emit("error",n instanceof Error?n:new i("Could not close the Agent room."))})))}),t.on(w.ParticipantAttributesChanged,(e,n)=>{t===this.room&&this.handleParticipant(n)}),t.on(w.TrackSubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===b.Kind.Audio&&this.setAudioTrack(e.mediaStreamTrack)}),t.on(w.TrackUnsubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===b.Kind.Audio&&this.setAudioTrack(null)}),t.on(w.DataReceived,(e,n,s,c)=>{if(!(t!==this.room||n?.kind!==y.AGENT||s!==_.RELIABLE||c!=="avatar.performance"))try{let d=L(JSON.parse(new TextDecoder().decode(e)));d&&k(this,"avatar",d)}catch{}}),t.on(w.Disconnected,()=>{t===this.room&&this.handleDisconnected("The Agent session disconnected before it was ready.")}),t.registerTextStreamHandler("lk.transcription",(e,n)=>{(async()=>{let s=this.agentIdentity?t.remoteParticipants.get(this.agentIdentity):void 0;if(t!==this.room||this.currentStatus!=="connected"||s?.kind!==y.AGENT)return;let c=e.info.attributes??{},d=g(c["lk.segment_id"]),l=g(c["lk.transcribed_track_id"]),a=c["lk.transcription_final"];if(!d||!l||a!=="true"&&a!=="false")return;let p=t.localParticipant.audioTrackPublications.has(l),u=s.audioTrackPublications.has(l);if(p===u)return;let m=p?"user":"assistant";if(n.identity!==(p?t.localParticipant.identity:s.identity))return;let h=m==="assistant"?A(c,"lk"):null;if(m==="assistant"&&!h||h&&(this.progressExecutions.set(d,h),this.completedProgressExecutions.has(h)))return;let T="";for await(let S of e){if(T+=S,t!==this.room||this.currentStatus!=="connected"||h&&this.completedProgressExecutions.has(h))return;if(T.length>0){let r={segmentId:d,role:m,text:T,complete:!1};this.progressText.set(d,r),this.emit("transcriptProgress",r)}}a==="true"&&T.length>0&&(this.emit("transcriptProgress",{segmentId:d,role:m,text:T,complete:!0}),this.progressExecutions.delete(d),this.progressText.delete(d))})().catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t.registerTextStreamHandler("animated-waffle.transcript",(e,n)=>{e.readAll().then(s=>{if(t!==this.room||n.identity!==this.agentIdentity||!s)return;let c=e.info.attributes??{},d=g(c["animated-waffle.message_id"]),l=c["animated-waffle.role"];if(!d||l!=="user"&&l!=="assistant"||c["animated-waffle.final"]!=="true")return;let a=l==="assistant"?A(c,"animated-waffle"):null;a&&this.completeTranscriptExecution(a);let p=Number(e.info.timestamp);this.emit("transcript",{id:d,role:l,text:s,final:!0,...Number.isFinite(p)&&p>0?{timestamp:new Date(p).toISOString()}:{}})}).catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?P(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(t){this.audioMuted=t,this.audio&&(this.audio.muted=t)}on(t,e){let n=this.listeners.get(t)??new Set;return n.add(e),this.listeners.set(t,n),()=>n.delete(e)}connect(t={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(P(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRoomOnConnect&&(this.room=this.createRoom(),this.recreateRoomOnConnect=!1);let e=++this.lifecycleVersion,n=this.startInternal(t,e).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let t=this.stopInternal().finally(()=>{this.stopPromise===t&&(this.stopPromise=null)});return this.stopPromise=t,t}async sendText(t,e){if(!t.trim())return;this.assertConnected();let n=e?.audioResponse===!1?{"animated-waffle.audio_response":"false"}:void 0;await this.room.localParticipant.sendText(t,{topic:"lk.chat",...n?{attributes:n}:{}})}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start_turn")}catch(t){throw this.turnId=null,t}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("end_turn"),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel_turn"),this.turnId=null)}async startInternal(t,e){this.activeInputMode=t.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let s=t.sessionId??crypto.randomUUID(),c=crypto.randomUUID(),d=this.options.startSession,l=()=>this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...t.sessionId?{sessionId:t.sessionId}:{},...t.overrides?{overrides:t.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(r=>fetch(this.apiOrigin+"/v1/agents/"+encodeURIComponent(this.options.agentId)+"/sessions",{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:"Bearer "+r,"Idempotency-Key":c},signal:n.signal,body:JSON.stringify({chat_session_id:s,input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic",...t.overrides?{overrides:t.overrides}:{}})})),a;try{a=await this.withTimeout(d?l():l().catch(r=>{if(n.signal.aborted)throw r;return l()}),C,"Timed out creating the Waffle session.",()=>n.abort())}catch(r){let f=r instanceof Error?r:new i("Could not create session.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let p=await a.json().catch(()=>null);if(!a.ok){let r=I(p),f=new i(g(r?.detail)??"Could not create session.",a.status,g(r?.code));throw this.fail(f),f}let u;try{u=K(p,this.activeInputMode,this.options.agentId)}catch(r){let f=r instanceof Error?r:new i("Could not parse the Waffle session.");throw this.fail(f),f}this.currentSession=u,this.emit("session",P(u)),k(this,"presentation",u.avatar),this.setStatus("connecting");let m,h,T=new Promise((r,f)=>{m=r,h=f});this.readyWaiter={reject:h,resolve:m,version:e};let S=this.room;try{if(await S.connect(u.room,u.token),await S.startAudio(),this.options.inputAudioEnabled!==!1){let r=await S.localParticipant.setMicrophoneEnabled(!0),f=r?.audioTrack?.mediaStreamTrack;if(!r||r.isMuted||r.isUpstreamPaused||!f||!f.enabled||f.muted||f.readyState!=="live")throw new i("The microphone track did not publish.")}for(let r of S.remoteParticipants.values())this.handleParticipant(r);await this.withTimeout(T,R,"Timed out waiting for the Waffle Agent.",()=>{S.disconnect().catch(()=>{})})}catch(r){this.readyWaiter?.version===e&&(this.readyWaiter=null),this.currentStatus!=="disconnected"&&await S.disconnect().catch(()=>{});let f=r instanceof Error?r:new i("Could not connect.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.readyWaiter?.version===e&&(this.readyWaiter=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.setStatus("connected"),P(u)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.turnControl=Promise.resolve(),this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.room.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected();let e=this.agentIdentity;if(!e)throw new i("The Waffle Agent is not ready.");let n=this.room,s=this.turnControl.then(()=>n.localParticipant.performRpc({destinationIdentity:e,method:t,payload:""}));this.turnControl=s.then(()=>{},c=>{this.emit("error",c instanceof Error?c:new i("Push-to-talk control failed."))})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleParticipant(t){if(t.kind===y.AGENT){this.agentIdentity=t.identity,this.emit("agentSpeaking",t.attributes["lk.agent.state"]==="speaking"),t.attributes["animated-waffle.runtime.ready"]==="true"&&this.readyWaiter?.resolve();for(let e of t.audioTrackPublications.values()){let n=e.track;if(n?.kind===b.Kind.Audio){this.setAudioTrack(n.mediaStreamTrack);break}}}}handleDisconnected(t){this.readyWaiter?.reject(new i(t)),this.readyWaiter=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(t){if(t===this.currentRemoteAudioTrack||(this.stopAudio(!1),this.currentRemoteAudioTrack=t,this.emit("remoteAudioTrack",t),k(this,"audioTrack",t),!t||typeof document>"u")||this.options.audioPlayback===!1)return;let e=document.createElement("audio");e.autoplay=!0,e.muted=this.audioMuted,e.setAttribute("playsinline",""),e.srcObject=new MediaStream([t]),this.audio=e,e.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(t=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),t&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),k(this,"audioTrack",null))}completeTranscriptExecution(t){this.completedProgressExecutions.add(t);for(let[e,n]of this.progressText)this.progressExecutions.get(e)===t&&(this.emit("transcriptProgress",{...n,complete:!0}),this.progressExecutions.delete(e),this.progressText.delete(e))}clearTranscriptProgress(){this.progressExecutions.clear(),this.progressText.clear(),this.completedProgressExecutions.clear()}setStatus(t){t==="disconnected"&&(this.recreateRoomOnConnect=!0),t==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),k(this,"presentation",null)),this.currentStatus!==t&&(this.currentStatus=t,this.emit("connectionState",t),this.emit("status",t))}fail(t){this.readyWaiter?.reject(t),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected"),this.emit("error",t)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let t=await this.options.sessionToken();if(!t.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return t}withTimeout(t,e,n,s){return new Promise((c,d)=>{let l=setTimeout(()=>{s?.(),d(new i(n))},e);t.then(a=>{clearTimeout(l),c(a)},a=>{clearTimeout(l),d(a)})})}emit(t,e){this.listeners.get(t)?.forEach(n=>{n(e)})}};export{E as WaffleClient,i as WaffleError};
+import{b as k}from"./chunk-BXRNROI7.js";import{DataPacket_Kind as C,ParticipantKind as y,Room as R,RoomEvent as S,Track as A}from"livekit-client";var O=15e3,M=3e4,N="https://animated-waffle.narya.ai",U=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,D=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,o){super(t);this.status=n;this.code=o;this.name="WaffleError"}status;code};function P(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function K(s=N){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function T(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function L(s){let e=T(s,"avatar.asset_hash");if(!D.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function E(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function g(s){return typeof s=="string"&&s.length>0?s:void 0}function x(s,e){let t=g(s[`${e}.speech_id`]),n=Number(s[`${e}.speech_step_index`]);return t&&Number.isSafeInteger(n)&&n>0?JSON.stringify([t,n]):null}function j(s,e,t){let n=P(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==2)throw new i("Session response has an unsupported protocol version.");let o=P(n.connection);if(!o||o.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let a=T(o.url,"connection.url");try{let h=new URL(a),m=h.hostname==="localhost"||h.hostname==="127.0.0.1"||h.hostname==="[::1]";if(h.protocol!=="wss:"&&!(h.protocol==="ws:"&&m))throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let d=T(n.agent_id,"agent_id");if(d!==t)throw new i("Session response contains a mismatched Agent id.");let r=g(n.avatar_id),l=P(n.avatar),p=null;if(r||l){if(!r||!l||T(l.id,"avatar.id")!==r)throw new i("Session response contains mismatched Avatar metadata.");let h=T(l.engine,"avatar.engine");if(h!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let m=T(l.model_url,"avatar.model_url");try{let u=new URL(m),v=u.hostname==="localhost"||u.hostname==="127.0.0.1"||u.hostname==="[::1]";if(u.protocol!=="https:"&&!(u.protocol==="http:"&&v))throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}p={id:r,name:T(l.name,"avatar.name"),engine:h,modelUrl:m,assetHash:L(l.asset_hash),runtimeProfile:T(l.runtime_profile,"avatar.runtime_profile")}}return{id:T(n.session_id,"session_id"),agentId:d,...r?{avatarId:r}:{},avatar:p,inputMode:e,room:a,token:T(o.token,"connection.token")}}function q(s){let e=P(s);if(!e||typeof e.type!="string")return null;let t={contextId:g(e.contextId)??null,segmentId:g(e.segmentId),occurrence:E(e.occurrence),audioTime:E(e.audioTime)};if(e.type==="avatar-emotion"){let n=g(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=g(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function H(s){if(!s||typeof s!="object")return null;let e=s;if(e.type!=="agent-declined")return null;let t=g(e.reason);return t?{reason:t}:null}function W(s){let{avatar:e,room:t,token:n,...o}=s;return o}var _=class{room;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";agentIdentity=null;progressExecutions=new Map;progressText=new Map;completedProgressExecutions=new Set;turnId=null;nextTurnId=0;turnControl=Promise.resolve();lifecycleVersion=0;recreateRoomOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=K(e.apiOrigin),!U.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.room=this.createRoom()}createRoom(){let e=new R({adaptiveStream:!0,dynacast:!0,audioCaptureDefaults:{voiceIsolation:!1}});return e.on(S.ParticipantConnected,t=>{e===this.room&&this.handleParticipant(t)}),e.on(S.ParticipantDisconnected,t=>{e!==this.room||t.kind!==y.AGENT||(this.agentIdentity=null,this.currentStatus!=="disconnecting"&&(this.handleDisconnected("The Waffle Agent disconnected."),e.disconnect().catch(n=>{this.emit("error",n instanceof Error?n:new i("Could not close the Agent room."))})))}),e.on(S.ParticipantAttributesChanged,(t,n)=>{e===this.room&&this.handleParticipant(n)}),e.on(S.TrackSubscribed,(t,n,o)=>{e===this.room&&o.kind===y.AGENT&&t.kind===A.Kind.Audio&&this.setAudioTrack(t.mediaStreamTrack)}),e.on(S.TrackUnsubscribed,(t,n,o)=>{e===this.room&&o.kind===y.AGENT&&t.kind===A.Kind.Audio&&this.setAudioTrack(null)}),e.on(S.DataReceived,(t,n,o,a)=>{if(!(e!==this.room||n?.kind!==y.AGENT||o!==C.RELIABLE)&&!(a!=="avatar.performance"&&a!=="animated-waffle.turn"))try{let d=JSON.parse(new TextDecoder().decode(t));if(a==="animated-waffle.turn"){let l=H(d);l&&this.emit("declined",l);return}let r=q(d);r&&k(this,"avatar",r)}catch{}}),e.on(S.Disconnected,()=>{e===this.room&&this.handleDisconnected("The Agent session disconnected before it was ready.")}),e.registerTextStreamHandler("lk.transcription",(t,n)=>{(async()=>{let o=this.agentIdentity?e.remoteParticipants.get(this.agentIdentity):void 0;if(e!==this.room||this.currentStatus!=="connected"||o?.kind!==y.AGENT)return;let a=t.info.attributes??{},d=g(a["lk.segment_id"]),r=g(a["lk.transcribed_track_id"]),l=a["lk.transcription_final"];if(!d||!r||l!=="true"&&l!=="false")return;let p=e.localParticipant.audioTrackPublications.has(r),h=o.audioTrackPublications.has(r);if(p===h)return;let m=p?"user":"assistant";if(n.identity!==(p?e.localParticipant.identity:o.identity))return;let u=m==="assistant"?x(a,"lk"):null;if(m==="assistant"&&!u||u&&(this.progressExecutions.set(d,u),this.completedProgressExecutions.has(u)))return;let v="";for await(let b of t){if(v+=b,e!==this.room||this.currentStatus!=="connected"||u&&this.completedProgressExecutions.has(u))return;if(v.length>0){let I={segmentId:d,role:m,text:v,complete:!1};this.progressText.set(d,I),this.emit("transcriptProgress",I)}}l==="true"&&v.length>0&&(this.emit("transcriptProgress",{segmentId:d,role:m,text:v,complete:!0}),this.progressExecutions.delete(d),this.progressText.delete(d))})().catch(o=>{this.emit("error",o instanceof Error?o:new i("Transcript stream failed."))})}),e.registerTextStreamHandler("animated-waffle.transcript",(t,n)=>{t.readAll().then(o=>{if(e!==this.room||n.identity!==this.agentIdentity||!o)return;let a=t.info.attributes??{},d=g(a["animated-waffle.message_id"]),r=a["animated-waffle.role"];if(!d||r!=="user"&&r!=="assistant"||a["animated-waffle.final"]!=="true")return;let l=r==="assistant"?x(a,"animated-waffle"):null;l&&this.completeTranscriptExecution(l);let p=Number(t.info.timestamp);this.emit("transcript",{id:d,role:r,text:o,final:!0,...Number.isFinite(p)&&p>0?{timestamp:new Date(p).toISOString()}:{}})}).catch(o=>{this.emit("error",o instanceof Error?o:new i("Transcript stream failed."))})}),e}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?W(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(e){this.audioMuted=e,this.audio&&(this.audio.muted=e)}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(W(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRoomOnConnect&&(this.room=this.createRoom(),this.recreateRoomOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e,t){if(!e.trim())return;this.assertConnected();let n=t?.audioResponse===!1?{"animated-waffle.audio_response":"false"}:void 0;await this.room.localParticipant.sendText(e,{topic:"lk.chat",...n?{attributes:n}:{}})}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start_turn")}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("end_turn"),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel_turn"),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let o=e.sessionId??crypto.randomUUID(),a=crypto.randomUUID(),d=this.options.startSession,r=d?void 0:this.options.sessionContext,l=d?void 0:JSON.stringify({chat_session_id:o,input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic",...e.overrides?{overrides:e.overrides}:{},...r?.userName?.trim()?{user_name:r.userName}:{},...r?.timeZone?.trim()?{time_zone:r.timeZone}:{}}),p=()=>this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,requestId:a,sessionId:o,...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(c=>fetch(this.apiOrigin+"/v1/agents/"+encodeURIComponent(this.options.agentId)+"/sessions",{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:"Bearer "+c,"Idempotency-Key":a},signal:n.signal,body:l})),h;try{h=await this.withTimeout(d?p():p().catch(c=>{if(n.signal.aborted)throw c;return p()}),O,"Timed out creating the Waffle session.",()=>n.abort())}catch(c){let f=c instanceof Error?c:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let m=await h.json().catch(()=>null);if(!h.ok){let c=P(m),f=new i(g(c?.detail)??"Could not create session.",h.status,g(c?.code));throw this.fail(f),f}let u;try{u=j(m,this.activeInputMode,this.options.agentId)}catch(c){let f=c instanceof Error?c:new i("Could not parse the Waffle session.");throw this.fail(f),f}this.currentSession=u,this.emit("session",W(u)),k(this,"presentation",u.avatar),this.setStatus("connecting");let v,b,I=new Promise((c,f)=>{v=c,b=f});this.readyWaiter={reject:b,resolve:v,version:t};let w=this.room;try{if(await w.connect(u.room,u.token),await w.startAudio(),this.options.inputAudioEnabled!==!1){let c=await w.localParticipant.setMicrophoneEnabled(!0),f=c?.audioTrack?.mediaStreamTrack;if(!c||c.isMuted||c.isUpstreamPaused||!f||!f.enabled||f.muted||f.readyState!=="live")throw new i("The microphone track did not publish.")}for(let c of w.remoteParticipants.values())this.handleParticipant(c);await this.withTimeout(I,M,"Timed out waiting for the Waffle Agent.",()=>{w.disconnect().catch(()=>{})})}catch(c){this.readyWaiter?.version===t&&(this.readyWaiter=null),this.currentStatus!=="disconnected"&&await w.disconnect().catch(()=>{});let f=c instanceof Error?c:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.setStatus("connected"),W(u)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.turnControl=Promise.resolve(),this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.room.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected();let t=this.agentIdentity;if(!t)throw new i("The Waffle Agent is not ready.");let n=this.room,o=this.turnControl.then(()=>n.localParticipant.performRpc({destinationIdentity:t,method:e,payload:""}));this.turnControl=o.then(()=>{},a=>{this.emit("error",a instanceof Error?a:new i("Push-to-talk control failed."))})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleParticipant(e){if(e.kind===y.AGENT){this.agentIdentity=e.identity,this.emit("agentSpeaking",e.attributes["lk.agent.state"]==="speaking"),e.attributes["animated-waffle.runtime.ready"]==="true"&&this.readyWaiter?.resolve();for(let t of e.audioTrackPublications.values()){let n=t.track;if(n?.kind===A.Kind.Audio){this.setAudioTrack(n.mediaStreamTrack);break}}}}handleDisconnected(e){this.readyWaiter?.reject(new i(e)),this.readyWaiter=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(e===this.currentRemoteAudioTrack||(this.stopAudio(!1),this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),k(this,"audioTrack",e),!e||typeof document>"u")||this.options.audioPlayback===!1)return;let t=document.createElement("audio");t.autoplay=!0,t.muted=this.audioMuted,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),k(this,"audioTrack",null))}completeTranscriptExecution(e){this.completedProgressExecutions.add(e);for(let[t,n]of this.progressText)this.progressExecutions.get(t)===e&&(this.emit("transcriptProgress",{...n,complete:!0}),this.progressExecutions.delete(t),this.progressText.delete(t))}clearTranscriptProgress(){this.progressExecutions.clear(),this.progressText.clear(),this.completedProgressExecutions.clear()}setStatus(e){e==="disconnected"&&(this.recreateRoomOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),k(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,o){return new Promise((a,d)=>{let r=setTimeout(()=>{o?.(),d(new i(n))},t);e.then(l=>{clearTimeout(r),a(l)},l=>{clearTimeout(r),d(l)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{_ as WaffleClient,i as WaffleError};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,