@animated-waffle/client 0.3.5 → 0.4.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.4.0
+
+- Breaking: replace session overrides `persona` and `systemPrompt` with a complete
+  `characterCard`. `WaffleSessionClient` implementations must expose the new
+  microphone/input controls and return promises for push-to-talk controls.
+- Add media clients for backend-authorized application sessions, with a versioned
+  capability handshake. Tools and context stay on the separate backend connection.
+- Expose live input-mode switching, microphone control, and interruption.
+- Return promises for queued push-to-talk controls and report control failures.
+
 ## 0.3.5
 
 - Add lazy, creator-BFF-authorized private Developer Feedback for persisted

package/README.md

@@ -161,3 +161,11 @@ not part of the public API and may change without notice.
 
 See the [SDK documentation](https://animated-waffle.narya.ai/docs/sdk) for the
 complete integration guide.
+
+## Application sessions
+
+For an application-owned conversation, use `application: true` with a
+`startSession` callback returning the media session created by
+`WaffleServer.startApplicationSession`. The backend retains the separate control
+grant; the browser never executes tools or forwards privileged callbacks.
+See the [application-session contract](../../docs/public/sdk.md#application-owned-tools-and-conversation-state).

package/dist/index.d.ts

@@ -1,3 +1,67 @@
+import { z } from 'zod';
+
+declare const characterCardSchema: z.ZodObject<{
+    spec: z.ZodLiteral<"chara_card_v3">;
+    spec_version: z.ZodLiteral<"3.0">;
+    data: z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodString;
+        personality: z.ZodString;
+        scenario: z.ZodString;
+        first_mes: z.ZodString;
+        mes_example: z.ZodString;
+        creator_notes: z.ZodString;
+        system_prompt: z.ZodString;
+        post_history_instructions: z.ZodString;
+        alternate_greetings: z.ZodArray<z.ZodString>;
+        tags: z.ZodArray<z.ZodString>;
+        creator: z.ZodString;
+        character_version: z.ZodString;
+        extensions: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+        group_only_greetings: z.ZodArray<z.ZodString>;
+        nickname: z.ZodOptional<z.ZodString>;
+        creator_notes_multilingual: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        source: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        creation_date: z.ZodOptional<z.ZodNumber>;
+        modification_date: z.ZodOptional<z.ZodNumber>;
+        assets: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            type: z.ZodString;
+            uri: z.ZodString;
+            name: z.ZodString;
+            ext: z.ZodString;
+        }, z.core.$catchall<z.ZodJSONSchema>>>>;
+        character_book: z.ZodOptional<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            description: z.ZodOptional<z.ZodString>;
+            scan_depth: z.ZodOptional<z.ZodNumber>;
+            token_budget: z.ZodOptional<z.ZodNumber>;
+            recursive_scanning: z.ZodOptional<z.ZodBoolean>;
+            extensions: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            entries: z.ZodArray<z.ZodObject<{
+                keys: z.ZodArray<z.ZodString>;
+                content: z.ZodString;
+                extensions: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+                enabled: z.ZodBoolean;
+                insertion_order: z.ZodNumber;
+                use_regex: z.ZodBoolean;
+                case_sensitive: z.ZodOptional<z.ZodBoolean>;
+                name: z.ZodOptional<z.ZodString>;
+                priority: z.ZodOptional<z.ZodNumber>;
+                id: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>;
+                comment: z.ZodOptional<z.ZodString>;
+                selective: z.ZodOptional<z.ZodBoolean>;
+                secondary_keys: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                constant: z.ZodOptional<z.ZodBoolean>;
+                position: z.ZodOptional<z.ZodEnum<{
+                    before_char: "before_char";
+                    after_char: "after_char";
+                }>>;
+            }, z.core.$catchall<z.ZodJSONSchema>>>;
+        }, z.core.$catchall<z.ZodJSONSchema>>>;
+    }, z.core.$catchall<z.ZodJSONSchema>>;
+}, z.core.$catchall<z.ZodJSONSchema>>;
+type CharacterCard = z.infer<typeof characterCardSchema>;
+
 type WaffleConnectionState = 'disconnected' | 'connecting' | 'connected' | 'disconnecting';
 /** @deprecated Use WaffleConnectionState. */
 type WaffleStatus = WaffleConnectionState;
@@ -87,13 +151,12 @@ interface WaffleDecline {
 }
 type WaffleSessionToken = () => string | Promise<string>;
 interface WaffleSessionOverrides {
-    persona?: string;
+    characterCard?: CharacterCard;
     llmProvider?: string | null;
     llmModel?: string | null;
     thinkingLevel?: string;
     webSearchEnabled?: boolean;
     memoryEnabled?: boolean;
-    systemPrompt?: string;
     asrProvider?: string | null;
     ttsProvider?: string;
     ttsModelId?: string;
@@ -105,6 +168,8 @@ interface WaffleSessionOverrides {
     avatarEnabled?: boolean;
 }
 interface WaffleClientBaseOptions {
+    /** Require an application session whose tools and context are controlled by the backend. */
+    application?: boolean;
     /** The published Agent to connect to. */
     agentId: string;
     /** Disable browser microphone access for text-triggered, output-only sessions. */
@@ -176,6 +241,7 @@ interface WaffleSessionClient {
     readonly status: WaffleStatus;
     readonly session: WaffleSession | null;
     readonly remoteAudioTrack: MediaStreamTrack | null;
+    readonly microphoneTrack: MediaStreamTrack | null;
     /**
      * Silence the Agent's voice while it keeps performing: the session, the
      * transcript and the Avatar's cues carry on, only playback goes quiet. Has
@@ -189,16 +255,27 @@ interface WaffleSessionClient {
     loadDeveloperFeedback(): Promise<WaffleDeveloperFeedbackState>;
     submitDeveloperFeedback(input: WaffleDeveloperFeedbackInput): Promise<WaffleDeveloperFeedbackState>;
     sendText(text: string, options?: WaffleSendTextOptions): Promise<void>;
-    startPushToTalk(): void;
-    commitPushToTalk(): void;
-    cancelPushToTalk(): void;
+    startPushToTalk(): Promise<void>;
+    commitPushToTalk(): Promise<void>;
+    cancelPushToTalk(): Promise<void>;
+    setInputMode(mode: WaffleInputMode): Promise<void>;
+    interrupt(): Promise<void>;
+    setMicrophoneEnabled(enabled: boolean): Promise<void>;
 }
 type Listener<K extends keyof WaffleClientEvents> = (payload: WaffleClientEvents[K]) => void;
 type WaffleErrorStage = 'session' | 'transport' | 'audio' | 'transcript' | 'avatar-load' | 'avatar-renderer';
+type PushToTalkControlMethod = 'start_turn' | 'end_turn' | 'cancel_turn';
 declare class WaffleError extends Error {
     readonly status?: number | undefined;
     readonly code?: string | undefined;
     readonly stage: WaffleErrorStage;
+    /**
+     * Set only on push-to-talk control-RPC failures (NAR-2321). Lets the UI
+     * telemetry attribute the failure to the exact control method and LiveKit
+     * error code without leaking those raw values into user-facing copy.
+     */
+    rpcMethod?: PushToTalkControlMethod;
+    livekitErrorCode?: number;
     constructor(message: string, status?: number | undefined, code?: string | undefined, stage?: WaffleErrorStage, options?: ErrorOptions);
 }
 declare class WaffleClient implements WaffleSessionClient {
@@ -224,6 +301,7 @@ declare class WaffleClient implements WaffleSessionClient {
     private turnId;
     private nextTurnId;
     private turnControl;
+    private failedStartTurnKey;
     private lifecycleVersion;
     private recreateRoomOnConnect;
     private developerFeedbackGrant;
@@ -241,16 +319,23 @@ declare class WaffleClient implements WaffleSessionClient {
     connect(options?: WaffleConnectOptions): Promise<WaffleSession>;
     disconnect(): Promise<void>;
     sendText(text: string, options?: WaffleSendTextOptions): Promise<void>;
+    /** Switch turn detection without replacing the conversation. */
+    setInputMode(mode: WaffleInputMode): Promise<void>;
+    interrupt(): Promise<void>;
+    get microphoneTrack(): MediaStreamTrack | null;
+    setMicrophoneEnabled(enabled: boolean): Promise<void>;
+    private applicationControl;
     loadDeveloperFeedback(): Promise<WaffleDeveloperFeedbackState>;
     submitDeveloperFeedback(input: WaffleDeveloperFeedbackInput): Promise<WaffleDeveloperFeedbackState>;
-    startPushToTalk(): void;
-    commitPushToTalk(): void;
+    startPushToTalk(): Promise<void>;
+    commitPushToTalk(): Promise<void>;
     /** @deprecated Use commitPushToTalk(). */
     stopPushToTalk(): void;
-    cancelPushToTalk(): void;
+    cancelPushToTalk(): Promise<void>;
     private startInternal;
     private stopInternal;
     private sendInputControl;
+    private performControlRpc;
     private assertPushToTalkAvailable;
     private assertConnected;
     private requestDeveloperFeedback;
@@ -269,4 +354,4 @@ declare class WaffleClient implements WaffleSessionClient {
     private emit;
 }
 
-export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, type WaffleDecline, type WaffleDeveloperFeedbackAction, type WaffleDeveloperFeedbackAvailability, type WaffleDeveloperFeedbackDimension, type WaffleDeveloperFeedbackGrant, type WaffleDeveloperFeedbackGrantProvider, type WaffleDeveloperFeedbackGrantRequest, type WaffleDeveloperFeedbackInput, type WaffleDeveloperFeedbackState, WaffleError, type WaffleErrorStage, type WaffleInputMode, type WaffleSendTextOptions, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript, type WaffleTranscriptProgress };
+export { type CharacterCard, type PushToTalkControlMethod, WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, type WaffleDecline, type WaffleDeveloperFeedbackAction, type WaffleDeveloperFeedbackAvailability, type WaffleDeveloperFeedbackDimension, type WaffleDeveloperFeedbackGrant, type WaffleDeveloperFeedbackGrantProvider, type WaffleDeveloperFeedbackGrantRequest, type WaffleDeveloperFeedbackInput, type WaffleDeveloperFeedbackState, WaffleError, type WaffleErrorStage, type WaffleInputMode, type WaffleSendTextOptions, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript, type WaffleTranscriptProgress };

package/dist/index.js

@@ -1 +1 @@
-import{b as I}from"./chunk-BXRNROI7.js";import{DataPacket_Kind as M,ParticipantKind as P,Room as G,RoomEvent as W,Track as x}from"livekit-client";var D=15e3,U=3e4,N="https://animated-waffle.narya.ai",A=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,K=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,o,s="session",l){super(t,l);this.status=n;this.code=o;this.stage=s;this.name="WaffleError"}status;code;stage};function b(r,e,t){return r instanceof i?r:new i(r instanceof Error?r.message:t,void 0,void 0,e,{cause:r})}function y(r){return r!==null&&typeof r=="object"&&!Array.isArray(r)?r:null}function q(r=N){let e;try{e=new URL(r)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function v(r,e){if(typeof r!="string"||r.trim().length===0)throw new i(`Session response is missing ${e}.`);return r}function L(r){let e=v(r,"avatar.asset_hash");if(!K.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function F(r){return typeof r=="number"&&Number.isFinite(r)?r:void 0}function g(r){return typeof r=="string"&&r.length>0?r:void 0}function C(r,e){let t=v(r,e);try{let n=new URL(t),o=n.hostname==="localhost"||n.hostname==="127.0.0.1"||n.hostname==="[::1]";if(n.protocol!=="https:"&&!(n.protocol==="http:"&&o))throw new Error;if(n.username||n.password||n.hash)throw new Error;return n.toString()}catch{throw new i(`Received an invalid ${e}.`)}}function j(r,e){let t=v(r,e);if(!/^\d{4}-\d{2}-\d{2}T/.test(t)||!Number.isFinite(Date.parse(t)))throw new i(`Received an invalid ${e}.`);return t}function E(r,e){let t=g(r[`${e}.speech_id`]),n=Number(r[`${e}.speech_step_index`]);return t&&Number.isSafeInteger(n)&&n>0?JSON.stringify([t,n]):null}function V(r,e,t,n){let o=y(r);if(!o)throw new i("Session response must be a JSON object.");if(o.protocol_version!==2)throw new i("Session response has an unsupported protocol version.");let s=y(o.connection);if(!s||s.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let l=v(s.url,"connection.url");try{let f=new URL(l),k=f.hostname==="localhost"||f.hostname==="127.0.0.1"||f.hostname==="[::1]";if(f.protocol!=="wss:"&&!(f.protocol==="ws:"&&k))throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let a=v(o.agent_id,"agent_id");if(a!==t)throw new i("Session response contains a mismatched Agent id.");let c=g(o.agent_studio_url);if(c)try{let f=new URL(c),k=f.hostname==="localhost"||f.hostname==="127.0.0.1"||f.hostname==="[::1]";if(f.protocol!=="https:"&&!(f.protocol==="http:"&&k))throw new Error}catch{throw new i("Session response contains an invalid Agent Studio URL.")}let d=g(o.avatar_id),p=y(o.avatar),w=null;if(d||p){if(!d||!p||v(p.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let f=v(p.engine,"avatar.engine");if(f!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let k=v(p.model_url,"avatar.model_url");try{let S=new URL(k),T=S.hostname==="localhost"||S.hostname==="127.0.0.1"||S.hostname==="[::1]";if(S.protocol!=="https:"&&!(S.protocol==="http:"&&T))throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}w={id:d,name:v(p.name,"avatar.name"),engine:f,modelUrl:k,assetHash:L(p.asset_hash),runtimeProfile:v(p.runtime_profile,"avatar.runtime_profile"),...H(p.presentation_profile)}}let h=v(o.session_id,"session_id");return{id:h,agentId:a,...c?{agentStudioUrl:c}:{},...d?{avatarId:d}:{},...n?{developerFeedback:{sourceSessionId:h}}:{},avatar:w,inputMode:e,room:l,token:v(s.token,"connection.token")}}function H(r){if(r===void 0)return{};let e=y(r);if(!e)throw new i("Session response contains an invalid Avatar presentation profile.");let t=v(e.framing,"avatar.presentation_profile.framing"),n=v(e.attention,"avatar.presentation_profile.attention"),o=v(e.emotion,"avatar.presentation_profile.emotion");if(!["default","full","upper","face"].includes(t)||!["natural","focused","lively"].includes(n)||!["natural","subtle","expressive"].includes(o))throw new i("Session response contains an unsupported Avatar presentation profile.");return{presentationProfile:{framing:t,attention:n,emotion:o}}}function B(r){let e=y(r);if(!e||typeof e.type!="string")return null;let t={contextId:g(e.contextId)??null,segmentId:g(e.segmentId),occurrence:F(e.occurrence),audioTime:F(e.audioTime)};if(e.type==="avatar-emotion"){let n=g(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=g(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function $(r){if(!r||typeof r!="object")return null;let e=r;if(e.type!=="agent-declined")return null;let t=g(e.reason);return t?{reason:t}:null}function _(r){let{avatar:e,room:t,token:n,...o}=r;return o}var R=class{room;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;startVersion=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";agentIdentity=null;progressExecutions=new Map;progressText=new Map;completedProgressExecutions=new Set;turnId=null;nextTurnId=0;turnControl=Promise.resolve();lifecycleVersion=0;recreateRoomOnConnect=!1;developerFeedbackGrant=null;developerFeedbackGrantPromise=null;developerFeedbackGrantAbort=null;constructor(e){if(this.options=e,this.apiOrigin=q(e.apiOrigin),!A.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.room=this.createRoom()}createRoom(){let e=new G({adaptiveStream:!0,dynacast:!0,audioCaptureDefaults:{voiceIsolation:!1}});return e.on(W.ParticipantConnected,t=>{e===this.room&&this.handleParticipant(t)}),e.on(W.ParticipantDisconnected,t=>{e!==this.room||t.kind!==P.AGENT||(this.agentIdentity=null,this.currentStatus!=="disconnecting"&&(this.handleDisconnected("The Waffle Agent disconnected."),e.disconnect().catch(n=>{this.emit("error",b(n,"transport","Could not close the Agent room."))})))}),e.on(W.ParticipantAttributesChanged,(t,n)=>{e===this.room&&this.handleParticipant(n)}),e.on(W.TrackSubscribed,(t,n,o)=>{e===this.room&&o.kind===P.AGENT&&t.kind===x.Kind.Audio&&this.setAudioTrack(t.mediaStreamTrack)}),e.on(W.TrackUnsubscribed,(t,n,o)=>{e===this.room&&o.kind===P.AGENT&&t.kind===x.Kind.Audio&&this.setAudioTrack(null)}),e.on(W.DataReceived,(t,n,o,s)=>{if(!(e!==this.room||n?.kind!==P.AGENT||o!==M.RELIABLE)&&!(s!=="avatar.performance"&&s!=="animated-waffle.turn"))try{let l=JSON.parse(new TextDecoder().decode(t));if(s==="animated-waffle.turn"){let c=$(l);c&&this.emit("declined",c);return}let a=B(l);a&&I(this,"avatar",a)}catch{}}),e.on(W.Disconnected,()=>{e===this.room&&this.handleDisconnected("The Agent session disconnected before it was ready.")}),e.registerTextStreamHandler("lk.transcription",(t,n)=>{(async()=>{let o=this.agentIdentity?e.remoteParticipants.get(this.agentIdentity):void 0;if(e!==this.room||this.currentStatus!=="connected"||o?.kind!==P.AGENT)return;let s=t.info.attributes??{},l=g(s["lk.segment_id"]),a=g(s["lk.transcribed_track_id"]),c=s["lk.transcription_final"];if(!l||!a||c!=="true"&&c!=="false")return;let d=e.localParticipant.audioTrackPublications.has(a),p=o.audioTrackPublications.has(a);if(d===p)return;let w=d?"user":"assistant";if(n.identity!==(d?e.localParticipant.identity:o.identity))return;let h=w==="assistant"?E(s,"lk"):null;if(w==="assistant"&&!h||h&&(this.progressExecutions.set(l,h),this.completedProgressExecutions.has(h)))return;let f="";for await(let k of t){if(f+=k,e!==this.room||this.currentStatus!=="connected"||h&&this.completedProgressExecutions.has(h))return;if(f.length>0){let S={segmentId:l,role:w,text:f,complete:!1};this.progressText.set(l,S),this.emit("transcriptProgress",S)}}c==="true"&&f.length>0&&(this.emit("transcriptProgress",{segmentId:l,role:w,text:f,complete:!0}),this.progressExecutions.delete(l),this.progressText.delete(l))})().catch(o=>{this.emit("error",b(o,"transcript","Transcript stream failed."))})}),e.registerTextStreamHandler("animated-waffle.transcript",(t,n)=>{t.readAll().then(o=>{if(e!==this.room||n.identity!==this.agentIdentity||!o)return;let s=t.info.attributes??{},l=g(s["animated-waffle.message_id"]),a=s["animated-waffle.role"];if(!l||a!=="user"&&a!=="assistant"||s["animated-waffle.final"]!=="true")return;let c=a==="assistant"?E(s,"animated-waffle"):null;c&&this.completeTranscriptExecution(c);let d=Number(t.info.timestamp);this.emit("transcript",{id:l,role:a,text:o,final:!0,...Number.isFinite(d)&&d>0?{timestamp:new Date(d).toISOString()}:{}})}).catch(o=>{this.emit("error",b(o,"transcript","Transcript stream failed."))})}),e}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?_(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(e){this.audioMuted=e,this.audio&&(this.audio.muted=e)}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(_(this.currentSession));if(this.startPromise&&this.startVersion===this.lifecycleVersion)return this.startPromise;let t=++this.lifecycleVersion,n=(async()=>{let o=this.stopPromise;if(o&&await o,t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.recreateRoomOnConnect&&(this.room=this.createRoom(),this.recreateRoomOnConnect=!1),this.startInternal(e,t)})().finally(()=>{this.startPromise===n&&(this.startPromise=null,this.startVersion=null)});return this.startVersion=t,this.startPromise=n,n}disconnect(){if(this.lifecycleVersion+=1,this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e,t){if(!e.trim())return;this.assertConnected();let n=t?.audioResponse===!1?{"animated-waffle.audio_response":"false"}:void 0;await this.room.localParticipant.sendText(e,{topic:"lk.chat",...n?{attributes:n}:{}})}async loadDeveloperFeedback(){return this.requestDeveloperFeedback("GET")}async submitDeveloperFeedback(e){let t=e.id??crypto.randomUUID();if(!A.test(t))throw new i("Developer Feedback id must be a UUID.",void 0,"invalid_feedback");if(e.scope==="message"&&!e.targetMessageId)throw new i("Message feedback requires a persisted Assistant message id.",void 0,"invalid_feedback");if(e.scope==="session"&&e.targetMessageId)throw new i("Session feedback cannot target one message.",void 0,"invalid_feedback");if(e.targetMessageId&&!A.test(e.targetMessageId))throw new i("Developer Feedback targetMessageId must be a UUID.",void 0,"invalid_feedback");if(e.action==="change"&&!e.note?.trim()&&!e.preferredReply?.trim()&&e.dimension!=="avatar"&&e.dimension!=="voice")throw new i("Describe the requested change or provide a preferred reply.",void 0,"invalid_feedback");return this.requestDeveloperFeedback("POST",{id:t,scope:e.scope,targetMessageId:e.targetMessageId??null,action:e.action,dimension:e.dimension,note:e.note??null,preferredReply:e.preferredReply??null})}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start_turn")}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("end_turn"),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel_turn"),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let o=e.sessionId??crypto.randomUUID(),s=crypto.randomUUID(),l=this.options.startSession,a=l?void 0:this.options.sessionContext,c=l?void 0:JSON.stringify({chat_session_id:o,input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic",...e.overrides?{overrides:e.overrides}:{},...a?.userName?.trim()?{user_name:a.userName}:{},...a?.timeZone?.trim()?{time_zone:a.timeZone}:{}}),d=()=>this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,requestId:s,sessionId:o,reportedSource:this.options.reportedSource??"SDK",...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(u=>fetch(this.apiOrigin+"/v1/agents/"+encodeURIComponent(this.options.agentId)+"/sessions",{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:"Bearer "+u,"Idempotency-Key":s,"X-Animated-Waffle-Reported-Source":this.options.reportedSource??"SDK"},signal:n.signal,body:c})),p;try{p=await this.withTimeout(l?d():d().catch(u=>{if(n.signal.aborted)throw u;return d()}),D,"Timed out creating the Waffle session.",()=>n.abort())}catch(u){let m=b(u,"session","Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(m),m)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let w=await p.json().catch(()=>null);if(!p.ok){let u=y(w),m=new i(g(u?.detail)??"Could not create session.",p.status,g(u?.code));throw this.fail(m),m}let h;try{h=V(w,this.activeInputMode,this.options.agentId,typeof this.options.developerFeedback=="function")}catch(u){let m=b(u,"session","Could not parse the Waffle session.");throw this.fail(m),m}this.currentSession?.id!==h.id&&this.clearDeveloperFeedbackGrant(),this.currentSession=h,this.emit("session",_(h)),I(this,"presentation",h.avatar),this.setStatus("connecting");let f,k,S=new Promise((u,m)=>{f=u,k=m});this.readyWaiter={reject:k,resolve:f,version:t};let T=this.room;try{if(await T.connect(h.room,h.token).catch(u=>{throw b(u,"transport","Could not connect to the Agent room.")}),await T.startAudio().catch(u=>{throw b(u,"audio","Could not start Agent audio.")}),this.options.inputAudioEnabled!==!1){let u=await T.localParticipant.setMicrophoneEnabled(!0).catch(O=>{throw b(O,"audio","Could not start the microphone.")}),m=u?.audioTrack?.mediaStreamTrack;if(!u||u.isMuted||u.isUpstreamPaused||!m||!m.enabled||m.muted||m.readyState!=="live")throw new i("The microphone track did not publish.",void 0,void 0,"audio")}for(let u of T.remoteParticipants.values())this.handleParticipant(u);await this.withTimeout(S,U,"Timed out waiting for the Waffle Agent.",()=>{T.disconnect().catch(()=>{})},"transport")}catch(u){this.readyWaiter?.version===t&&(this.readyWaiter=null),this.currentStatus!=="disconnected"&&await T.disconnect().catch(()=>{});let m=b(u,"transport","Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(m),m)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.setStatus("connected"),_(h)}async stopInternal(){if(this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.turnControl=Promise.resolve(),this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.room.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected();let t=this.agentIdentity;if(!t)throw new i("The Waffle Agent is not ready.");let n=this.room,o=this.turnControl.then(()=>n.localParticipant.performRpc({destinationIdentity:t,method:e,payload:""}));this.turnControl=o.then(()=>{},s=>{this.emit("error",b(s,"transport","Push-to-talk control failed."))})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}async requestDeveloperFeedback(e,t){if(typeof this.options.developerFeedback!="function")throw new i("Developer Feedback is not enabled for this client.",void 0,"developer_feedback_disabled");let n=await this.resolveDeveloperFeedbackGrant(),o;try{o=await fetch(n.endpoint,{method:e,credentials:"omit",cache:"no-store",headers:{Accept:"application/json",Authorization:`Bearer ${n.token}`,...t?{"Content-Type":"application/json"}:{}},...t?{body:JSON.stringify(t)}:{}})}catch{throw new i("Could not save Developer Feedback.",void 0,"developer_feedback_network_error")}let s=await o.json().catch(()=>null);if(!o.ok){let p=y(s);throw new i(g(p?.detail)??"Could not save Developer Feedback.",o.status,g(p?.code))}let l=y(s),a=l?.round_id,c=l?.pending_count,d=l?.dashboard_url;if(a!==null&&(typeof a!="string"||!A.test(a))||typeof c!="number"||!Number.isSafeInteger(c)||c<0||d!==null&&typeof d!="string")throw new i("Developer Feedback response is invalid.");return typeof d=="string"&&C(d,"dashboard_url"),{roundId:a,pendingCount:c,dashboardUrl:d}}resolveDeveloperFeedbackGrant(){let e=this.currentSession,t=this.options.developerFeedback;if(!e||typeof t!="function")throw new i("Start a Developer Feedback-enabled Waffle session first.",void 0,"developer_feedback_unavailable");if(this.developerFeedbackGrant?.sourceSessionId===e.id&&Date.parse(this.developerFeedbackGrant.expiresAt)>Date.now())return Promise.resolve(this.developerFeedbackGrant);if(this.developerFeedbackGrantPromise)return this.developerFeedbackGrantPromise;let n=new AbortController;this.developerFeedbackGrantAbort=n;let o=this.withTimeout(Promise.resolve(t({agentId:e.agentId,sourceSessionId:e.id,signal:n.signal})),D,"Timed out requesting a Developer Feedback grant.",()=>n.abort()).then(async s=>{let l=await s.json().catch(()=>null);if(!s.ok){let d=y(l);throw new i(g(d?.detail)??"Could not authorize Developer Feedback.",s.status,g(d?.code))}let a=y(l),c={endpoint:C(a?.endpoint,"developerFeedback.endpoint"),token:v(a?.token,"developerFeedback.token"),expiresAt:j(a?.expires_at,"developer_feedback.expires_at"),sourceSessionId:v(a?.source_session_id,"developer_feedback.source_session_id")};if(c.sourceSessionId!==e.id)throw new i("Developer Feedback grant targets a different Session.");if(Date.parse(c.expiresAt)<=Date.now())throw new i("Developer Feedback grant is already expired.");if(this.currentSession?.id!==e.id)throw new i("The Waffle session changed while authorizing feedback.");return this.developerFeedbackGrant=c,c}).finally(()=>{this.developerFeedbackGrantPromise===o&&(this.developerFeedbackGrantPromise=null),this.developerFeedbackGrantAbort===n&&(this.developerFeedbackGrantAbort=null)});return this.developerFeedbackGrantPromise=o,o}handleParticipant(e){if(e.kind===P.AGENT){this.agentIdentity=e.identity,this.emit("agentSpeaking",e.attributes["lk.agent.state"]==="speaking"),e.attributes["animated-waffle.runtime.ready"]==="true"&&this.readyWaiter?.resolve();for(let t of e.audioTrackPublications.values()){let n=t.track;if(n?.kind===x.Kind.Audio){this.setAudioTrack(n.mediaStreamTrack);break}}}}handleDisconnected(e){this.readyWaiter?.reject(new i(e,void 0,void 0,"transport")),this.readyWaiter=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(e===this.currentRemoteAudioTrack||(this.stopAudio(!1),this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),I(this,"audioTrack",e),!e||typeof document>"u")||this.options.audioPlayback===!1)return;let t=document.createElement("audio");t.autoplay=!0,t.muted=this.audioMuted,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",b(n,"audio","Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),I(this,"audioTrack",null))}completeTranscriptExecution(e){this.completedProgressExecutions.add(e);for(let[t,n]of this.progressText)this.progressExecutions.get(t)===e&&(this.emit("transcriptProgress",{...n,complete:!0}),this.progressExecutions.delete(t),this.progressText.delete(t))}clearTranscriptProgress(){this.progressExecutions.clear(),this.progressText.clear(),this.completedProgressExecutions.clear()}setStatus(e){e==="disconnected"&&(this.recreateRoomOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.clearDeveloperFeedbackGrant(),this.currentSession=null,this.emit("session",null),I(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}clearDeveloperFeedbackGrant(){this.developerFeedbackGrantAbort?.abort(),this.developerFeedbackGrantAbort=null,this.developerFeedbackGrantPromise=null,this.developerFeedbackGrant=null}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,o,s="session"){return new Promise((l,a)=>{let c=setTimeout(()=>{o?.(),a(new i(n,void 0,void 0,s))},t);e.then(d=>{clearTimeout(c),l(d)},d=>{clearTimeout(c),a(d)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{R as WaffleClient,i as WaffleError};
+import{b as _}from"./chunk-BXRNROI7.js";import{DataPacket_Kind as U,ParticipantKind as W,Room as G,RoomEvent as P,RpcError as M,Track as C}from"livekit-client";var D=15e3,K=3e4,q="https://animated-waffle.narya.ai",A=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,L=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,o,a="session",s){super(t,s);this.status=n;this.code=o;this.stage=a;this.name="WaffleError"}status;code;stage;rpcMethod;livekitErrorCode},I={APPLICATION_ERROR:1500,CONNECTION_TIMEOUT:1501,RESPONSE_TIMEOUT:1502,RECIPIENT_DISCONNECTED:1503},j=new Set([I.CONNECTION_TIMEOUT,I.RESPONSE_TIMEOUT]);function V(r){return r instanceof M&&j.has(r.code)}function H(r){return r===I.RECIPIENT_DISCONNECTED?"push_to_talk_agent_unavailable":r===I.CONNECTION_TIMEOUT||r===I.RESPONSE_TIMEOUT?"push_to_talk_timeout":"push_to_talk_failed"}function B(r){return r===I.RECIPIENT_DISCONNECTED?"The Waffle agent dropped out for a moment. Release and press again to talk.":r===I.CONNECTION_TIMEOUT||r===I.RESPONSE_TIMEOUT?"The voice turn took too long to reach the agent. Release and try again.":"That voice turn didn\u2019t go through. Press and hold to talk again."}function $(r,e){let t=r instanceof M?r.code:void 0,n=new i(B(t),void 0,H(t),"transport",{cause:r});return n.rpcMethod=e,typeof t=="number"&&(n.livekitErrorCode=t),n}function T(r,e,t){return r instanceof i?r:new i(r instanceof Error?r.message:t,void 0,void 0,e,{cause:r})}function S(r){return r!==null&&typeof r=="object"&&!Array.isArray(r)?r:null}function J(r=q){let e;try{e=new URL(r)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function v(r,e){if(typeof r!="string"||r.trim().length===0)throw new i(`Session response is missing ${e}.`);return r}function z(r){let e=v(r,"avatar.asset_hash");if(!L.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function x(r){return typeof r=="number"&&Number.isFinite(r)?r:void 0}function g(r){return typeof r=="string"&&r.length>0?r:void 0}function R(r,e){let t=v(r,e);try{let n=new URL(t),o=n.hostname==="localhost"||n.hostname==="127.0.0.1"||n.hostname==="[::1]";if(n.protocol!=="https:"&&!(n.protocol==="http:"&&o))throw new Error;if(n.username||n.password||n.hash)throw new Error;return n.toString()}catch{throw new i(`Received an invalid ${e}.`)}}function Z(r,e){let t=v(r,e);if(!/^\d{4}-\d{2}-\d{2}T/.test(t)||!Number.isFinite(Date.parse(t)))throw new i(`Received an invalid ${e}.`);return t}function O(r,e){let t=g(r[`${e}.speech_id`]),n=Number(r[`${e}.speech_step_index`]);return t&&Number.isSafeInteger(n)&&n>0?JSON.stringify([t,n]):null}function Y(r,e,t,n){let o=S(r);if(!o)throw new i("Session response must be a JSON object.");if(o.protocol_version!==2)throw new i("Session response has an unsupported protocol version.");let a=S(o.connection);if(!a||a.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let s=v(a.url,"connection.url");try{let p=new URL(s),k=p.hostname==="localhost"||p.hostname==="127.0.0.1"||p.hostname==="[::1]";if(p.protocol!=="wss:"&&!(p.protocol==="ws:"&&k))throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let l=v(o.agent_id,"agent_id");if(l!==t)throw new i("Session response contains a mismatched Agent id.");let c=g(o.agent_studio_url);if(c)try{let p=new URL(c),k=p.hostname==="localhost"||p.hostname==="127.0.0.1"||p.hostname==="[::1]";if(p.protocol!=="https:"&&!(p.protocol==="http:"&&k))throw new Error}catch{throw new i("Session response contains an invalid Agent Studio URL.")}let d=g(o.avatar_id),f=S(o.avatar),b=null;if(d||f){if(!d||!f||v(f.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let p=v(f.engine,"avatar.engine");if(p!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let k=v(f.model_url,"avatar.model_url");try{let w=new URL(k),y=w.hostname==="localhost"||w.hostname==="127.0.0.1"||w.hostname==="[::1]";if(w.protocol!=="https:"&&!(w.protocol==="http:"&&y))throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}b={id:d,name:v(f.name,"avatar.name"),engine:p,modelUrl:k,assetHash:z(f.asset_hash),runtimeProfile:v(f.runtime_profile,"avatar.runtime_profile"),...X(f.presentation_profile)}}let h=v(o.session_id,"session_id");return{id:h,agentId:l,...c?{agentStudioUrl:c}:{},...d?{avatarId:d}:{},...n?{developerFeedback:{sourceSessionId:h}}:{},avatar:b,inputMode:e,room:s,token:v(a.token,"connection.token")}}function X(r){if(r===void 0)return{};let e=S(r);if(!e)throw new i("Session response contains an invalid Avatar presentation profile.");let t=v(e.framing,"avatar.presentation_profile.framing"),n=v(e.attention,"avatar.presentation_profile.attention"),o=v(e.emotion,"avatar.presentation_profile.emotion");if(!["default","full","upper","face"].includes(t)||!["natural","focused","lively"].includes(n)||!["natural","subtle","expressive"].includes(o))throw new i("Session response contains an unsupported Avatar presentation profile.");return{presentationProfile:{framing:t,attention:n,emotion:o}}}function Q(r){let e=S(r);if(!e||typeof e.type!="string")return null;let t={contextId:g(e.contextId)??null,segmentId:g(e.segmentId),occurrence:x(e.occurrence),audioTime:x(e.audioTime)};if(e.type==="avatar-emotion"){let n=g(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=g(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function ee(r){if(!r||typeof r!="object")return null;let e=r;if(e.type!=="agent-declined")return null;let t=g(e.reason);return t?{reason:t}:null}function E(r){let{avatar:e,room:t,token:n,...o}=r;return o}var F=class{room;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;startVersion=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";agentIdentity=null;progressExecutions=new Map;progressText=new Map;completedProgressExecutions=new Set;turnId=null;nextTurnId=0;turnControl=Promise.resolve();failedStartTurnKey=null;lifecycleVersion=0;recreateRoomOnConnect=!1;developerFeedbackGrant=null;developerFeedbackGrantPromise=null;developerFeedbackGrantAbort=null;constructor(e){if(this.options=e,this.apiOrigin=J(e.apiOrigin),!A.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.room=this.createRoom()}createRoom(){let e=new G({adaptiveStream:!0,dynacast:!0,audioCaptureDefaults:{voiceIsolation:!1}});return e.on(P.ParticipantConnected,t=>{e===this.room&&this.handleParticipant(t)}),e.on(P.ParticipantDisconnected,t=>{e!==this.room||t.kind!==W.AGENT||(this.agentIdentity=null,this.currentStatus!=="disconnecting"&&(this.handleDisconnected("The Waffle Agent disconnected."),e.disconnect().catch(n=>{this.emit("error",T(n,"transport","Could not close the Agent room."))})))}),e.on(P.ParticipantAttributesChanged,(t,n)=>{e===this.room&&this.handleParticipant(n)}),e.on(P.TrackSubscribed,(t,n,o)=>{e===this.room&&o.kind===W.AGENT&&t.kind===C.Kind.Audio&&this.setAudioTrack(t.mediaStreamTrack)}),e.on(P.TrackUnsubscribed,(t,n,o)=>{e===this.room&&o.kind===W.AGENT&&t.kind===C.Kind.Audio&&this.setAudioTrack(null)}),e.on(P.DataReceived,(t,n,o,a)=>{if(!(e!==this.room||n?.kind!==W.AGENT||o!==U.RELIABLE)&&!(a!=="avatar.performance"&&a!=="animated-waffle.turn"))try{let s=JSON.parse(new TextDecoder().decode(t));if(a==="animated-waffle.turn"){let c=ee(s);c&&this.emit("declined",c);return}let l=Q(s);l&&_(this,"avatar",l)}catch{}}),e.on(P.Disconnected,()=>{e===this.room&&this.handleDisconnected("The Agent session disconnected before it was ready.")}),e.registerTextStreamHandler("lk.transcription",(t,n)=>{(async()=>{let o=this.agentIdentity?e.remoteParticipants.get(this.agentIdentity):void 0;if(e!==this.room||this.currentStatus!=="connected"||o?.kind!==W.AGENT)return;let a=t.info.attributes??{},s=g(a["lk.segment_id"]),l=g(a["lk.transcribed_track_id"]),c=a["lk.transcription_final"];if(!s||!l||c!=="true"&&c!=="false")return;let d=e.localParticipant.audioTrackPublications.has(l),f=o.audioTrackPublications.has(l);if(d===f)return;let b=d?"user":"assistant";if(n.identity!==(d?e.localParticipant.identity:o.identity))return;let h=b==="assistant"?O(a,"lk"):null;if(b==="assistant"&&!h||h&&(this.progressExecutions.set(s,h),this.completedProgressExecutions.has(h)))return;let p="";for await(let k of t){if(p+=k,e!==this.room||this.currentStatus!=="connected"||h&&this.completedProgressExecutions.has(h))return;if(p.length>0){let w={segmentId:s,role:b,text:p,complete:!1};this.progressText.set(s,w),this.emit("transcriptProgress",w)}}c==="true"&&p.length>0&&(this.emit("transcriptProgress",{segmentId:s,role:b,text:p,complete:!0}),this.progressExecutions.delete(s),this.progressText.delete(s))})().catch(o=>{this.emit("error",T(o,"transcript","Transcript stream failed."))})}),e.registerTextStreamHandler("animated-waffle.transcript",(t,n)=>{t.readAll().then(o=>{if(e!==this.room||n.identity!==this.agentIdentity||!o)return;let a=t.info.attributes??{},s=g(a["animated-waffle.message_id"]),l=a["animated-waffle.role"];if(!s||l!=="user"&&l!=="assistant"||a["animated-waffle.final"]!=="true")return;let c=l==="assistant"?O(a,"animated-waffle"):null;c&&this.completeTranscriptExecution(c);let d=Number(t.info.timestamp);this.emit("transcript",{id:s,role:l,text:o,final:!0,...Number.isFinite(d)&&d>0?{timestamp:new Date(d).toISOString()}:{}})}).catch(o=>{this.emit("error",T(o,"transcript","Transcript stream failed."))})}),e}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?E(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(e){this.audioMuted=e,this.audio&&(this.audio.muted=e)}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(E(this.currentSession));if(this.startPromise&&this.startVersion===this.lifecycleVersion)return this.startPromise;let t=++this.lifecycleVersion,n=(async()=>{let o=this.stopPromise;if(o&&await o,t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.recreateRoomOnConnect&&(this.room=this.createRoom(),this.recreateRoomOnConnect=!1),this.startInternal(e,t)})().finally(()=>{this.startPromise===n&&(this.startPromise=null,this.startVersion=null)});return this.startVersion=t,this.startPromise=n,n}disconnect(){if(this.lifecycleVersion+=1,this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e,t){if(!e.trim())return;this.assertConnected();let n=t?.audioResponse===!1?{"animated-waffle.audio_response":"false"}:void 0;await this.room.localParticipant.sendText(e,{topic:"lk.chat",...n?{attributes:n}:{}})}async setInputMode(e){await this.applicationControl("input_mode",{mode:e}),this.activeInputMode=e,this.turnId=null}async interrupt(){await this.applicationControl("interrupt",{})}get microphoneTrack(){return this.room.localParticipant.getTrackPublication(C.Source.Microphone)?.audioTrack?.mediaStreamTrack??null}async setMicrophoneEnabled(e){if(this.assertConnected(),e&&this.options.inputAudioEnabled===!1)throw new i("Input audio is disabled.",void 0,"input_audio_disabled");await this.room.localParticipant.setMicrophoneEnabled(e)}async applicationControl(e,t){if(this.assertConnected(),!this.options.application||!this.agentIdentity)throw new i("Application sessions are required.",void 0,"application_session_required");return this.room.localParticipant.performRpc({destinationIdentity:this.agentIdentity,method:"application.control",payload:JSON.stringify({action:e,...t})})}async loadDeveloperFeedback(){return this.requestDeveloperFeedback("GET")}async submitDeveloperFeedback(e){let t=e.id??crypto.randomUUID();if(!A.test(t))throw new i("Developer Feedback id must be a UUID.",void 0,"invalid_feedback");if(e.scope==="message"&&!e.targetMessageId)throw new i("Message feedback requires a persisted Assistant message id.",void 0,"invalid_feedback");if(e.scope==="session"&&e.targetMessageId)throw new i("Session feedback cannot target one message.",void 0,"invalid_feedback");if(e.targetMessageId&&!A.test(e.targetMessageId))throw new i("Developer Feedback targetMessageId must be a UUID.",void 0,"invalid_feedback");if(e.action==="change"&&!e.note?.trim()&&!e.preferredReply?.trim()&&e.dimension!=="avatar"&&e.dimension!=="voice")throw new i("Describe the requested change or provide a preferred reply.",void 0,"invalid_feedback");return this.requestDeveloperFeedback("POST",{id:t,scope:e.scope,targetMessageId:e.targetMessageId??null,action:e.action,dimension:e.dimension,note:e.note??null,preferredReply:e.preferredReply??null})}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId!==null)return this.turnControl;this.nextTurnId+=1,this.turnId=this.nextTurnId;try{return this.sendInputControl("start_turn",String(this.turnId))}catch(e){throw this.turnId=null,e}}commitPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null)return this.turnControl;let e=String(this.turnId);return this.turnId=null,this.sendInputControl("end_turn",e)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null)return this.turnControl;let e=String(this.turnId);return this.turnId=null,this.sendInputControl("cancel_turn",e)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let o=e.sessionId??crypto.randomUUID(),a=crypto.randomUUID(),s=this.options.startSession,l=s?void 0:this.options.sessionContext,c=s?void 0:JSON.stringify({chat_session_id:o,input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic",...e.overrides?{overrides:e.overrides}:{},...l?.userName?.trim()?{user_name:l.userName}:{},...l?.timeZone?.trim()?{time_zone:l.timeZone}:{}}),d=()=>this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,requestId:a,sessionId:o,reportedSource:this.options.reportedSource??"SDK",...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(u=>fetch(this.apiOrigin+"/v1/agents/"+encodeURIComponent(this.options.agentId)+"/sessions",{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:"Bearer "+u,"Idempotency-Key":a,"X-Animated-Waffle-Reported-Source":this.options.reportedSource??"SDK"},signal:n.signal,body:c})),f;try{f=await this.withTimeout(s?d():d().catch(u=>{if(n.signal.aborted)throw u;return d()}),D,"Timed out creating the Waffle session.",()=>n.abort())}catch(u){let m=T(u,"session","Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(m),m)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let b=await f.json().catch(()=>null);if(!f.ok){let u=S(b),m=new i(g(u?.detail)??"Could not create session.",f.status,g(u?.code));throw this.fail(m),m}let h;try{h=Y(b,this.activeInputMode,this.options.agentId,typeof this.options.developerFeedback=="function")}catch(u){let m=T(u,"session","Could not parse the Waffle session.");throw this.fail(m),m}this.currentSession?.id!==h.id&&this.clearDeveloperFeedbackGrant(),this.currentSession=h,this.emit("session",E(h)),_(this,"presentation",h.avatar),this.setStatus("connecting");let p,k,w=new Promise((u,m)=>{p=u,k=m});this.readyWaiter={reject:k,resolve:p,version:t};let y=this.room;try{if(await y.connect(h.room,h.token).catch(u=>{throw T(u,"transport","Could not connect to the Agent room.")}),await y.startAudio().catch(u=>{throw T(u,"audio","Could not start Agent audio.")}),this.options.inputAudioEnabled!==!1){let u=await y.localParticipant.setMicrophoneEnabled(!0).catch(N=>{throw T(N,"audio","Could not start the microphone.")}),m=u?.audioTrack?.mediaStreamTrack;if(!u||u.isMuted||u.isUpstreamPaused||!m||!m.enabled||m.muted||m.readyState!=="live")throw new i("The microphone track did not publish.",void 0,void 0,"audio")}for(let u of y.remoteParticipants.values())this.handleParticipant(u);await this.withTimeout(w,K,"Timed out waiting for the Waffle Agent.",()=>{y.disconnect().catch(()=>{})},"transport")}catch(u){this.readyWaiter?.version===t&&(this.readyWaiter=null),this.currentStatus!=="disconnected"&&await y.disconnect().catch(()=>{});let m=T(u,"transport","Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(m),m)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.setStatus("connected"),E(h)}async stopInternal(){if(this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.turnControl=Promise.resolve(),this.failedStartTurnKey=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.room.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected();let n=this.agentIdentity;if(!n)throw new i("The Waffle Agent is not ready.",void 0,"push_to_talk_agent_unavailable","transport");let o=this.room,a=this.turnControl.then(()=>{if(e!=="start_turn"&&this.failedStartTurnKey===t){this.failedStartTurnKey=null;return}return this.performControlRpc(o,n,e,t)});return this.turnControl=a.then(()=>{},s=>{e==="start_turn"&&(this.failedStartTurnKey=t),this.turnId!==null&&String(this.turnId)===t&&(this.turnId=null),this.emit("error",$(s,e))}),this.turnControl}async performControlRpc(e,t,n,o){let a=JSON.stringify({turnId:o}),s=n==="start_turn"?2:1,l;for(let c=1;c<=s;c+=1)try{return await e.localParticipant.performRpc({destinationIdentity:t,method:n,payload:a})}catch(d){if(l=d,c<s&&V(d))continue;throw d}throw l}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}async requestDeveloperFeedback(e,t){if(typeof this.options.developerFeedback!="function")throw new i("Developer Feedback is not enabled for this client.",void 0,"developer_feedback_disabled");let n=await this.resolveDeveloperFeedbackGrant(),o;try{o=await fetch(n.endpoint,{method:e,credentials:"omit",cache:"no-store",headers:{Accept:"application/json",Authorization:`Bearer ${n.token}`,...t?{"Content-Type":"application/json"}:{}},...t?{body:JSON.stringify(t)}:{}})}catch{throw new i("Could not save Developer Feedback.",void 0,"developer_feedback_network_error")}let a=await o.json().catch(()=>null);if(!o.ok){let f=S(a);throw new i(g(f?.detail)??"Could not save Developer Feedback.",o.status,g(f?.code))}let s=S(a),l=s?.round_id,c=s?.pending_count,d=s?.dashboard_url;if(l!==null&&(typeof l!="string"||!A.test(l))||typeof c!="number"||!Number.isSafeInteger(c)||c<0||d!==null&&typeof d!="string")throw new i("Developer Feedback response is invalid.");return typeof d=="string"&&R(d,"dashboard_url"),{roundId:l,pendingCount:c,dashboardUrl:d}}resolveDeveloperFeedbackGrant(){let e=this.currentSession,t=this.options.developerFeedback;if(!e||typeof t!="function")throw new i("Start a Developer Feedback-enabled Waffle session first.",void 0,"developer_feedback_unavailable");if(this.developerFeedbackGrant?.sourceSessionId===e.id&&Date.parse(this.developerFeedbackGrant.expiresAt)>Date.now())return Promise.resolve(this.developerFeedbackGrant);if(this.developerFeedbackGrantPromise)return this.developerFeedbackGrantPromise;let n=new AbortController;this.developerFeedbackGrantAbort=n;let o=this.withTimeout(Promise.resolve(t({agentId:e.agentId,sourceSessionId:e.id,signal:n.signal})),D,"Timed out requesting a Developer Feedback grant.",()=>n.abort()).then(async a=>{let s=await a.json().catch(()=>null);if(!a.ok){let d=S(s);throw new i(g(d?.detail)??"Could not authorize Developer Feedback.",a.status,g(d?.code))}let l=S(s),c={endpoint:R(l?.endpoint,"developerFeedback.endpoint"),token:v(l?.token,"developerFeedback.token"),expiresAt:Z(l?.expires_at,"developer_feedback.expires_at"),sourceSessionId:v(l?.source_session_id,"developer_feedback.source_session_id")};if(c.sourceSessionId!==e.id)throw new i("Developer Feedback grant targets a different Session.");if(Date.parse(c.expiresAt)<=Date.now())throw new i("Developer Feedback grant is already expired.");if(this.currentSession?.id!==e.id)throw new i("The Waffle session changed while authorizing feedback.");return this.developerFeedbackGrant=c,c}).finally(()=>{this.developerFeedbackGrantPromise===o&&(this.developerFeedbackGrantPromise=null),this.developerFeedbackGrantAbort===n&&(this.developerFeedbackGrantAbort=null)});return this.developerFeedbackGrantPromise=o,o}handleParticipant(e){if(e.kind===W.AGENT){if(this.agentIdentity=e.identity,this.emit("agentSpeaking",e.attributes["lk.agent.state"]==="speaking"),e.attributes["animated-waffle.runtime.ready"]==="true"){if(this.options.application&&e.attributes["animated-waffle.application.version"]!=="2"){this.readyWaiter?.reject(new i("The voice service does not support application sessions.",void 0,"application_session_unsupported"));return}this.readyWaiter?.resolve()}for(let t of e.audioTrackPublications.values()){let n=t.track;if(n?.kind===C.Kind.Audio){this.setAudioTrack(n.mediaStreamTrack);break}}}}handleDisconnected(e){this.readyWaiter?.reject(new i(e,void 0,void 0,"transport")),this.readyWaiter=null,this.turnId=null,this.turnControl=Promise.resolve(),this.failedStartTurnKey=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(e===this.currentRemoteAudioTrack||(this.stopAudio(!1),this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),_(this,"audioTrack",e),!e||typeof document>"u")||this.options.audioPlayback===!1)return;let t=document.createElement("audio");t.autoplay=!0,t.muted=this.audioMuted,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",T(n,"audio","Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),_(this,"audioTrack",null))}completeTranscriptExecution(e){this.completedProgressExecutions.add(e);for(let[t,n]of this.progressText)this.progressExecutions.get(t)===e&&(this.emit("transcriptProgress",{...n,complete:!0}),this.progressExecutions.delete(t),this.progressText.delete(t))}clearTranscriptProgress(){this.progressExecutions.clear(),this.progressText.clear(),this.completedProgressExecutions.clear()}setStatus(e){e==="disconnected"&&(this.recreateRoomOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.clearDeveloperFeedbackGrant(),this.currentSession=null,this.emit("session",null),_(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}clearDeveloperFeedbackGrant(){this.developerFeedbackGrantAbort?.abort(),this.developerFeedbackGrantAbort=null,this.developerFeedbackGrantPromise=null,this.developerFeedbackGrant=null}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,o,a="session"){return new Promise((s,l)=>{let c=setTimeout(()=>{o?.(),l(new i(n,void 0,void 0,a))},t);e.then(d=>{clearTimeout(c),s(d)},d=>{clearTimeout(c),l(d)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{F as WaffleClient,i as WaffleError};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.3.5",
+  "version": "0.4.0",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,
@@ -34,12 +34,14 @@
     "access": "public"
   },
   "dependencies": {
-    "livekit-client": "2.21.0"
+    "livekit-client": "2.21.0",
+    "zod": "^4.3.6"
   },
   "devDependencies": {
     "tsup": "^8.5.0",
     "typescript": "^5.7.2",
-    "vitest": "^3.0.5"
+    "vitest": "^3.0.5",
+    "@companion-demo/schema": "0.0.0"
   },
   "scripts": {
     "build": "tsup",