@animated-waffle/client 0.2.6 → 0.3.0

package/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## 0.3.0
+
+- Require managed Agent session protocol v2 and connect to LiveKit WebRTC.
+- Add ephemeral `transcriptProgress` captions while keeping `transcript` as completed persisted messages.
+- Preserve the public connection, text, push-to-talk, audio-track, and reconnect lifecycle without transport-specific types.
+- Breaking: remove `openaiModel` and `openaiVoice` from managed `WaffleSessionOverrides`; the managed SDK is provider-neutral. The separate Live Conversation OpenAI Realtime implementation is unchanged.
+
+0.3.0 must be published before a managed backend is cut over to protocol v2.

package/README.md

@@ -6,7 +6,18 @@ transcripts. React applications should use
 [`@animated-waffle/react`](../waffle-react), which re-exports this entire API
 and adds the Avatar renderer.
 
-The Agent owns its avatar, voice, and prompt. Pipecat, Daily, API routes, and
+Install the current npm release without copying a mutable registry version:
+
+```bash
+pnpm add @animated-waffle/client@latest
+```
+
+The workspace [`package.json`](./package.json) defines the 0.3.0 candidate;
+npm publication is pending. That candidate requires managed Agent session
+protocol v2 and explicitly rejects the retired v1 grant. Publish the coordinated
+SDK and update consumers before cutting the managed backend over.
+
+The Agent owns its avatar, voice, and prompt. Media runtime, API routes, and
 avatar asset URLs are SDK implementation details. Production is the default
 API origin; tests and non-production integrations may pass `apiOrigin`.
 
@@ -40,6 +51,9 @@ waffle.on('transcript', ({ id, role, text, final }) => {
   // Completed Narya conversation rows use the same id as persisted history.
   if (final) console.log(id, role, text)
 })
+waffle.on('transcriptProgress', ({ segmentId, role, text, complete }) => {
+  // Ephemeral ASR/playout caption; never persist segmentId as a message id.
+})
 
 await waffle.connect()
 ```
@@ -59,7 +73,8 @@ const waffle = new WaffleClient({
 
 ## API surface
 
-- `connect({ inputMode? })` — `'push-to-talk'` (default) or `'automatic'`.
+- `connect({ inputMode?, sessionId?, overrides? })` — `'push-to-talk'`
+  (default) or `'automatic'`; resolves only when the Agent runtime is ready.
 - `inputAudioEnabled: false` — never requests or enables the browser microphone;
   text input, Agent audio, transcripts, and reconnects continue to work. Push-
   to-talk controls fail closed with `input_audio_disabled`. The default is
@@ -70,13 +85,20 @@ const waffle = new WaffleClient({
   `remoteAudioTrack` yourself; an application that routes the voice through its
   own element, mixer or Web Audio graph must turn this off, or the voice is
   heard twice. `muted` then has nothing to do.
-- `disconnect()`, `sendText(text)`.
+- `disconnect()`, `sendText(text, { audioResponse? })`. `audioResponse` defaults
+  to `true`; `false` keeps the typed turn and transcript but suppresses reply
+  audio for that turn.
 - `startPushToTalk()` / `commitPushToTalk()` / `cancelPushToTalk()`.
-- `on('status' | 'session' | 'transcript' | 'agentSpeaking' | 'error', fn)` —
+- `on('connectionState' | 'session' | 'remoteAudioTrack' | 'transcriptProgress' | 'transcript' |
+  'agentSpeaking' | 'error', fn)` —
   returns the unsubscribe function.
-- Conversation transcripts carry the same required `id` as persisted history.
-  Provider sentence/progress framing never defines a public conversation row.
-- `status`, `session` getters; failures throw `WaffleError` with `code`.
+- Conversation transcripts are completed persisted rows, carry the same required
+  `id` as history, and currently always have `final: true`. Provider
+  sentence/progress framing never defines a public conversation row.
+- Transcript progress is ephemeral and keyed by a runtime `segmentId`; user ASR
+  and assistant playout update it incrementally until `complete` closes it.
+- `connectionState`, `session`, `remoteAudioTrack` getters; `status` is the
+  deprecated 0.1 alias. Failures throw `WaffleError` with `code`.
 
 `./internal` is a private contract consumed by `@animated-waffle/react`; it is
 not part of the public API and may change without notice.

package/dist/index.d.ts

@@ -13,14 +13,19 @@ interface WaffleTranscript {
     id: string;
     role: 'user' | 'assistant';
     text: string;
-    /**
-     * False while this text is still provisional. Only interim speech recognition
-     * is provisional: a caller replaces the previous non-final entry from that
-     * speaker rather than appending. Agent text is always final.
-     */
+    /** Currently always true: transcript events are completed persisted rows. */
     final: boolean;
     timestamp?: string;
 }
+/** Session-scoped caption progress. It is never a persisted conversation row. */
+interface WaffleTranscriptProgress {
+    /** Runtime-owned segment identity; not a database message id. */
+    segmentId: string;
+    role: 'user' | 'assistant';
+    text: string;
+    /** The runtime closed this ephemeral segment. */
+    complete: boolean;
+}
 interface WaffleClientEvents {
     connectionState: WaffleConnectionState;
     /** @deprecated Subscribe to connectionState. */
@@ -29,6 +34,7 @@ interface WaffleClientEvents {
     error: Error;
     agentSpeaking: boolean;
     remoteAudioTrack: MediaStreamTrack | null;
+    transcriptProgress: WaffleTranscriptProgress;
     transcript: WaffleTranscript;
 }
 type WaffleSessionToken = () => string | Promise<string>;
@@ -49,8 +55,6 @@ interface WaffleSessionOverrides {
     defaultDeviceProfileId?: string | null;
     avatarId?: string;
     avatarEnabled?: boolean;
-    openaiModel?: string | null;
-    openaiVoice?: string;
 }
 interface WaffleClientBaseOptions {
     /** The published Agent to connect to. */
@@ -81,13 +85,9 @@ interface WaffleManagedClientOptions extends WaffleClientBaseOptions {
 type WaffleClientOptions = WaffleTokenClientOptions | WaffleManagedClientOptions;
 interface WaffleSendTextOptions {
     /**
-     * Whether the Agent speaks its reply.
-     *
-     * Defaults to true, which is the only behavior this method had. Passing
-     * false makes one turn text-only: the reply still streams as transcript and
-     * still lands in the conversation context, but no audio is synthesized —
-     * what a typed exchange inside an otherwise spoken session wants, and what a
-     * message thread wants for every turn.
+     * Whether the Agent speaks its reply. Defaults to true. When false, the
+     * reply still streams as transcript and is persisted in the conversation,
+     * but no audio is synthesized.
      */
     readonly audioResponse?: boolean;
 }
@@ -135,8 +135,7 @@ declare class WaffleError extends Error {
     constructor(message: string, status?: number | undefined, code?: string | undefined);
 }
 declare class WaffleClient implements WaffleSessionClient {
-    private realtime;
-    private readonly realtimeCallbacks;
+    private room;
     private readonly listeners;
     private readonly options;
     private readonly apiOrigin;
@@ -150,12 +149,17 @@ declare class WaffleClient implements WaffleSessionClient {
     private audioMuted;
     private currentRemoteAudioTrack;
     private activeInputMode;
+    private agentIdentity;
+    private readonly progressExecutions;
+    private readonly progressText;
+    private readonly completedProgressExecutions;
     private turnId;
     private nextTurnId;
+    private turnControl;
     private lifecycleVersion;
-    private recreateRealtimeOnConnect;
+    private recreateRoomOnConnect;
     constructor(options: WaffleClientOptions);
-    private createRealtime;
+    private createRoom;
     get status(): WaffleStatus;
     get connectionState(): WaffleConnectionState;
     get session(): WaffleSession | null;
@@ -176,11 +180,12 @@ declare class WaffleClient implements WaffleSessionClient {
     private sendInputControl;
     private assertPushToTalkAvailable;
     private assertConnected;
-    private handleTransportState;
-    private handleReady;
+    private handleParticipant;
     private handleDisconnected;
     private setAudioTrack;
     private stopAudio;
+    private completeTranscriptExecution;
+    private clearTranscriptProgress;
     private setStatus;
     private fail;
     private resolveSessionToken;
@@ -188,4 +193,4 @@ declare class WaffleClient implements WaffleSessionClient {
     private emit;
 }
 
-export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSendTextOptions, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript };
+export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSendTextOptions, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript, type WaffleTranscriptProgress };

package/dist/index.js

@@ -1 +1 @@
-import{b as m}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,b="https://animated-waffle.narya.ai",I=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,P=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,r){super(t);this.status=n;this.code=r;this.name="WaffleError"}status;code};function h(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function A(s=b){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function C(s){let e=u(s,"avatar.asset_hash");if(!P.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function f(s){return typeof s=="string"&&s.length>0?s:void 0}function _(s,e,t){let n=h(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let r=h(n.connection);if(!r||r.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let p=u(r.url,"connection.url");try{if(new URL(p).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=f(n.avatar_id),l=h(n.avatar),v=null;if(d||l){if(!d||!l||u(l.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(l.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let a=u(l.model_url,"avatar.model_url");try{if(new URL(a).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(l.name,"avatar.name"),engine:o,modelUrl:a,assetHash:C(l.asset_hash),runtimeProfile:u(l.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:p,token:u(r.token,"connection.token")}}function O(s){let e=h(s);if(!e||typeof e.type!="string")return null;let t={contextId:f(e.contextId)??null,segmentId:f(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime)};if(e.type==="avatar-emotion"){let n=f(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=f(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function E(s){let e=h(s),t=f(e?.messageId),n=e?.role;return e?.type!=="conversation-transcript"||!t||n!=="user"&&n!=="assistant"||typeof e.text!="string"||e.final!==!0?null:{id:t,role:n,text:e.text,final:!0}}function S(s){let{avatar:e,room:t,token:n,...r}=s;return r}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=A(e.apiOrigin),!I.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onServerMessage:t=>{let n=E(t);n&&this.emit("transcript",n);let r=O(t);r&&m(this,"avatar",r)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(h(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:this.options.inputAudioEnabled===!1?"none":"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(e){this.audioMuted=e,this.audio&&(this.audio.muted=e)}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e,t){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:t?.audioResponse??!0,run_immediately:!0}))}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let r;try{r=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let a=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let p=await r.json().catch(()=>null);if(!r.ok){let o=h(p),a=new i(f(o?.detail)??"Could not create session.",r.status,f(o?.code));throw this.fail(a),a}let c;try{c=_(p,this.activeInputMode,this.options.agentId)}catch(o){let a=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(a),a}this.currentSession=c,this.emit("session",S(c)),m(this,"presentation",c.avatar),this.setStatus("connecting");let d,l,v=new Promise((o,a)=>{d=o,l=a});this.readyWaiter={reject:l,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let a=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(a),a)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.options.inputAudioEnabled!==!1&&this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),m(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u"||this.options.audioPlayback===!1)return;let t=document.createElement("audio");t.autoplay=!0,t.muted=this.audioMuted,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),m(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),m(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,r){return new Promise((p,c)=>{let d=setTimeout(()=>{r?.(),c(new i(n))},t);e.then(l=>{clearTimeout(d),p(l)},l=>{clearTimeout(d),c(l)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};
+import{b as k}from"./chunk-BXRNROI7.js";import{DataPacket_Kind as _,ParticipantKind as y,Room as x,RoomEvent as w,Track as b}from"livekit-client";var C=15e3,R=3e4,O="https://animated-waffle.narya.ai",M=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,N=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(e,n,s){super(e);this.status=n;this.code=s;this.name="WaffleError"}status;code};function I(o){return o!==null&&typeof o=="object"&&!Array.isArray(o)?o:null}function U(o=O){let t;try{t=new URL(o)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let e=t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="[::1]";if(t.protocol!=="https:"&&!(t.protocol==="http:"&&e)||t.username||t.password||t.pathname!=="/"&&t.pathname!==""||t.search||t.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return t.origin}function v(o,t){if(typeof o!="string"||o.trim().length===0)throw new i(`Session response is missing ${t}.`);return o}function D(o){let t=v(o,"avatar.asset_hash");if(!N.test(t))throw new i("Session response contains an invalid Avatar asset hash.");return t}function W(o){return typeof o=="number"&&Number.isFinite(o)?o:void 0}function g(o){return typeof o=="string"&&o.length>0?o:void 0}function A(o,t){let e=g(o[`${t}.speech_id`]),n=Number(o[`${t}.speech_step_index`]);return e&&Number.isSafeInteger(n)&&n>0?JSON.stringify([e,n]):null}function K(o,t,e){let n=I(o);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==2)throw new i("Session response has an unsupported protocol version.");let s=I(n.connection);if(!s||s.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let c=v(s.url,"connection.url");try{let u=new URL(c),m=u.hostname==="localhost"||u.hostname==="127.0.0.1"||u.hostname==="[::1]";if(u.protocol!=="wss:"&&!(u.protocol==="ws:"&&m))throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let d=v(n.agent_id,"agent_id");if(d!==e)throw new i("Session response contains a mismatched Agent id.");let l=g(n.avatar_id),a=I(n.avatar),p=null;if(l||a){if(!l||!a||v(a.id,"avatar.id")!==l)throw new i("Session response contains mismatched Avatar metadata.");let u=v(a.engine,"avatar.engine");if(u!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let m=v(a.model_url,"avatar.model_url");try{let h=new URL(m),T=h.hostname==="localhost"||h.hostname==="127.0.0.1"||h.hostname==="[::1]";if(h.protocol!=="https:"&&!(h.protocol==="http:"&&T))throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}p={id:l,name:v(a.name,"avatar.name"),engine:u,modelUrl:m,assetHash:D(a.asset_hash),runtimeProfile:v(a.runtime_profile,"avatar.runtime_profile")}}return{id:v(n.session_id,"session_id"),agentId:d,...l?{avatarId:l}:{},avatar:p,inputMode:t,room:c,token:v(s.token,"connection.token")}}function L(o){let t=I(o);if(!t||typeof t.type!="string")return null;let e={contextId:g(t.contextId)??null,segmentId:g(t.segmentId),occurrence:W(t.occurrence),audioTime:W(t.audioTime)};if(t.type==="avatar-emotion"){let n=g(t.emotion);return n?{phase:"emotion",emotion:n,...e}:null}if(t.type==="avatar-vocal-action"){let n=g(t.action);return n?{phase:"vocal-action",action:n,...e}:null}return null}function P(o){let{avatar:t,room:e,token:n,...s}=o;return s}var E=class{room;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;audioMuted=!1;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";agentIdentity=null;progressExecutions=new Map;progressText=new Map;completedProgressExecutions=new Set;turnId=null;nextTurnId=0;turnControl=Promise.resolve();lifecycleVersion=0;recreateRoomOnConnect=!1;constructor(t){if(this.options=t,this.apiOrigin=U(t.apiOrigin),!M.test(t.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!t.startSession&&typeof t.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.room=this.createRoom()}createRoom(){let t=new x({adaptiveStream:!0,dynacast:!0,audioCaptureDefaults:{voiceIsolation:!1}});return t.on(w.ParticipantConnected,e=>{t===this.room&&this.handleParticipant(e)}),t.on(w.ParticipantDisconnected,e=>{t!==this.room||e.kind!==y.AGENT||(this.agentIdentity=null,this.currentStatus!=="disconnecting"&&(this.handleDisconnected("The Waffle Agent disconnected."),t.disconnect().catch(n=>{this.emit("error",n instanceof Error?n:new i("Could not close the Agent room."))})))}),t.on(w.ParticipantAttributesChanged,(e,n)=>{t===this.room&&this.handleParticipant(n)}),t.on(w.TrackSubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===b.Kind.Audio&&this.setAudioTrack(e.mediaStreamTrack)}),t.on(w.TrackUnsubscribed,(e,n,s)=>{t===this.room&&s.kind===y.AGENT&&e.kind===b.Kind.Audio&&this.setAudioTrack(null)}),t.on(w.DataReceived,(e,n,s,c)=>{if(!(t!==this.room||n?.kind!==y.AGENT||s!==_.RELIABLE||c!=="avatar.performance"))try{let d=L(JSON.parse(new TextDecoder().decode(e)));d&&k(this,"avatar",d)}catch{}}),t.on(w.Disconnected,()=>{t===this.room&&this.handleDisconnected("The Agent session disconnected before it was ready.")}),t.registerTextStreamHandler("lk.transcription",(e,n)=>{(async()=>{let s=this.agentIdentity?t.remoteParticipants.get(this.agentIdentity):void 0;if(t!==this.room||this.currentStatus!=="connected"||s?.kind!==y.AGENT)return;let c=e.info.attributes??{},d=g(c["lk.segment_id"]),l=g(c["lk.transcribed_track_id"]),a=c["lk.transcription_final"];if(!d||!l||a!=="true"&&a!=="false")return;let p=t.localParticipant.audioTrackPublications.has(l),u=s.audioTrackPublications.has(l);if(p===u)return;let m=p?"user":"assistant";if(n.identity!==(p?t.localParticipant.identity:s.identity))return;let h=m==="assistant"?A(c,"lk"):null;if(m==="assistant"&&!h||h&&(this.progressExecutions.set(d,h),this.completedProgressExecutions.has(h)))return;let T="";for await(let S of e){if(T+=S,t!==this.room||this.currentStatus!=="connected"||h&&this.completedProgressExecutions.has(h))return;if(T.length>0){let r={segmentId:d,role:m,text:T,complete:!1};this.progressText.set(d,r),this.emit("transcriptProgress",r)}}a==="true"&&T.length>0&&(this.emit("transcriptProgress",{segmentId:d,role:m,text:T,complete:!0}),this.progressExecutions.delete(d),this.progressText.delete(d))})().catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t.registerTextStreamHandler("animated-waffle.transcript",(e,n)=>{e.readAll().then(s=>{if(t!==this.room||n.identity!==this.agentIdentity||!s)return;let c=e.info.attributes??{},d=g(c["animated-waffle.message_id"]),l=c["animated-waffle.role"];if(!d||l!=="user"&&l!=="assistant"||c["animated-waffle.final"]!=="true")return;let a=l==="assistant"?A(c,"animated-waffle"):null;a&&this.completeTranscriptExecution(a);let p=Number(e.info.timestamp);this.emit("transcript",{id:d,role:l,text:s,final:!0,...Number.isFinite(p)&&p>0?{timestamp:new Date(p).toISOString()}:{}})}).catch(s=>{this.emit("error",s instanceof Error?s:new i("Transcript stream failed."))})}),t}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?P(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}get muted(){return this.audioMuted}set muted(t){this.audioMuted=t,this.audio&&(this.audio.muted=t)}on(t,e){let n=this.listeners.get(t)??new Set;return n.add(e),this.listeners.set(t,n),()=>n.delete(e)}connect(t={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(P(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRoomOnConnect&&(this.room=this.createRoom(),this.recreateRoomOnConnect=!1);let e=++this.lifecycleVersion,n=this.startInternal(t,e).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let t=this.stopInternal().finally(()=>{this.stopPromise===t&&(this.stopPromise=null)});return this.stopPromise=t,t}async sendText(t,e){if(!t.trim())return;this.assertConnected();let n=e?.audioResponse===!1?{"animated-waffle.audio_response":"false"}:void 0;await this.room.localParticipant.sendText(t,{topic:"lk.chat",...n?{attributes:n}:{}})}startPushToTalk(){if(this.assertPushToTalkAvailable(),this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start_turn")}catch(t){throw this.turnId=null,t}}}commitPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("end_turn"),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.assertPushToTalkAvailable(),this.turnId!==null&&(this.sendInputControl("cancel_turn"),this.turnId=null)}async startInternal(t,e){this.activeInputMode=t.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let s=t.sessionId??crypto.randomUUID(),c=crypto.randomUUID(),d=this.options.startSession,l=()=>this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...t.sessionId?{sessionId:t.sessionId}:{},...t.overrides?{overrides:t.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(r=>fetch(this.apiOrigin+"/v1/agents/"+encodeURIComponent(this.options.agentId)+"/sessions",{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:"Bearer "+r,"Idempotency-Key":c},signal:n.signal,body:JSON.stringify({chat_session_id:s,input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic",...t.overrides?{overrides:t.overrides}:{}})})),a;try{a=await this.withTimeout(d?l():l().catch(r=>{if(n.signal.aborted)throw r;return l()}),C,"Timed out creating the Waffle session.",()=>n.abort())}catch(r){let f=r instanceof Error?r:new i("Could not create session.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let p=await a.json().catch(()=>null);if(!a.ok){let r=I(p),f=new i(g(r?.detail)??"Could not create session.",a.status,g(r?.code));throw this.fail(f),f}let u;try{u=K(p,this.activeInputMode,this.options.agentId)}catch(r){let f=r instanceof Error?r:new i("Could not parse the Waffle session.");throw this.fail(f),f}this.currentSession=u,this.emit("session",P(u)),k(this,"presentation",u.avatar),this.setStatus("connecting");let m,h,T=new Promise((r,f)=>{m=r,h=f});this.readyWaiter={reject:h,resolve:m,version:e};let S=this.room;try{if(await S.connect(u.room,u.token),await S.startAudio(),this.options.inputAudioEnabled!==!1){let r=await S.localParticipant.setMicrophoneEnabled(!0),f=r?.audioTrack?.mediaStreamTrack;if(!r||r.isMuted||r.isUpstreamPaused||!f||!f.enabled||f.muted||f.readyState!=="live")throw new i("The microphone track did not publish.")}for(let r of S.remoteParticipants.values())this.handleParticipant(r);await this.withTimeout(T,R,"Timed out waiting for the Waffle Agent.",()=>{S.disconnect().catch(()=>{})})}catch(r){this.readyWaiter?.version===e&&(this.readyWaiter=null),this.currentStatus!=="disconnected"&&await S.disconnect().catch(()=>{});let f=r instanceof Error?r:new i("Could not connect.");throw e!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(f),f)}finally{this.readyWaiter?.version===e&&(this.readyWaiter=null)}if(e!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");return this.setStatus("connected"),P(u)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.turnControl=Promise.resolve(),this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.room.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected();let e=this.agentIdentity;if(!e)throw new i("The Waffle Agent is not ready.");let n=this.room,s=this.turnControl.then(()=>n.localParticipant.performRpc({destinationIdentity:e,method:t,payload:""}));this.turnControl=s.then(()=>{},c=>{this.emit("error",c instanceof Error?c:new i("Push-to-talk control failed."))})}assertPushToTalkAvailable(){if(this.options.inputAudioEnabled===!1)throw new i("Push-to-talk requires input audio to be enabled.",void 0,"input_audio_disabled")}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleParticipant(t){if(t.kind===y.AGENT){this.agentIdentity=t.identity,this.emit("agentSpeaking",t.attributes["lk.agent.state"]==="speaking"),t.attributes["animated-waffle.runtime.ready"]==="true"&&this.readyWaiter?.resolve();for(let e of t.audioTrackPublications.values()){let n=e.track;if(n?.kind===b.Kind.Audio){this.setAudioTrack(n.mediaStreamTrack);break}}}}handleDisconnected(t){this.readyWaiter?.reject(new i(t)),this.readyWaiter=null,this.agentIdentity=null,this.clearTranscriptProgress(),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(t){if(t===this.currentRemoteAudioTrack||(this.stopAudio(!1),this.currentRemoteAudioTrack=t,this.emit("remoteAudioTrack",t),k(this,"audioTrack",t),!t||typeof document>"u")||this.options.audioPlayback===!1)return;let e=document.createElement("audio");e.autoplay=!0,e.muted=this.audioMuted,e.setAttribute("playsinline",""),e.srcObject=new MediaStream([t]),this.audio=e,e.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(t=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),t&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),k(this,"audioTrack",null))}completeTranscriptExecution(t){this.completedProgressExecutions.add(t);for(let[e,n]of this.progressText)this.progressExecutions.get(e)===t&&(this.emit("transcriptProgress",{...n,complete:!0}),this.progressExecutions.delete(e),this.progressText.delete(e))}clearTranscriptProgress(){this.progressExecutions.clear(),this.progressText.clear(),this.completedProgressExecutions.clear()}setStatus(t){t==="disconnected"&&(this.recreateRoomOnConnect=!0),t==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),k(this,"presentation",null)),this.currentStatus!==t&&(this.currentStatus=t,this.emit("connectionState",t),this.emit("status",t))}fail(t){this.readyWaiter?.reject(t),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected"),this.emit("error",t)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let t=await this.options.sessionToken();if(!t.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return t}withTimeout(t,e,n,s){return new Promise((c,d)=>{let l=setTimeout(()=>{s?.(),d(new i(n))},e);t.then(a=>{clearTimeout(l),c(a)},a=>{clearTimeout(l),d(a)})})}emit(t,e){this.listeners.get(t)?.forEach(n=>{n(e)})}};export{E as WaffleClient,i as WaffleError};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.2.6",
+  "version": "0.3.0",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,
@@ -13,6 +13,7 @@
   },
   "files": [
     "dist",
+    "CHANGELOG.md",
     "README.md",
     "LICENSE"
   ],
@@ -33,8 +34,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@pipecat-ai/client-js": "1.12.0",
-    "@pipecat-ai/daily-transport": "1.6.7"
+    "livekit-client": "2.21.0"
   },
   "devDependencies": {
     "tsup": "^8.5.0",