@animated-waffle/client 0.1.0 → 0.1.2

package/README.md

@@ -58,7 +58,7 @@ const waffle = new WaffleClient({
 
 - `connect({ inputMode? })` — `'push-to-talk'` (default) or `'automatic'`.
 - `disconnect()`, `sendText(text)`.
-- `startPushToTalk()` / `stopPushToTalk()` / `cancelPushToTalk()`.
+- `startPushToTalk()` / `commitPushToTalk()` / `cancelPushToTalk()`.
 - `on('status' | 'session' | 'transcript' | 'agentSpeaking' | 'error', fn)` —
   returns the unsubscribe function.
 - `status`, `session` getters; failures throw `WaffleError` with `code`.

package/dist/index.d.ts

@@ -1,35 +1,102 @@
-type WaffleStatus = 'disconnected' | 'connecting' | 'connected' | 'disconnecting';
+type WaffleConnectionState = 'disconnected' | 'connecting' | 'connected' | 'disconnecting';
+/** @deprecated Use WaffleConnectionState. */
+type WaffleStatus = WaffleConnectionState;
 type WaffleInputMode = 'automatic' | 'push-to-talk';
 interface WaffleSession {
     id: string;
     agentId: string;
-    avatarId: string;
+    avatarId?: string;
     inputMode: WaffleInputMode;
 }
 interface WaffleTranscript {
     role: 'user' | 'assistant';
     text: string;
+    /**
+     * False while this text is still provisional. Only interim speech recognition
+     * is provisional: a caller replaces the previous non-final entry from that
+     * speaker rather than appending. Agent text is always final.
+     */
     final: boolean;
     timestamp?: string;
 }
 interface WaffleClientEvents {
+    connectionState: WaffleConnectionState;
+    /** @deprecated Subscribe to connectionState. */
     status: WaffleStatus;
     session: WaffleSession | null;
     error: Error;
     agentSpeaking: boolean;
+    remoteAudioTrack: MediaStreamTrack | null;
     transcript: WaffleTranscript;
 }
 type WaffleSessionToken = () => string | Promise<string>;
-interface WaffleClientOptions {
+interface WaffleSessionOverrides {
+    persona?: string;
+    llmProvider?: string | null;
+    llmModel?: string | null;
+    thinkingLevel?: string;
+    webSearchEnabled?: boolean;
+    memoryEnabled?: boolean;
+    systemPrompt?: string;
+    asrProvider?: string | null;
+    ttsProvider?: string;
+    ttsModelId?: string;
+    ttsVoiceId?: string;
+    voiceId?: string | null;
+    voiceRouteId?: string | null;
+    defaultDeviceProfileId?: string | null;
+    avatarId?: string;
+    avatarEnabled?: boolean;
+    openaiModel?: string | null;
+    openaiVoice?: string;
+}
+interface WaffleClientBaseOptions {
     /** The published Agent to connect to. */
     agentId: string;
+}
+interface WaffleTokenClientOptions extends WaffleClientBaseOptions {
     /** A short-lived token minted for this Agent by the Narya API. */
     sessionToken: WaffleSessionToken;
     /** Override the Narya API origin for a non-production environment. */
     apiOrigin?: string;
+    startSession?: never;
+}
+interface WaffleManagedClientOptions extends WaffleClientBaseOptions {
+    /** Start a session through the embedding application's authenticated backend. */
+    startSession: WaffleSessionStarter;
+    sessionToken?: never;
+    apiOrigin?: never;
 }
-interface WaffleConnectOptions {
+type WaffleClientOptions = WaffleTokenClientOptions | WaffleManagedClientOptions;
+interface WaffleSessionStartOptions {
     inputMode?: WaffleInputMode;
+    /** Existing application session to associate with this realtime run. */
+    sessionId?: string;
+    /** Temporary session-only Agent overrides. The server remains authoritative. */
+    overrides?: Readonly<WaffleSessionOverrides>;
+}
+type WaffleConnectOptions = WaffleSessionStartOptions;
+interface WaffleSessionStartInput {
+    agentId: string;
+    inputMode: WaffleInputMode;
+    sessionId?: string;
+    overrides?: Readonly<WaffleSessionOverrides>;
+    signal: AbortSignal;
+}
+type WaffleSessionStarter = (input: WaffleSessionStartInput) => Response | Promise<Response>;
+interface WaffleSessionClient {
+    readonly connectionState: WaffleConnectionState;
+    /** @deprecated Use connectionState. */
+    readonly status: WaffleStatus;
+    readonly session: WaffleSession | null;
+    readonly remoteAudioTrack: MediaStreamTrack | null;
+    on<K extends keyof WaffleClientEvents>(event: K, listener: (payload: WaffleClientEvents[K]) => void): () => void;
+    connect(options?: WaffleSessionStartOptions): Promise<WaffleSession>;
+    disconnect(): Promise<void>;
+    sendText(text: string): Promise<void>;
+    startPushToTalk(): void;
+    commitPushToTalk(): void;
+    cancelPushToTalk(): void;
 }
 type Listener<K extends keyof WaffleClientEvents> = (payload: WaffleClientEvents[K]) => void;
 declare class WaffleError extends Error {
@@ -37,8 +104,9 @@ declare class WaffleError extends Error {
     readonly code?: string | undefined;
     constructor(message: string, status?: number | undefined, code?: string | undefined);
 }
-declare class WaffleClient {
-    private readonly realtime;
+declare class WaffleClient implements WaffleSessionClient {
+    private realtime;
+    private readonly realtimeCallbacks;
     private readonly listeners;
     private readonly options;
     private readonly apiOrigin;
@@ -47,19 +115,27 @@ declare class WaffleClient {
     private startPromise;
     private stopPromise;
     private requestAbort;
+    private readyWaiter;
     private audio;
+    private currentRemoteAudioTrack;
     private activeInputMode;
     private turnId;
     private nextTurnId;
     private lifecycleVersion;
+    private recreateRealtimeOnConnect;
     constructor(options: WaffleClientOptions);
+    private createRealtime;
     get status(): WaffleStatus;
+    get connectionState(): WaffleConnectionState;
     get session(): WaffleSession | null;
+    get remoteAudioTrack(): MediaStreamTrack | null;
     on<K extends keyof WaffleClientEvents>(event: K, listener: Listener<K>): () => void;
     connect(options?: WaffleConnectOptions): Promise<WaffleSession>;
     disconnect(): Promise<void>;
     sendText(text: string): Promise<void>;
     startPushToTalk(): void;
+    commitPushToTalk(): void;
+    /** @deprecated Use commitPushToTalk(). */
     stopPushToTalk(): void;
     cancelPushToTalk(): void;
     private startInternal;
@@ -67,6 +143,23 @@ declare class WaffleClient {
     private sendInputControl;
     private assertConnected;
     private handleUserTranscript;
+    /**
+     * The runtime announces one spoken segment three times with the same text: as
+     * it is queued for synthesis (`spoken_status: 'new'`), on word-level progress
+     * (`'in-progress'`), and once synthesis completes (`'completed'`).
+     *
+     * Only the last one is reported. The other two cannot be paired with it:
+     * `segment_id` is the *frame* id on this path, so the queued and completed
+     * events for one sentence carry different ids, and they are not even
+     * adjacent — a segment queued at the end of one turn can complete after the
+     * next user message. Forwarding all three duplicated every sentence in a
+     * caller's transcript with no key to reconcile them.
+     *
+     * Reporting only the completed event costs the caption the few hundred
+     * milliseconds of synthesis, and it arrives as the agent starts saying it.
+     * Text the runtime will not speak carries no lifecycle at all and is
+     * reported as it arrives.
+     */
     private handleBotOutput;
     private handleTransportState;
     private handleReady;
@@ -80,4 +173,4 @@ declare class WaffleClient {
     private emit;
 }
 
-export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, WaffleError, type WaffleInputMode, type WaffleSession, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript };
+export { WaffleClient, type WaffleClientEvents, type WaffleClientOptions, type WaffleConnectOptions, type WaffleConnectionState, WaffleError, type WaffleInputMode, type WaffleSession, type WaffleSessionClient, type WaffleSessionOverrides, type WaffleSessionStartInput, type WaffleSessionStartOptions, type WaffleSessionStarter, type WaffleSessionToken, type WaffleStatus, type WaffleTranscript };

package/dist/index.js

@@ -1 +1 @@
-import{b as d}from"./chunk-BXRNROI7.js";import{PipecatClient as g}from"@pipecat-ai/client-js";import{DailyTransport as S}from"@pipecat-ai/daily-transport";var w=15e3,T=3e4,y="https://animated-waffle.narya.ai",k=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,I=/^sha256:[a-f0-9]{64}$/,s=class extends Error{constructor(e,n,o){super(e);this.status=n;this.code=o;this.name="WaffleError"}status;code};function p(i){return i!==null&&typeof i=="object"&&!Array.isArray(i)?i:null}function b(i=y){let t;try{t=new URL(i)}catch{throw new s("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let e=t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="[::1]";if(t.protocol!=="https:"&&!(t.protocol==="http:"&&e)||t.username||t.password||t.pathname!=="/"&&t.pathname!==""||t.search||t.hash)throw new s("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return t.origin}function u(i,t){if(typeof i!="string"||i.trim().length===0)throw new s(`Session response is missing ${t}.`);return i}function _(i){let t=u(i,"avatar.asset_hash");if(!I.test(t))throw new s("Session response contains an invalid Avatar asset hash.");return t}function m(i){return typeof i=="number"&&Number.isFinite(i)?i:void 0}function h(i){return typeof i=="string"&&i.length>0?i:void 0}function A(i,t){let e=p(i);if(!e)throw new s("Session response must be a JSON object.");if(e.protocol_version!==1)throw new s("Session response has an unsupported protocol version.");let n=p(e.connection);if(!n||n.transport!=="webrtc")throw new s("Session response is missing a WebRTC connection.");let o=u(n.url,"connection.url");try{if(new URL(o).protocol!=="https:")throw new Error}catch{throw new s("Session response contains an invalid connection URL.")}let c=u(e.avatar_id,"avatar_id"),a=p(e.avatar);if(!a||u(a.id,"avatar.id")!==c)throw new s("Session response contains mismatched Avatar metadata.");let r=u(a.engine,"avatar.engine");if(r!=="vrm")throw new s("Session response contains an unsupported Avatar engine.");let l=u(a.model_url,"avatar.model_url");try{if(new URL(l).protocol!=="https:")throw new Error}catch{throw new s("Session response contains an invalid Avatar model URL.")}return{id:u(e.session_id,"session_id"),agentId:u(e.agent_id,"agent_id"),avatarId:c,avatar:{id:c,name:u(a.name,"avatar.name"),engine:r,modelUrl:l,assetHash:_(a.asset_hash),runtimeProfile:u(a.runtime_profile,"avatar.runtime_profile")},inputMode:t,room:o,token:u(n.token,"connection.token")}}function C(i){let t=p(i);if(!t||typeof t.type!="string")return null;let e={contextId:h(t.contextId)??null,segmentId:h(t.segmentId),occurrence:m(t.occurrence),audioTime:m(t.audioTime),offset:m(t.offset)};if(t.type==="avatar-emotion"){let n=h(t.emotion);return n?{phase:"emotion",emotion:n,...e}:null}if(t.type==="avatar-vocal-action"){let n=h(t.action);return n?{phase:"vocal-action",action:n,...e}:null}return null}function f(i){let{avatar:t,room:e,token:n,...o}=i;return o}var v=class{realtime;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;audio=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;constructor(t){if(this.options=t,this.apiOrigin=b(t.apiOrigin),!k.test(t.agentId))throw new s("agentId must be a UUID.",void 0,"invalid_agent_id");if(typeof t.sessionToken!="function")throw new s("sessionToken must be a function.",void 0,"invalid_session_token");let e=new S({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}}),n={onTransportStateChanged:o=>this.handleTransportState(o),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onUserTranscript:o=>this.handleUserTranscript(o),onBotOutput:o=>this.handleBotOutput(o),onServerMessage:o=>{let c=C(o);c&&d(this,"avatar",c)},onTrackStarted:(o,c)=>{o.kind==="audio"&&c?.local===!1&&this.setAudioTrack(o)},onTrackStopped:(o,c)=>{o.kind==="audio"&&c?.local===!1&&this.setAudioTrack(null)},onError:o=>this.fail(new s(p(o)?.message?.toString()??"Realtime connection failed."))};this.realtime=new g({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:n})}get status(){return this.currentStatus}get session(){return this.currentSession?f(this.currentSession):null}on(t,e){let n=this.listeners.get(t)??new Set;return n.add(e),this.listeners.set(t,n),()=>n.delete(e)}connect(t={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(f(this.currentSession));if(this.startPromise)return this.startPromise;let e=++this.lifecycleVersion,n=this.startInternal(t,e).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let t=this.stopInternal().finally(()=>{this.stopPromise===t&&(this.stopPromise=null)});return this.stopPromise=t,t}async sendText(t){let e=t.trim();e&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.realtime.enableMic(!0),this.sendInputControl("start",this.turnId)}catch(t){throw this.turnId=null,t}}}stopPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(t,e){this.activeInputMode=t.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let o;try{let r=await this.resolveSessionToken();o=await this.withTimeout(fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${r}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})}),w,"Timed out creating the Waffle session.",()=>n.abort())}catch(r){let l=r instanceof Error?r:new s("Could not create session.");throw e!==this.lifecycleVersion?new s("The connection attempt was cancelled."):(this.fail(l),l)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(e!==this.lifecycleVersion)throw new s("The connection attempt was cancelled.");let c=await o.json().catch(()=>null);if(!o.ok){let r=p(c),l=new s(h(r?.detail)??"Could not create session.",o.status,h(r?.code));throw this.fail(l),l}let a;try{a=A(c,this.activeInputMode)}catch(r){let l=r instanceof Error?r:new s("Could not parse the Waffle session.");throw this.fail(l),l}this.currentSession=a,this.emit("session",f(a)),d(this,"presentation",a.avatar),this.setStatus("connecting");try{await this.withTimeout(this.realtime.connect({url:a.room,token:a.token}).then(()=>{}),T,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(r){this.currentSession=null,this.emit("session",null),d(this,"presentation",null),await this.realtime.disconnect().catch(()=>{});let l=r instanceof Error?r:new s("Could not connect.");throw e!==this.lifecycleVersion?new s("The connection attempt was cancelled."):(this.fail(l),l)}return this.activeInputMode==="automatic"&&this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),f(a)}async stopInternal(){if(this.lifecycleVersion+=1,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.currentSession=null,this.emit("session",null),d(this,"presentation",null),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(t,e){if(this.activeInputMode!=="push-to-talk")throw new s('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:t,turnId:e})}assertConnected(){if(this.status!=="connected")throw new s("Start the Waffle session before sending input.")}handleUserTranscript(t){this.emit("transcript",{role:"user",text:t.text,final:t.final,timestamp:t.timestamp})}handleBotOutput(t){this.emit("transcript",{role:"assistant",text:t.text,final:!0})}handleTransportState(t){t==="ready"?this.handleReady():t==="disconnecting"?this.setStatus("disconnecting"):t==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.setStatus("connected")}handleDisconnected(){this.currentSession=null,this.emit("session",null),d(this,"presentation",null),this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(t){if(d(this,"audioTrack",t),this.stopAudio(!1),!t||typeof document>"u")return;let e=document.createElement("audio");e.autoplay=!0,e.setAttribute("playsinline",""),e.srcObject=new MediaStream([t]),this.audio=e,e.play().catch(n=>{this.emit("error",n instanceof Error?n:new s("Browser blocked agent audio."))})}stopAudio(t=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),t&&d(this,"audioTrack",null)}setStatus(t){this.currentStatus!==t&&(this.currentStatus=t,this.emit("status",t))}fail(t){this.setStatus("disconnected"),this.emit("error",t)}async resolveSessionToken(){let t=await this.options.sessionToken();if(!t.trim())throw new s("sessionToken must not be empty.",void 0,"invalid_session_token");return t}withTimeout(t,e,n,o){return new Promise((c,a)=>{let r=setTimeout(()=>{o?.(),a(new s(n))},e);t.then(l=>{clearTimeout(r),c(l)},l=>{clearTimeout(r),a(l)})})}emit(t,e){this.listeners.get(t)?.forEach(n=>{n(e)})}};export{v as WaffleClient,s as WaffleError};
+import{b as h}from"./chunk-BXRNROI7.js";import{PipecatClient as y}from"@pipecat-ai/client-js";import{DailyTransport as w}from"@pipecat-ai/daily-transport";var k=15e3,W=3e4,I="https://animated-waffle.narya.ai",C=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,b=/^sha256:[a-f0-9]{64}$/,i=class extends Error{constructor(t,n,l){super(t);this.status=n;this.code=l;this.name="WaffleError"}status;code};function m(s){return s!==null&&typeof s=="object"&&!Array.isArray(s)?s:null}function P(s=I){let e;try{e=new URL(s)}catch{throw new i("apiOrigin must be a valid URL.",void 0,"invalid_api_origin")}let t=e.hostname==="localhost"||e.hostname==="127.0.0.1"||e.hostname==="[::1]";if(e.protocol!=="https:"&&!(e.protocol==="http:"&&t)||e.username||e.password||e.pathname!=="/"&&e.pathname!==""||e.search||e.hash)throw new i("apiOrigin must be an HTTPS origin or an HTTP loopback origin.",void 0,"invalid_api_origin");return e.origin}function u(s,e){if(typeof s!="string"||s.trim().length===0)throw new i(`Session response is missing ${e}.`);return s}function A(s){let e=u(s,"avatar.asset_hash");if(!b.test(e))throw new i("Session response contains an invalid Avatar asset hash.");return e}function g(s){return typeof s=="number"&&Number.isFinite(s)?s:void 0}function p(s){return typeof s=="string"&&s.length>0?s:void 0}function O(s,e,t){let n=m(s);if(!n)throw new i("Session response must be a JSON object.");if(n.protocol_version!==1)throw new i("Session response has an unsupported protocol version.");let l=m(n.connection);if(!l||l.transport!=="webrtc")throw new i("Session response is missing a WebRTC connection.");let f=u(l.url,"connection.url");try{if(new URL(f).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid connection URL.")}let c=u(n.agent_id,"agent_id");if(c!==t)throw new i("Session response contains a mismatched Agent id.");let d=p(n.avatar_id),a=m(n.avatar),v=null;if(d||a){if(!d||!a||u(a.id,"avatar.id")!==d)throw new i("Session response contains mismatched Avatar metadata.");let o=u(a.engine,"avatar.engine");if(o!=="vrm")throw new i("Session response contains an unsupported Avatar engine.");let r=u(a.model_url,"avatar.model_url");try{if(new URL(r).protocol!=="https:")throw new Error}catch{throw new i("Session response contains an invalid Avatar model URL.")}v={id:d,name:u(a.name,"avatar.name"),engine:o,modelUrl:r,assetHash:A(a.asset_hash),runtimeProfile:u(a.runtime_profile,"avatar.runtime_profile")}}return{id:u(n.session_id,"session_id"),agentId:c,...d?{avatarId:d}:{},avatar:v,inputMode:e,room:f,token:u(l.token,"connection.token")}}function _(s){let e=m(s);if(!e||typeof e.type!="string")return null;let t={contextId:p(e.contextId)??null,segmentId:p(e.segmentId),occurrence:g(e.occurrence),audioTime:g(e.audioTime),offset:g(e.offset)};if(e.type==="avatar-emotion"){let n=p(e.emotion);return n?{phase:"emotion",emotion:n,...t}:null}if(e.type==="avatar-vocal-action"){let n=p(e.action);return n?{phase:"vocal-action",action:n,...t}:null}return null}function S(s){let{avatar:e,room:t,token:n,...l}=s;return l}var T=class{realtime;realtimeCallbacks;listeners=new Map;options;apiOrigin;currentStatus="disconnected";currentSession=null;startPromise=null;stopPromise=null;requestAbort=null;readyWaiter=null;audio=null;currentRemoteAudioTrack=null;activeInputMode="push-to-talk";turnId=null;nextTurnId=0;lifecycleVersion=0;recreateRealtimeOnConnect=!1;constructor(e){if(this.options=e,this.apiOrigin=P(e.apiOrigin),!C.test(e.agentId))throw new i("agentId must be a UUID.",void 0,"invalid_agent_id");if(!e.startSession&&typeof e.sessionToken!="function")throw new i("sessionToken must be a function.",void 0,"invalid_session_token");this.realtimeCallbacks={onTransportStateChanged:t=>this.handleTransportState(t),onBotReady:()=>this.handleReady(),onDisconnected:()=>this.handleDisconnected(),onBotStartedSpeaking:()=>this.emit("agentSpeaking",!0),onBotStoppedSpeaking:()=>this.emit("agentSpeaking",!1),onUserTranscript:t=>this.handleUserTranscript(t),onBotOutput:t=>this.handleBotOutput(t),onServerMessage:t=>{let n=_(t);n&&h(this,"avatar",n)},onTrackStarted:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(t)},onTrackStopped:(t,n)=>{t.kind==="audio"&&n?.local===!1&&this.setAudioTrack(null)},onError:t=>this.fail(new i(m(t)?.message?.toString()??"Realtime connection failed."))},this.realtime=this.createRealtime()}createRealtime(){let e=new w({bufferLocalAudioUntilBotReady:!0,inputSettings:{audio:{processor:{type:"noise-cancellation"}}}});return new y({transport:e,enableCam:!1,enableMic:!1,disconnectOnBotDisconnect:!0,callbacks:this.realtimeCallbacks})}get status(){return this.currentStatus}get connectionState(){return this.currentStatus}get session(){return this.currentSession?S(this.currentSession):null}get remoteAudioTrack(){return this.currentRemoteAudioTrack}on(e,t){let n=this.listeners.get(e)??new Set;return n.add(t),this.listeners.set(e,n),()=>n.delete(t)}connect(e={}){if(this.currentSession&&this.status==="connected")return Promise.resolve(S(this.currentSession));if(this.startPromise)return this.startPromise;this.recreateRealtimeOnConnect&&(this.realtime=this.createRealtime(),this.recreateRealtimeOnConnect=!1);let t=++this.lifecycleVersion,n=this.startInternal(e,t).finally(()=>{this.startPromise===n&&(this.startPromise=null)});return this.startPromise=n,n}disconnect(){if(this.stopPromise)return this.stopPromise;let e=this.stopInternal().finally(()=>{this.stopPromise===e&&(this.stopPromise=null)});return this.stopPromise=e,e}async sendText(e){e.trim()&&(this.assertConnected(),await this.realtime.sendText(e,{audio_response:!0,run_immediately:!0}),this.emit("transcript",{role:"user",text:e,final:!0}))}startPushToTalk(){if(this.turnId===null){this.nextTurnId+=1,this.turnId=this.nextTurnId;try{this.sendInputControl("start",this.turnId)}catch(e){throw this.turnId=null,e}}}commitPushToTalk(){this.turnId!==null&&(this.sendInputControl("stop",this.turnId),this.turnId=null)}stopPushToTalk(){this.commitPushToTalk()}cancelPushToTalk(){this.turnId!==null&&(this.sendInputControl("cancel",this.turnId),this.turnId=null)}async startInternal(e,t){this.activeInputMode=e.inputMode??"push-to-talk",this.setStatus("connecting");let n=new AbortController;this.requestAbort=n;let l;try{l=await this.withTimeout(this.options.startSession?Promise.resolve(this.options.startSession({agentId:this.options.agentId,inputMode:this.activeInputMode,...e.sessionId?{sessionId:e.sessionId}:{},...e.overrides?{overrides:e.overrides}:{},signal:n.signal})):this.resolveSessionToken().then(o=>fetch(`${this.apiOrigin}/v1/agents/${encodeURIComponent(this.options.agentId)}/sessions`,{method:"POST",headers:{Accept:"application/json","Content-Type":"application/json",Authorization:`Bearer ${o}`},signal:n.signal,body:JSON.stringify({input_mode:this.activeInputMode==="push-to-talk"?"push_to_talk":"automatic"})})),k,"Timed out creating the Waffle session.",()=>n.abort())}catch(o){let r=o instanceof Error?o:new i("Could not create session.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.requestAbort===n&&(this.requestAbort=null)}if(t!==this.lifecycleVersion)throw new i("The connection attempt was cancelled.");let f=await l.json().catch(()=>null);if(!l.ok){let o=m(f),r=new i(p(o?.detail)??"Could not create session.",l.status,p(o?.code));throw this.fail(r),r}let c;try{c=O(f,this.activeInputMode,this.options.agentId)}catch(o){let r=o instanceof Error?o:new i("Could not parse the Waffle session.");throw this.fail(r),r}this.currentSession=c,this.emit("session",S(c)),h(this,"presentation",c.avatar),this.setStatus("connecting");let d,a,v=new Promise((o,r)=>{d=o,a=r});this.readyWaiter={reject:a,resolve:d,version:t};try{await this.withTimeout(Promise.all([this.realtime.connect({url:c.room,token:c.token}),v]).then(()=>{}),W,"Timed out waiting for the Waffle Agent.",()=>{this.realtime.disconnect().catch(()=>{})})}catch(o){await this.realtime.disconnect().catch(()=>{});let r=o instanceof Error?o:new i("Could not connect.");throw t!==this.lifecycleVersion?new i("The connection attempt was cancelled."):(this.fail(r),r)}finally{this.readyWaiter?.version===t&&(this.readyWaiter=null)}return this.realtime.enableMic(!0),this.setStatus("connected"),this.setAudioTrack(this.realtime.tracks().bot?.audio??null),S(c)}async stopInternal(){if(this.lifecycleVersion+=1,this.readyWaiter?.reject(new i("The connection attempt was cancelled.")),this.readyWaiter=null,this.requestAbort?.abort(),this.turnId=null,this.stopAudio(),this.currentStatus!=="disconnected"){this.setStatus("disconnecting");try{await this.realtime.disconnect()}finally{this.emit("agentSpeaking",!1),this.setStatus("disconnected")}}}sendInputControl(e,t){if(this.activeInputMode!=="push-to-talk")throw new i('Push-to-talk controls require inputMode: "push-to-talk".');this.assertConnected(),this.realtime.sendClientMessage("voice.input_control",{mode:"push_to_talk",state:e,turnId:t})}assertConnected(){if(this.status!=="connected")throw new i("Start the Waffle session before sending input.")}handleUserTranscript(e){this.emit("transcript",{role:"user",text:e.text,final:e.final,timestamp:e.timestamp})}handleBotOutput(e){e.spoken_status==="new"||e.spoken_status==="in-progress"||this.emit("transcript",{role:"assistant",text:e.text,final:!0})}handleTransportState(e){e==="ready"?this.handleReady():e==="disconnecting"?this.setStatus("disconnecting"):e==="disconnected"?this.handleDisconnected():this.currentSession&&this.currentStatus!=="connected"&&this.setStatus("connecting")}handleReady(){this.currentSession&&this.readyWaiter?.resolve()}handleDisconnected(){this.readyWaiter?.reject(new i("The realtime session disconnected before it was ready.")),this.readyWaiter=null,this.stopAudio(),this.emit("agentSpeaking",!1),this.setStatus("disconnected")}setAudioTrack(e){if(this.currentRemoteAudioTrack=e,this.emit("remoteAudioTrack",e),h(this,"audioTrack",e),this.stopAudio(!1),!e||typeof document>"u")return;let t=document.createElement("audio");t.autoplay=!0,t.setAttribute("playsinline",""),t.srcObject=new MediaStream([e]),this.audio=t,t.play().catch(n=>{this.emit("error",n instanceof Error?n:new i("Browser blocked agent audio."))})}stopAudio(e=!0){this.audio&&(this.audio.pause(),this.audio.srcObject=null,this.audio=null),e&&this.currentRemoteAudioTrack&&(this.currentRemoteAudioTrack=null,this.emit("remoteAudioTrack",null),h(this,"audioTrack",null))}setStatus(e){e==="disconnected"&&(this.recreateRealtimeOnConnect=!0),e==="disconnected"&&this.currentSession&&(this.currentSession=null,this.emit("session",null),h(this,"presentation",null)),this.currentStatus!==e&&(this.currentStatus=e,this.emit("connectionState",e),this.emit("status",e))}fail(e){this.readyWaiter?.reject(e),this.readyWaiter=null,this.setStatus("disconnected"),this.emit("error",e)}async resolveSessionToken(){if(!this.options.sessionToken)throw new i("No session token provider is configured.");let e=await this.options.sessionToken();if(!e.trim())throw new i("sessionToken must not be empty.",void 0,"invalid_session_token");return e}withTimeout(e,t,n,l){return new Promise((f,c)=>{let d=setTimeout(()=>{l?.(),c(new i(n))},t);e.then(a=>{clearTimeout(d),f(a)},a=>{clearTimeout(d),c(a)})})}emit(e,t){this.listeners.get(e)?.forEach(n=>{n(t)})}};export{T as WaffleClient,i as WaffleError};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animated-waffle/client",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Framework-independent browser SDK for Narya Animated Waffle talking Agents.",
   "type": "module",
   "sideEffects": false,