@runtypelabs/persona 4.24.0 → 4.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approval-ui.d.cts +63 -0
- package/dist/approval-ui.d.ts +63 -0
- package/dist/artifacts-ui.d.cts +63 -0
- package/dist/artifacts-ui.d.ts +63 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions.d.cts +63 -0
- package/dist/context-mentions.d.ts +63 -0
- package/dist/event-stream-view.d.cts +63 -0
- package/dist/event-stream-view.d.ts +63 -0
- package/dist/forms-ui.d.cts +74 -0
- package/dist/forms-ui.d.ts +74 -0
- package/dist/index.cjs +42 -42
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +75 -1
- package/dist/index.d.ts +75 -1
- package/dist/index.global.js +27 -27
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +42 -42
- package/dist/index.js.map +1 -1
- package/dist/session-reconnect.d.cts +63 -0
- package/dist/session-reconnect.d.ts +63 -0
- package/dist/smart-dom-reader.d.cts +63 -0
- package/dist/smart-dom-reader.d.ts +63 -0
- package/dist/theme-editor-preview.cjs +49 -49
- package/dist/theme-editor-preview.d.cts +63 -0
- package/dist/theme-editor-preview.d.ts +63 -0
- package/dist/theme-editor-preview.js +27 -27
- package/dist/theme-editor.d.cts +63 -0
- package/dist/theme-editor.d.ts +63 -0
- package/dist/voice-runtime-7DA7MJH4.js +1 -0
- package/dist/voice-runtime.cjs +1 -1
- package/dist/voice-runtime.d.cts +33 -0
- package/dist/voice-runtime.d.ts +33 -0
- package/dist/voice-runtime.js +1 -1
- package/package.json +1 -1
- package/src/index-core.ts +2 -0
- package/src/session.ts +30 -1
- package/src/session.voice.test.ts +72 -1
- package/src/types.ts +71 -0
- package/src/ui.ts +59 -0
- package/src/ui.voice-prewarm.test.ts +222 -0
- package/src/voice/runtype-voice-provider.ts +238 -27
- package/src/voice/voice.test.ts +272 -2
- package/dist/voice-runtime-EFR3R5ES.js +0 -1
package/dist/theme-editor.d.cts
CHANGED
|
@@ -6582,6 +6582,22 @@ type AgentWidgetStatusIndicatorConfig = {
|
|
|
6582
6582
|
/** Status text while a reconnect attempt is in flight. */
|
|
6583
6583
|
resumingText?: string;
|
|
6584
6584
|
};
|
|
6585
|
+
/** Built-in `voiceRecognition.prewarm` triggers. See {@link AgentWidgetVoiceRecognitionConfig.prewarm}. */
|
|
6586
|
+
type AgentWidgetVoicePrewarmTrigger = 'hover';
|
|
6587
|
+
/**
|
|
6588
|
+
* Custom `voiceRecognition.prewarm` trigger. Runs while the composer has a mic
|
|
6589
|
+
* button, and again when the button is rebuilt or `update()` swaps in a
|
|
6590
|
+
* different function. Call `warm()` whenever the visitor shows intent to talk
|
|
6591
|
+
* (it is safe to call repeatedly). Return a cleanup to run on replacement.
|
|
6592
|
+
*/
|
|
6593
|
+
type AgentWidgetVoicePrewarmHook = (context: {
|
|
6594
|
+
/** Fire-and-forget voice warm-up; latched and error-swallowing. */
|
|
6595
|
+
warm: () => void;
|
|
6596
|
+
/** The composer's current mic button. */
|
|
6597
|
+
micButton: HTMLButtonElement;
|
|
6598
|
+
/** The widget's mount element. */
|
|
6599
|
+
mount: HTMLElement;
|
|
6600
|
+
}) => void | (() => void);
|
|
6585
6601
|
type AgentWidgetVoiceRecognitionConfig = {
|
|
6586
6602
|
/**
|
|
6587
6603
|
* Enable the mic button and voice input. The mic still only renders when the
|
|
@@ -6647,6 +6663,29 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6647
6663
|
* Fires once per turn after the reply completes.
|
|
6648
6664
|
*/
|
|
6649
6665
|
onMetrics?: (metrics: VoiceMetrics) => void;
|
|
6666
|
+
/**
|
|
6667
|
+
* Warm the voice path when the visitor shows intent to talk, before the mic
|
|
6668
|
+
* click, so the call starts faster.
|
|
6669
|
+
*
|
|
6670
|
+
* - `'hover'`: when the pointer enters the mic button, it receives focus, or
|
|
6671
|
+
* (on touch) the first pointer-down on it.
|
|
6672
|
+
* - `false`: never.
|
|
6673
|
+
* - A function: custom trigger. Called with `{ warm, micButton, mount }`
|
|
6674
|
+
* while the mic button exists (and again when the button is rebuilt or
|
|
6675
|
+
* `update()` swaps in a different function); call `warm()` whenever you
|
|
6676
|
+
* decide, e.g. when the panel opens. May return a cleanup that runs on
|
|
6677
|
+
* destroy, rebuild, or replacement. Replaces the built-in trigger.
|
|
6678
|
+
*
|
|
6679
|
+
* What `warm()` does depends on the provider: `runtype` fires its prewarm
|
|
6680
|
+
* (see `provider.runtype.prewarmMode`), a `custom` provider's optional
|
|
6681
|
+
* {@link VoiceProvider.prewarm} runs, and the browser (Web Speech) path
|
|
6682
|
+
* starts the client-token session init (as `sessionInit` does), since speech
|
|
6683
|
+
* recognition itself cannot start before the click. Warming is
|
|
6684
|
+
* fire-and-forget, latched, and never surfaces an error or UI state.
|
|
6685
|
+
*
|
|
6686
|
+
* @default 'hover' in client token mode, otherwise `false`
|
|
6687
|
+
*/
|
|
6688
|
+
prewarm?: AgentWidgetVoicePrewarmTrigger | AgentWidgetVoicePrewarmHook | false;
|
|
6650
6689
|
provider?: {
|
|
6651
6690
|
type: 'browser' | 'runtype' | 'custom';
|
|
6652
6691
|
browser?: {
|
|
@@ -6668,6 +6707,23 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6668
6707
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
6669
6708
|
*/
|
|
6670
6709
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
6710
|
+
/**
|
|
6711
|
+
* How `prewarm()` warms the voice path (see `voiceRecognition.prewarm`).
|
|
6712
|
+
*
|
|
6713
|
+
* - `'request'` (default): a fire-and-forget
|
|
6714
|
+
* `POST {host}/v1/client/agents/{agentId}/voice/prewarm`.
|
|
6715
|
+
* - `'attach'`: open the voice WebSocket ahead of the click without
|
|
6716
|
+
* starting a call; the click then starts the call on that socket. It can
|
|
6717
|
+
* create server work before the visitor clicks, so it is opt-in.
|
|
6718
|
+
* @default 'request'
|
|
6719
|
+
*/
|
|
6720
|
+
prewarmMode?: 'request' | 'attach';
|
|
6721
|
+
/**
|
|
6722
|
+
* `'attach'` mode: how long (ms) an attached socket waits for the click
|
|
6723
|
+
* before it closes. The server clamps it to 30000..600000.
|
|
6724
|
+
* @default 30000
|
|
6725
|
+
*/
|
|
6726
|
+
attachIdleMs?: number;
|
|
6671
6727
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
6672
6728
|
pauseDuration?: number;
|
|
6673
6729
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -6968,6 +7024,13 @@ interface VoiceProvider {
|
|
|
6968
7024
|
deactivateBargeIn?(): Promise<void>;
|
|
6969
7025
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
6970
7026
|
stopPlayback?(): void;
|
|
7027
|
+
/**
|
|
7028
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
7029
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
7030
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
7031
|
+
* repeatedly; throttle as needed.
|
|
7032
|
+
*/
|
|
7033
|
+
prewarm?(): void;
|
|
6971
7034
|
}
|
|
6972
7035
|
/**
|
|
6973
7036
|
* Extra context for an approval decision, surfaced to `onDecision` and to the
|
package/dist/theme-editor.d.ts
CHANGED
|
@@ -6582,6 +6582,22 @@ type AgentWidgetStatusIndicatorConfig = {
|
|
|
6582
6582
|
/** Status text while a reconnect attempt is in flight. */
|
|
6583
6583
|
resumingText?: string;
|
|
6584
6584
|
};
|
|
6585
|
+
/** Built-in `voiceRecognition.prewarm` triggers. See {@link AgentWidgetVoiceRecognitionConfig.prewarm}. */
|
|
6586
|
+
type AgentWidgetVoicePrewarmTrigger = 'hover';
|
|
6587
|
+
/**
|
|
6588
|
+
* Custom `voiceRecognition.prewarm` trigger. Runs while the composer has a mic
|
|
6589
|
+
* button, and again when the button is rebuilt or `update()` swaps in a
|
|
6590
|
+
* different function. Call `warm()` whenever the visitor shows intent to talk
|
|
6591
|
+
* (it is safe to call repeatedly). Return a cleanup to run on replacement.
|
|
6592
|
+
*/
|
|
6593
|
+
type AgentWidgetVoicePrewarmHook = (context: {
|
|
6594
|
+
/** Fire-and-forget voice warm-up; latched and error-swallowing. */
|
|
6595
|
+
warm: () => void;
|
|
6596
|
+
/** The composer's current mic button. */
|
|
6597
|
+
micButton: HTMLButtonElement;
|
|
6598
|
+
/** The widget's mount element. */
|
|
6599
|
+
mount: HTMLElement;
|
|
6600
|
+
}) => void | (() => void);
|
|
6585
6601
|
type AgentWidgetVoiceRecognitionConfig = {
|
|
6586
6602
|
/**
|
|
6587
6603
|
* Enable the mic button and voice input. The mic still only renders when the
|
|
@@ -6647,6 +6663,29 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6647
6663
|
* Fires once per turn after the reply completes.
|
|
6648
6664
|
*/
|
|
6649
6665
|
onMetrics?: (metrics: VoiceMetrics) => void;
|
|
6666
|
+
/**
|
|
6667
|
+
* Warm the voice path when the visitor shows intent to talk, before the mic
|
|
6668
|
+
* click, so the call starts faster.
|
|
6669
|
+
*
|
|
6670
|
+
* - `'hover'`: when the pointer enters the mic button, it receives focus, or
|
|
6671
|
+
* (on touch) the first pointer-down on it.
|
|
6672
|
+
* - `false`: never.
|
|
6673
|
+
* - A function: custom trigger. Called with `{ warm, micButton, mount }`
|
|
6674
|
+
* while the mic button exists (and again when the button is rebuilt or
|
|
6675
|
+
* `update()` swaps in a different function); call `warm()` whenever you
|
|
6676
|
+
* decide, e.g. when the panel opens. May return a cleanup that runs on
|
|
6677
|
+
* destroy, rebuild, or replacement. Replaces the built-in trigger.
|
|
6678
|
+
*
|
|
6679
|
+
* What `warm()` does depends on the provider: `runtype` fires its prewarm
|
|
6680
|
+
* (see `provider.runtype.prewarmMode`), a `custom` provider's optional
|
|
6681
|
+
* {@link VoiceProvider.prewarm} runs, and the browser (Web Speech) path
|
|
6682
|
+
* starts the client-token session init (as `sessionInit` does), since speech
|
|
6683
|
+
* recognition itself cannot start before the click. Warming is
|
|
6684
|
+
* fire-and-forget, latched, and never surfaces an error or UI state.
|
|
6685
|
+
*
|
|
6686
|
+
* @default 'hover' in client token mode, otherwise `false`
|
|
6687
|
+
*/
|
|
6688
|
+
prewarm?: AgentWidgetVoicePrewarmTrigger | AgentWidgetVoicePrewarmHook | false;
|
|
6650
6689
|
provider?: {
|
|
6651
6690
|
type: 'browser' | 'runtype' | 'custom';
|
|
6652
6691
|
browser?: {
|
|
@@ -6668,6 +6707,23 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6668
6707
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
6669
6708
|
*/
|
|
6670
6709
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
6710
|
+
/**
|
|
6711
|
+
* How `prewarm()` warms the voice path (see `voiceRecognition.prewarm`).
|
|
6712
|
+
*
|
|
6713
|
+
* - `'request'` (default): a fire-and-forget
|
|
6714
|
+
* `POST {host}/v1/client/agents/{agentId}/voice/prewarm`.
|
|
6715
|
+
* - `'attach'`: open the voice WebSocket ahead of the click without
|
|
6716
|
+
* starting a call; the click then starts the call on that socket. It can
|
|
6717
|
+
* create server work before the visitor clicks, so it is opt-in.
|
|
6718
|
+
* @default 'request'
|
|
6719
|
+
*/
|
|
6720
|
+
prewarmMode?: 'request' | 'attach';
|
|
6721
|
+
/**
|
|
6722
|
+
* `'attach'` mode: how long (ms) an attached socket waits for the click
|
|
6723
|
+
* before it closes. The server clamps it to 30000..600000.
|
|
6724
|
+
* @default 30000
|
|
6725
|
+
*/
|
|
6726
|
+
attachIdleMs?: number;
|
|
6671
6727
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
6672
6728
|
pauseDuration?: number;
|
|
6673
6729
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -6968,6 +7024,13 @@ interface VoiceProvider {
|
|
|
6968
7024
|
deactivateBargeIn?(): Promise<void>;
|
|
6969
7025
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
6970
7026
|
stopPlayback?(): void;
|
|
7027
|
+
/**
|
|
7028
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
7029
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
7030
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
7031
|
+
* repeatedly; throttle as needed.
|
|
7032
|
+
*/
|
|
7033
|
+
prewarm?(): void;
|
|
6971
7034
|
}
|
|
6972
7035
|
/**
|
|
6973
7036
|
* Extra context for an approval decision, surfaced to `onDecision` and to the
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{a as v}from"./chunk-5EIIHQLQ.js";var g=16e3,k=24e3,S=4096,C=4,E=1380533830,A=3e4,M=5e3,P=3e4,T=3e4,V=6e5;function _(r){return r.byteLength>=44&&new DataView(r).getUint32(0,!1)===E?new Uint8Array(r,44):new Uint8Array(r)}function w(r){let t=r.replace(/\/+$/,"");return/^wss?:\/\//i.test(t)?t:/^https?:\/\//i.test(t)?t.replace(/^http/i,"ws"):`${typeof window<"u"&&window.location?.protocol==="https:"?"wss:":"ws:"}//${t}`}function R(r){return w(r).replace(/^ws/i,"http")}function L(r){return typeof r!="number"||!Number.isFinite(r)?P:Math.min(V,Math.max(T,Math.round(r)))}var f=class{constructor(t){this.config=t;this.type="runtype";this.ws=null;this.captureContext=null;this.levelCallbacks=[];this.mediaStream=null;this.sourceNode=null;this.processor=null;this.playback=null;this.callLive=!1;this.isSpeaking=!1;this.callGeneration=0;this.intentionalClose=!1;this.lastPrewarmAt=Number.NEGATIVE_INFINITY;this.attached=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.transcriptCallbacks=[];this.metricsCallbacks=[]}async connect(){}prewarm(){if(this.callLive||this.attached)return;let t=this.config?.agentId,e=this.config?.clientToken,i=this.config?.host;if(!t||!e||!i)return;let s=Date.now();if(!(s-this.lastPrewarmAt<A)){this.lastPrewarmAt=s;try{this.config?.prewarmMode==="attach"?this.attach(i,t,e):this.requestPrewarm(i,t,e)}catch{}}}requestPrewarm(t,e,i){if(typeof fetch!="function")return;let s=`${R(t)}/v1/client/agents/${encodeURIComponent(e)}/voice/prewarm`;fetch(s,{method:"POST",headers:{Authorization:`Bearer ${i}`},keepalive:!0}).catch(()=>{})}attach(t,e,i){let s=new URLSearchParams({clientCapabilities:"attach"}),o=this.config?.attachIdleMs,h=L(o);o!==void 0&&s.set("attachIdleMs",String(h));let c=new WebSocket(`${this.voiceSocketUrl(t,e)}?${s}`,["runtype.bearer","runtype.attach",i]);c.binaryType="arraybuffer";let u=()=>{},d=new Promise(n=>{u=n}),a={ws:c,state:"connecting",ready:d,settle:u,openedAt:0,idleTimer:void 0};this.attached=a,c.onopen=()=>{this.attached===a&&(a.state=c.protocol==="runtype.attach"?"attached":"live",a.openedAt=Date.now(),this.armAttachIdle(a,h),a.state==="attached"&&c.send('{"type":"ping"}'),a.settle())},c.onmessage=n=>{if(this.attached!==a||typeof n.data!="string")return;let l;try{l=JSON.parse(n.data)}catch{return}l.type==="attached"&&typeof l.idleMs=="number"&&this.armAttachIdle(a,l.idleMs-(Date.now()-a.openedAt))},c.onerror=()=>{},c.onclose=()=>{this.attached===a&&this.releaseAttached()}}armAttachIdle(t,e){clearTimeout(t.idleTimer),t.idleTimer=setTimeout(()=>{this.attached===t&&this.releaseAttached()},Math.max(0,e))}releaseAttached(){let t=this.attached;if(!t)return;this.attached=null,clearTimeout(t.idleTimer);let{ws:e}=t;if(e.onopen=null,e.onmessage=null,e.onerror=null,e.onclose=null,t.settle(),e.readyState===WebSocket.CONNECTING||e.readyState===WebSocket.OPEN)try{e.close(1e3,"prewarm released")}catch{}}async takeAttachedSocket(){let t=this.attached;if(!t)return null;if(t.state==="connecting"){let e;await Promise.race([t.ready,new Promise(i=>{e=setTimeout(i,M)})]),clearTimeout(e)}return this.attached!==t?null:t.state==="connecting"||t.ws.readyState!==WebSocket.OPEN?(this.releaseAttached(),null):(this.attached=null,clearTimeout(t.idleTimer),{ws:t.ws,live:t.state==="live"})}voiceSocketUrl(t,e){return`${w(t)}/ws/agents/${encodeURIComponent(e)}/voice`}async startListening(){if(this.callLive)return;let t=this.config?.agentId,e=this.config?.clientToken,i=this.config?.host;if(!t)throw new Error("Runtype voice requires an agentId");if(!e)throw new Error("Runtype voice requires a clientToken");if(!i)throw new Error("Runtype voice requires a host (or widget apiUrl)");let s=++this.callGeneration;this.intentionalClose=!1,this.callLive=!0;try{let o=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:g,channelCount:1,echoCancellation:!0}});if(s!==this.callGeneration){o.getTracks().forEach(n=>n.stop());return}this.mediaStream=o;let h=window.AudioContext||window.webkitAudioContext,c=new h({sampleRate:g});c.state==="suspended"&&await c.resume().catch(()=>{}),this.captureContext=c;let u=this.config?.createPlaybackEngine?await this.config.createPlaybackEngine():new v(k);if(s!==this.callGeneration){u.destroy(),o.getTracks().forEach(n=>n.stop()),c.close().catch(()=>{});return}this.playback=u,u.onFinished(()=>{s===this.callGeneration&&(this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening"))});let d=await this.takeAttachedSocket();if(s!==this.callGeneration){d?.ws.close(1e3,"client ended call");return}if(d){let{ws:n}=d;this.ws=n,this.bindCallSocket(n,s),d.live||n.send('{"type":"start"}'),this.emitStatus("listening"),this.startCapture(c,o,n,s);return}let a=new WebSocket(this.voiceSocketUrl(i,t),["runtype.bearer",e]);a.binaryType="arraybuffer",this.ws=a,a.onopen=()=>{s===this.callGeneration&&(this.emitStatus("listening"),this.startCapture(c,o,a,s))},this.bindCallSocket(a,s)}catch(o){throw this.cleanup(),this.emitError(o),this.emitStatus("error"),o}}bindCallSocket(t,e){t.onmessage=i=>this.handleMessage(i,e),t.onerror=()=>{e===this.callGeneration&&(this.emitError(new Error("Voice connection failed")),this.emitStatus("error"),this.cleanup())},t.onclose=i=>{if(this.intentionalClose){this.intentionalClose=!1;return}if(e===this.callGeneration){if(i.code!==1e3){let s=i.code?` (code ${i.code})`:"";this.emitError(new Error(`Voice connection closed${s}`)),this.emitStatus("error")}else this.emitStatus("idle");this.cleanup()}}}async stopListening(){this.cleanup(),this.emitStatus("idle")}async disconnect(){this.cleanup(),this.emitStatus("disconnected"),this.resultCallbacks=[],this.errorCallbacks=[],this.statusCallbacks=[],this.transcriptCallbacks=[],this.metricsCallbacks=[]}stopPlayback(){this.playback&&this.playback.flush(),this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening")}getInterruptionMode(){return"barge-in"}isBargeInActive(){return this.callLive}onLevel(t){this.levelCallbacks.push(t)}async deactivateBargeIn(){this.cleanup(),this.emitStatus("idle")}startCapture(t,e,i,s){let o=t.createMediaStreamSource(e);this.sourceNode=o;let h=t.createScriptProcessor(S,1,1);this.processor=h,h.onaudioprocess=c=>{if(s!==this.callGeneration||i.readyState!==WebSocket.OPEN)return;let u=c.inputBuffer.getChannelData(0),d=new Int16Array(u.length),a=0;for(let n=0;n<u.length;n++){let l=Math.max(-1,Math.min(1,u[n]));a+=l*l,d[n]=l<0?l*32768:l*32767}if(i.send(d.buffer),this.levelCallbacks.length>0){let n=Math.sqrt(a/u.length),l=Math.max(0,Math.min(1,n*C));for(let b of this.levelCallbacks)b(l)}},o.connect(h),h.connect(t.destination)}handleMessage(t,e){if(e!==this.callGeneration)return;if(t.data instanceof ArrayBuffer){this.handleAudioFrame(t.data,e);return}let i;try{i=JSON.parse(t.data)}catch{return}switch(i.type){case"transcript_interim":this.emitStatus("listening"),this.emitTranscript("user",i.text??"",!1);break;case"transcript_final":{let s=i.role==="assistant"?"assistant":"user";this.emitStatus(s==="user"?"processing":"speaking"),this.emitTranscript(s,i.text??"",!0);break}case"audio_end":this.playback?this.playback.markStreamEnd():(this.isSpeaking=!1,this.emitStatus("listening"));break;case"metrics":this.emitMetrics({llmMs:i.llm_ms,ttsMs:i.tts_ms,firstAudioMs:i.first_audio_ms,totalMs:i.total_ms});break;case"error":this.emitError(new Error(i.error||"Voice error")),this.emitStatus("error");break}}handleAudioFrame(t,e){if(e!==this.callGeneration||!this.playback)return;let i=_(t);i.length!==0&&(this.isSpeaking||(this.isSpeaking=!0,this.emitStatus("speaking")),this.playback.enqueue(i))}cleanup(){if(this.callGeneration+=1,this.callLive=!1,this.isSpeaking=!1,this.releaseAttached(),this.processor&&(this.processor.onaudioprocess=null,this.processor.disconnect(),this.processor=null),this.sourceNode&&(this.sourceNode.disconnect(),this.sourceNode=null),this.mediaStream&&(this.mediaStream.getTracks().forEach(t=>t.stop()),this.mediaStream=null),this.captureContext&&(this.captureContext.close().catch(()=>{}),this.captureContext=null),this.playback&&(this.playback.destroy(),this.playback=null),this.ws){this.intentionalClose=!0;try{this.ws.close(1e3,"client ended call")}catch{}this.ws=null}}onResult(t){this.resultCallbacks.push(t)}onError(t){this.errorCallbacks.push(t)}onStatusChange(t){this.statusCallbacks.push(t)}onTranscript(t){this.transcriptCallbacks.push(t)}onMetrics(t){this.metricsCallbacks.push(t)}emitStatus(t){this.statusCallbacks.forEach(e=>e(t))}emitError(t){this.errorCallbacks.forEach(e=>e(t))}emitTranscript(t,e,i){this.transcriptCallbacks.forEach(s=>s(t,e,i))}emitMetrics(t){this.metricsCallbacks.forEach(e=>e(t))}};var p=class{constructor(t={}){this.config=t;this.type="browser";this.recognition=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.isListening=!1;this.w=typeof window<"u"?window:void 0}async connect(){this.statusCallbacks.forEach(t=>t("connected"))}async startListening(){try{if(this.isListening)throw new Error("Already listening");if(!this.w)throw new Error("Window object not available");let t=this.w.SpeechRecognition||this.w.webkitSpeechRecognition;if(!t)throw new Error("Browser speech recognition not supported");this.recognition=new t,this.recognition.lang=this.config?.language||"en-US",this.recognition.continuous=this.config?.continuous||!1,this.recognition.interimResults=!0,this.recognition.onresult=e=>{let i=Array.from(e.results).map(o=>o[0]).map(o=>o.transcript).join(""),s=e.results[e.results.length-1].isFinal;this.resultCallbacks.forEach(o=>o({text:i,confidence:s?.8:.5,provider:"browser"})),s&&!this.config?.continuous&&this.stopListening()},this.recognition.onerror=e=>{this.errorCallbacks.forEach(i=>i(new Error(e.error))),this.statusCallbacks.forEach(i=>i("error"))},this.recognition.onstart=()=>{this.isListening=!0,this.statusCallbacks.forEach(e=>e("listening"))},this.recognition.onend=()=>{this.isListening=!1,this.statusCallbacks.forEach(e=>e("idle"))},this.recognition.start()}catch(t){throw this.errorCallbacks.forEach(e=>e(t)),this.statusCallbacks.forEach(e=>e("error")),t}}async stopListening(){this.recognition&&(this.recognition.stop(),this.recognition=null),this.isListening=!1,this.statusCallbacks.forEach(t=>t("idle"))}onResult(t){this.resultCallbacks.push(t)}onError(t){this.errorCallbacks.push(t)}onStatusChange(t){this.statusCallbacks.push(t)}async disconnect(){await this.stopListening(),this.statusCallbacks.forEach(t=>t("disconnected"))}static isSupported(){return"SpeechRecognition"in window||"webkitSpeechRecognition"in window}};function m(r){switch(r.type){case"runtype":if(!r.runtype)throw new Error("Runtype voice provider requires configuration");return new f(r.runtype);case"browser":if(!p.isSupported())throw new Error("Browser speech recognition not supported");return new p(r.browser||{});case"custom":{let t=r.custom;if(!t)throw new Error("Custom voice provider requires a `custom` provider instance or factory");let e=typeof t=="function"?t():t;if(!e||typeof e.startListening!="function")throw new Error("Custom voice provider `custom` must be a VoiceProvider (or a factory returning one)");return e}default:throw new Error(`Unknown voice provider type: ${r.type}`)}}function y(r){if(r?.type==="custom"&&r.custom)return m({type:"custom",custom:r.custom});if(r?.type==="runtype"&&r.runtype)return m({type:"runtype",runtype:r.runtype});if(p.isSupported())return m({type:"browser",browser:r?.browser||{language:"en-US"}});throw new Error("No supported voice providers available")}function I(r){try{return y(r),!0}catch{return!1}}export{p as BrowserVoiceProvider,f as RuntypeVoiceProvider,y as createBestAvailableVoiceProvider,m as createVoiceProvider,I as isVoiceSupported};
|
package/dist/voice-runtime.cjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var g=Object.defineProperty;var C=Object.getOwnPropertyDescriptor;var k=Object.getOwnPropertyNames;var E=Object.prototype.hasOwnProperty;var x=(r,e)=>{for(var t in e)g(r,t,{get:e[t],enumerable:!0})},A=(r,e,t,i)=>{if(e&&typeof e=="object"||typeof e=="function")for(let s of k(e))!E.call(r,s)&&s!==t&&g(r,s,{get:()=>e[s],enumerable:!(i=C(e,s))||i.enumerable});return r};var V=r=>A(g({},"__esModule",{value:!0}),r);var F={};x(F,{BrowserVoiceProvider:()=>p,RuntypeVoiceProvider:()=>d,createBestAvailableVoiceProvider:()=>b,createVoiceProvider:()=>f,isVoiceSupported:()=>w});module.exports=V(F);var v=class{constructor(e=24e3,t={}){this.ctx=null;this.nextStartTime=0;this.activeSources=[];this.finishedCallbacks=[];this.startedCallbacks=[];this.playing=!1;this.streamEnded=!1;this.pendingCount=0;this.started=!1;this.userPaused=!1;this.pendingBuffers=[];this.pendingSamples=0;this.remainder=null;this.sampleRate=e;let i=Math.max(0,t.prebufferMs??0);this.waterlineSamples=Math.round(e*i/1e3),this.buffering=this.waterlineSamples>0}ensureContext(){if(!this.ctx){let t=typeof window<"u"?window:void 0;if(!t)throw new Error("AudioPlaybackManager requires a browser environment");let i=t.AudioContext||t.webkitAudioContext;this.ctx=new i({sampleRate:this.sampleRate})}let e=this.ctx;return e.state==="suspended"&&!this.userPaused&&e.resume(),e}enqueue(e){if(e.length===0)return;let t=e;if(this.remainder){let s=new Uint8Array(this.remainder.length+e.length);s.set(this.remainder),s.set(e,this.remainder.length),t=s,this.remainder=null}if(t.length%2!==0&&(this.remainder=new Uint8Array([t[t.length-1]]),t=t.subarray(0,t.length-1)),t.length===0)return;let i=this.pcmToFloat32(t);i.length!==0&&(this.buffering?(this.pendingBuffers.push(i),this.pendingSamples+=i.length,this.pendingSamples>=this.waterlineSamples&&this.releaseBuffer()):this.scheduleSamples(i))}markStreamEnd(){this.pendingBuffers.length>0&&this.releaseBuffer(),this.streamEnded=!0,this.checkFinished()}flush(){for(let e of this.activeSources)try{e.stop(),e.disconnect()}catch{}this.activeSources=[],this.pendingCount=0,this.nextStartTime=0,this.playing=!1,this.streamEnded=!1,this.finishedCallbacks=[],this.startedCallbacks=[],this.remainder=null,this.pendingBuffers=[],this.pendingSamples=0,this.buffering=this.waterlineSamples>0,this.started=!1}isPlaying(){return this.playing}onFinished(e){this.finishedCallbacks.push(e)}onStarted(e){this.startedCallbacks.push(e)}pause(){this.userPaused=!0,this.ctx&&this.ctx.state==="running"&&this.ctx.suspend()}resume(){this.userPaused=!1,this.ctx&&this.ctx.state==="suspended"&&this.ctx.resume()}async destroy(){this.flush(),this.ctx&&(await this.ctx.close(),this.ctx=null)}releaseBuffer(){this.buffering=!1;let e=this.pendingBuffers;this.pendingBuffers=[],this.pendingSamples=0;for(let t of e)this.scheduleSamples(t)}scheduleSamples(e){if(e.length===0)return;let t=this.ensureContext(),i=t.createBuffer(1,e.length,this.sampleRate);i.getChannelData(0).set(e);let s=t.createBufferSource();s.buffer=i,s.connect(t.destination);let n=t.currentTime;if(this.nextStartTime===0?this.nextStartTime=n:this.nextStartTime<n&&(this.nextStartTime=n,this.waterlineSamples>0&&(this.buffering=!0)),s.start(this.nextStartTime),this.nextStartTime+=i.duration,this.activeSources.push(s),this.pendingCount++,this.playing=!0,!this.started){this.started=!0;let a=this.startedCallbacks.slice();this.startedCallbacks=[];for(let l of a)l()}s.onended=()=>{let a=this.activeSources.indexOf(s);a!==-1&&this.activeSources.splice(a,1),this.pendingCount--,this.checkFinished()}}checkFinished(){if(this.streamEnded&&this.pendingCount<=0&&this.pendingBuffers.length===0){this.playing=!1,this.streamEnded=!1;let e=this.finishedCallbacks.slice();this.finishedCallbacks=[];for(let t of e)t()}}pcmToFloat32(e){let t=Math.floor(e.length/2),i=new Float32Array(t),s=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let n=0;n<t;n++){let a=s.getInt16(n*2,!0);i[n]=a/32768}return i}};var y=16e3,P=24e3,M=4096,R=4,B=1380533830;function L(r){return r.byteLength>=44&&new DataView(r).getUint32(0,!1)===B?new Uint8Array(r,44):new Uint8Array(r)}function T(r){let e=r.replace(/\/+$/,"");return/^wss?:\/\//i.test(e)?e:/^https?:\/\//i.test(e)?e.replace(/^http/i,"ws"):`${typeof window<"u"&&window.location?.protocol==="https:"?"wss:":"ws:"}//${e}`}var d=class{constructor(e){this.config=e;this.type="runtype";this.ws=null;this.captureContext=null;this.levelCallbacks=[];this.mediaStream=null;this.sourceNode=null;this.processor=null;this.playback=null;this.callLive=!1;this.isSpeaking=!1;this.callGeneration=0;this.intentionalClose=!1;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.transcriptCallbacks=[];this.metricsCallbacks=[]}async connect(){}async startListening(){if(this.callLive)return;let e=this.config?.agentId,t=this.config?.clientToken,i=this.config?.host;if(!e)throw new Error("Runtype voice requires an agentId");if(!t)throw new Error("Runtype voice requires a clientToken");if(!i)throw new Error("Runtype voice requires a host (or widget apiUrl)");let s=++this.callGeneration;this.intentionalClose=!1,this.callLive=!0;try{let n=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:y,channelCount:1,echoCancellation:!0}});if(s!==this.callGeneration){n.getTracks().forEach(o=>o.stop());return}this.mediaStream=n;let a=window.AudioContext||window.webkitAudioContext,l=new a({sampleRate:y});l.state==="suspended"&&await l.resume().catch(()=>{}),this.captureContext=l;let h=this.config?.createPlaybackEngine?await this.config.createPlaybackEngine():new v(P);if(s!==this.callGeneration){h.destroy(),n.getTracks().forEach(o=>o.stop()),l.close().catch(()=>{});return}this.playback=h,h.onFinished(()=>{s===this.callGeneration&&(this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening"))});let m=`${T(i)}/ws/agents/${encodeURIComponent(e)}/voice`,c=new WebSocket(m,["runtype.bearer",t]);c.binaryType="arraybuffer",this.ws=c,c.onopen=()=>{s===this.callGeneration&&(this.emitStatus("listening"),this.startCapture(l,n,c,s))},c.onmessage=o=>this.handleMessage(o,s),c.onerror=()=>{s===this.callGeneration&&(this.emitError(new Error("Voice connection failed")),this.emitStatus("error"),this.cleanup())},c.onclose=o=>{if(this.intentionalClose){this.intentionalClose=!1;return}if(s===this.callGeneration){if(o.code!==1e3){let u=o.code?` (code ${o.code})`:"";this.emitError(new Error(`Voice connection closed${u}`)),this.emitStatus("error")}else this.emitStatus("idle");this.cleanup()}}}catch(n){throw this.cleanup(),this.emitError(n),this.emitStatus("error"),n}}async stopListening(){this.cleanup(),this.emitStatus("idle")}async disconnect(){this.cleanup(),this.emitStatus("disconnected"),this.resultCallbacks=[],this.errorCallbacks=[],this.statusCallbacks=[],this.transcriptCallbacks=[],this.metricsCallbacks=[]}stopPlayback(){this.playback&&this.playback.flush(),this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening")}getInterruptionMode(){return"barge-in"}isBargeInActive(){return this.callLive}onLevel(e){this.levelCallbacks.push(e)}async deactivateBargeIn(){this.cleanup(),this.emitStatus("idle")}startCapture(e,t,i,s){let n=e.createMediaStreamSource(t);this.sourceNode=n;let a=e.createScriptProcessor(M,1,1);this.processor=a,a.onaudioprocess=l=>{if(s!==this.callGeneration||i.readyState!==WebSocket.OPEN)return;let h=l.inputBuffer.getChannelData(0),m=new Int16Array(h.length),c=0;for(let o=0;o<h.length;o++){let u=Math.max(-1,Math.min(1,h[o]));c+=u*u,m[o]=u<0?u*32768:u*32767}if(i.send(m.buffer),this.levelCallbacks.length>0){let o=Math.sqrt(c/h.length),u=Math.max(0,Math.min(1,o*R));for(let S of this.levelCallbacks)S(u)}},n.connect(a),a.connect(e.destination)}handleMessage(e,t){if(t!==this.callGeneration)return;if(e.data instanceof ArrayBuffer){this.handleAudioFrame(e.data,t);return}let i;try{i=JSON.parse(e.data)}catch{return}switch(i.type){case"transcript_interim":this.emitStatus("listening"),this.emitTranscript("user",i.text??"",!1);break;case"transcript_final":{let s=i.role==="assistant"?"assistant":"user";this.emitStatus(s==="user"?"processing":"speaking"),this.emitTranscript(s,i.text??"",!0);break}case"audio_end":this.playback?this.playback.markStreamEnd():(this.isSpeaking=!1,this.emitStatus("listening"));break;case"metrics":this.emitMetrics({llmMs:i.llm_ms,ttsMs:i.tts_ms,firstAudioMs:i.first_audio_ms,totalMs:i.total_ms});break;case"error":this.emitError(new Error(i.error||"Voice error")),this.emitStatus("error");break}}handleAudioFrame(e,t){if(t!==this.callGeneration||!this.playback)return;let i=L(e);i.length!==0&&(this.isSpeaking||(this.isSpeaking=!0,this.emitStatus("speaking")),this.playback.enqueue(i))}cleanup(){if(this.callGeneration+=1,this.callLive=!1,this.isSpeaking=!1,this.processor&&(this.processor.onaudioprocess=null,this.processor.disconnect(),this.processor=null),this.sourceNode&&(this.sourceNode.disconnect(),this.sourceNode=null),this.mediaStream&&(this.mediaStream.getTracks().forEach(e=>e.stop()),this.mediaStream=null),this.captureContext&&(this.captureContext.close().catch(()=>{}),this.captureContext=null),this.playback&&(this.playback.destroy(),this.playback=null),this.ws){this.intentionalClose=!0;try{this.ws.close(1e3,"client ended call")}catch{}this.ws=null}}onResult(e){this.resultCallbacks.push(e)}onError(e){this.errorCallbacks.push(e)}onStatusChange(e){this.statusCallbacks.push(e)}onTranscript(e){this.transcriptCallbacks.push(e)}onMetrics(e){this.metricsCallbacks.push(e)}emitStatus(e){this.statusCallbacks.forEach(t=>t(e))}emitError(e){this.errorCallbacks.forEach(t=>t(e))}emitTranscript(e,t,i){this.transcriptCallbacks.forEach(s=>s(e,t,i))}emitMetrics(e){this.metricsCallbacks.forEach(t=>t(e))}};var p=class{constructor(e={}){this.config=e;this.type="browser";this.recognition=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.isListening=!1;this.w=typeof window<"u"?window:void 0}async connect(){this.statusCallbacks.forEach(e=>e("connected"))}async startListening(){try{if(this.isListening)throw new Error("Already listening");if(!this.w)throw new Error("Window object not available");let e=this.w.SpeechRecognition||this.w.webkitSpeechRecognition;if(!e)throw new Error("Browser speech recognition not supported");this.recognition=new e,this.recognition.lang=this.config?.language||"en-US",this.recognition.continuous=this.config?.continuous||!1,this.recognition.interimResults=!0,this.recognition.onresult=t=>{let i=Array.from(t.results).map(n=>n[0]).map(n=>n.transcript).join(""),s=t.results[t.results.length-1].isFinal;this.resultCallbacks.forEach(n=>n({text:i,confidence:s?.8:.5,provider:"browser"})),s&&!this.config?.continuous&&this.stopListening()},this.recognition.onerror=t=>{this.errorCallbacks.forEach(i=>i(new Error(t.error))),this.statusCallbacks.forEach(i=>i("error"))},this.recognition.onstart=()=>{this.isListening=!0,this.statusCallbacks.forEach(t=>t("listening"))},this.recognition.onend=()=>{this.isListening=!1,this.statusCallbacks.forEach(t=>t("idle"))},this.recognition.start()}catch(e){throw this.errorCallbacks.forEach(t=>t(e)),this.statusCallbacks.forEach(t=>t("error")),e}}async stopListening(){this.recognition&&(this.recognition.stop(),this.recognition=null),this.isListening=!1,this.statusCallbacks.forEach(e=>e("idle"))}onResult(e){this.resultCallbacks.push(e)}onError(e){this.errorCallbacks.push(e)}onStatusChange(e){this.statusCallbacks.push(e)}async disconnect(){await this.stopListening(),this.statusCallbacks.forEach(e=>e("disconnected"))}static isSupported(){return"SpeechRecognition"in window||"webkitSpeechRecognition"in window}};function f(r){switch(r.type){case"runtype":if(!r.runtype)throw new Error("Runtype voice provider requires configuration");return new d(r.runtype);case"browser":if(!p.isSupported())throw new Error("Browser speech recognition not supported");return new p(r.browser||{});case"custom":{let e=r.custom;if(!e)throw new Error("Custom voice provider requires a `custom` provider instance or factory");let t=typeof e=="function"?e():e;if(!t||typeof t.startListening!="function")throw new Error("Custom voice provider `custom` must be a VoiceProvider (or a factory returning one)");return t}default:throw new Error(`Unknown voice provider type: ${r.type}`)}}function b(r){if(r?.type==="custom"&&r.custom)return f({type:"custom",custom:r.custom});if(r?.type==="runtype"&&r.runtype)return f({type:"runtype",runtype:r.runtype});if(p.isSupported())return f({type:"browser",browser:r?.browser||{language:"en-US"}});throw new Error("No supported voice providers available")}function w(r){try{return b(r),!0}catch{return!1}}0&&(module.exports={BrowserVoiceProvider,RuntypeVoiceProvider,createBestAvailableVoiceProvider,createVoiceProvider,isVoiceSupported});
|
|
1
|
+
"use strict";var g=Object.defineProperty;var C=Object.getOwnPropertyDescriptor;var E=Object.getOwnPropertyNames;var A=Object.prototype.hasOwnProperty;var x=(r,t)=>{for(var e in t)g(r,e,{get:t[e],enumerable:!0})},P=(r,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of E(t))!A.call(r,s)&&s!==e&&g(r,s,{get:()=>t[s],enumerable:!(i=C(t,s))||i.enumerable});return r};var T=r=>P(g({},"__esModule",{value:!0}),r);var O={};x(O,{BrowserVoiceProvider:()=>p,RuntypeVoiceProvider:()=>f,createBestAvailableVoiceProvider:()=>b,createVoiceProvider:()=>m,isVoiceSupported:()=>S});module.exports=T(O);var v=class{constructor(t=24e3,e={}){this.ctx=null;this.nextStartTime=0;this.activeSources=[];this.finishedCallbacks=[];this.startedCallbacks=[];this.playing=!1;this.streamEnded=!1;this.pendingCount=0;this.started=!1;this.userPaused=!1;this.pendingBuffers=[];this.pendingSamples=0;this.remainder=null;this.sampleRate=t;let i=Math.max(0,e.prebufferMs??0);this.waterlineSamples=Math.round(t*i/1e3),this.buffering=this.waterlineSamples>0}ensureContext(){if(!this.ctx){let e=typeof window<"u"?window:void 0;if(!e)throw new Error("AudioPlaybackManager requires a browser environment");let i=e.AudioContext||e.webkitAudioContext;this.ctx=new i({sampleRate:this.sampleRate})}let t=this.ctx;return t.state==="suspended"&&!this.userPaused&&t.resume(),t}enqueue(t){if(t.length===0)return;let e=t;if(this.remainder){let s=new Uint8Array(this.remainder.length+t.length);s.set(this.remainder),s.set(t,this.remainder.length),e=s,this.remainder=null}if(e.length%2!==0&&(this.remainder=new Uint8Array([e[e.length-1]]),e=e.subarray(0,e.length-1)),e.length===0)return;let i=this.pcmToFloat32(e);i.length!==0&&(this.buffering?(this.pendingBuffers.push(i),this.pendingSamples+=i.length,this.pendingSamples>=this.waterlineSamples&&this.releaseBuffer()):this.scheduleSamples(i))}markStreamEnd(){this.pendingBuffers.length>0&&this.releaseBuffer(),this.streamEnded=!0,this.checkFinished()}flush(){for(let t of this.activeSources)try{t.stop(),t.disconnect()}catch{}this.activeSources=[],this.pendingCount=0,this.nextStartTime=0,this.playing=!1,this.streamEnded=!1,this.finishedCallbacks=[],this.startedCallbacks=[],this.remainder=null,this.pendingBuffers=[],this.pendingSamples=0,this.buffering=this.waterlineSamples>0,this.started=!1}isPlaying(){return this.playing}onFinished(t){this.finishedCallbacks.push(t)}onStarted(t){this.startedCallbacks.push(t)}pause(){this.userPaused=!0,this.ctx&&this.ctx.state==="running"&&this.ctx.suspend()}resume(){this.userPaused=!1,this.ctx&&this.ctx.state==="suspended"&&this.ctx.resume()}async destroy(){this.flush(),this.ctx&&(await this.ctx.close(),this.ctx=null)}releaseBuffer(){this.buffering=!1;let t=this.pendingBuffers;this.pendingBuffers=[],this.pendingSamples=0;for(let e of t)this.scheduleSamples(e)}scheduleSamples(t){if(t.length===0)return;let e=this.ensureContext(),i=e.createBuffer(1,t.length,this.sampleRate);i.getChannelData(0).set(t);let s=e.createBufferSource();s.buffer=i,s.connect(e.destination);let n=e.currentTime;if(this.nextStartTime===0?this.nextStartTime=n:this.nextStartTime<n&&(this.nextStartTime=n,this.waterlineSamples>0&&(this.buffering=!0)),s.start(this.nextStartTime),this.nextStartTime+=i.duration,this.activeSources.push(s),this.pendingCount++,this.playing=!0,!this.started){this.started=!0;let l=this.startedCallbacks.slice();this.startedCallbacks=[];for(let c of l)c()}s.onended=()=>{let l=this.activeSources.indexOf(s);l!==-1&&this.activeSources.splice(l,1),this.pendingCount--,this.checkFinished()}}checkFinished(){if(this.streamEnded&&this.pendingCount<=0&&this.pendingBuffers.length===0){this.playing=!1,this.streamEnded=!1;let t=this.finishedCallbacks.slice();this.finishedCallbacks=[];for(let e of t)e()}}pcmToFloat32(t){let e=Math.floor(t.length/2),i=new Float32Array(e),s=new DataView(t.buffer,t.byteOffset,t.byteLength);for(let n=0;n<e;n++){let l=s.getInt16(n*2,!0);i[n]=l/32768}return i}};var y=16e3,M=24e3,V=4096,R=4,_=1380533830,L=3e4,I=5e3,B=3e4,N=3e4,U=6e5;function F(r){return r.byteLength>=44&&new DataView(r).getUint32(0,!1)===_?new Uint8Array(r,44):new Uint8Array(r)}function w(r){let t=r.replace(/\/+$/,"");return/^wss?:\/\//i.test(t)?t:/^https?:\/\//i.test(t)?t.replace(/^http/i,"ws"):`${typeof window<"u"&&window.location?.protocol==="https:"?"wss:":"ws:"}//${t}`}function W(r){return w(r).replace(/^ws/i,"http")}function G(r){return typeof r!="number"||!Number.isFinite(r)?B:Math.min(U,Math.max(N,Math.round(r)))}var f=class{constructor(t){this.config=t;this.type="runtype";this.ws=null;this.captureContext=null;this.levelCallbacks=[];this.mediaStream=null;this.sourceNode=null;this.processor=null;this.playback=null;this.callLive=!1;this.isSpeaking=!1;this.callGeneration=0;this.intentionalClose=!1;this.lastPrewarmAt=Number.NEGATIVE_INFINITY;this.attached=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.transcriptCallbacks=[];this.metricsCallbacks=[]}async connect(){}prewarm(){if(this.callLive||this.attached)return;let t=this.config?.agentId,e=this.config?.clientToken,i=this.config?.host;if(!t||!e||!i)return;let s=Date.now();if(!(s-this.lastPrewarmAt<L)){this.lastPrewarmAt=s;try{this.config?.prewarmMode==="attach"?this.attach(i,t,e):this.requestPrewarm(i,t,e)}catch{}}}requestPrewarm(t,e,i){if(typeof fetch!="function")return;let s=`${W(t)}/v1/client/agents/${encodeURIComponent(e)}/voice/prewarm`;fetch(s,{method:"POST",headers:{Authorization:`Bearer ${i}`},keepalive:!0}).catch(()=>{})}attach(t,e,i){let s=new URLSearchParams({clientCapabilities:"attach"}),n=this.config?.attachIdleMs,l=G(n);n!==void 0&&s.set("attachIdleMs",String(l));let c=new WebSocket(`${this.voiceSocketUrl(t,e)}?${s}`,["runtype.bearer","runtype.attach",i]);c.binaryType="arraybuffer";let u=()=>{},d=new Promise(o=>{u=o}),a={ws:c,state:"connecting",ready:d,settle:u,openedAt:0,idleTimer:void 0};this.attached=a,c.onopen=()=>{this.attached===a&&(a.state=c.protocol==="runtype.attach"?"attached":"live",a.openedAt=Date.now(),this.armAttachIdle(a,l),a.state==="attached"&&c.send('{"type":"ping"}'),a.settle())},c.onmessage=o=>{if(this.attached!==a||typeof o.data!="string")return;let h;try{h=JSON.parse(o.data)}catch{return}h.type==="attached"&&typeof h.idleMs=="number"&&this.armAttachIdle(a,h.idleMs-(Date.now()-a.openedAt))},c.onerror=()=>{},c.onclose=()=>{this.attached===a&&this.releaseAttached()}}armAttachIdle(t,e){clearTimeout(t.idleTimer),t.idleTimer=setTimeout(()=>{this.attached===t&&this.releaseAttached()},Math.max(0,e))}releaseAttached(){let t=this.attached;if(!t)return;this.attached=null,clearTimeout(t.idleTimer);let{ws:e}=t;if(e.onopen=null,e.onmessage=null,e.onerror=null,e.onclose=null,t.settle(),e.readyState===WebSocket.CONNECTING||e.readyState===WebSocket.OPEN)try{e.close(1e3,"prewarm released")}catch{}}async takeAttachedSocket(){let t=this.attached;if(!t)return null;if(t.state==="connecting"){let e;await Promise.race([t.ready,new Promise(i=>{e=setTimeout(i,I)})]),clearTimeout(e)}return this.attached!==t?null:t.state==="connecting"||t.ws.readyState!==WebSocket.OPEN?(this.releaseAttached(),null):(this.attached=null,clearTimeout(t.idleTimer),{ws:t.ws,live:t.state==="live"})}voiceSocketUrl(t,e){return`${w(t)}/ws/agents/${encodeURIComponent(e)}/voice`}async startListening(){if(this.callLive)return;let t=this.config?.agentId,e=this.config?.clientToken,i=this.config?.host;if(!t)throw new Error("Runtype voice requires an agentId");if(!e)throw new Error("Runtype voice requires a clientToken");if(!i)throw new Error("Runtype voice requires a host (or widget apiUrl)");let s=++this.callGeneration;this.intentionalClose=!1,this.callLive=!0;try{let n=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:y,channelCount:1,echoCancellation:!0}});if(s!==this.callGeneration){n.getTracks().forEach(o=>o.stop());return}this.mediaStream=n;let l=window.AudioContext||window.webkitAudioContext,c=new l({sampleRate:y});c.state==="suspended"&&await c.resume().catch(()=>{}),this.captureContext=c;let u=this.config?.createPlaybackEngine?await this.config.createPlaybackEngine():new v(M);if(s!==this.callGeneration){u.destroy(),n.getTracks().forEach(o=>o.stop()),c.close().catch(()=>{});return}this.playback=u,u.onFinished(()=>{s===this.callGeneration&&(this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening"))});let d=await this.takeAttachedSocket();if(s!==this.callGeneration){d?.ws.close(1e3,"client ended call");return}if(d){let{ws:o}=d;this.ws=o,this.bindCallSocket(o,s),d.live||o.send('{"type":"start"}'),this.emitStatus("listening"),this.startCapture(c,n,o,s);return}let a=new WebSocket(this.voiceSocketUrl(i,t),["runtype.bearer",e]);a.binaryType="arraybuffer",this.ws=a,a.onopen=()=>{s===this.callGeneration&&(this.emitStatus("listening"),this.startCapture(c,n,a,s))},this.bindCallSocket(a,s)}catch(n){throw this.cleanup(),this.emitError(n),this.emitStatus("error"),n}}bindCallSocket(t,e){t.onmessage=i=>this.handleMessage(i,e),t.onerror=()=>{e===this.callGeneration&&(this.emitError(new Error("Voice connection failed")),this.emitStatus("error"),this.cleanup())},t.onclose=i=>{if(this.intentionalClose){this.intentionalClose=!1;return}if(e===this.callGeneration){if(i.code!==1e3){let s=i.code?` (code ${i.code})`:"";this.emitError(new Error(`Voice connection closed${s}`)),this.emitStatus("error")}else this.emitStatus("idle");this.cleanup()}}}async stopListening(){this.cleanup(),this.emitStatus("idle")}async disconnect(){this.cleanup(),this.emitStatus("disconnected"),this.resultCallbacks=[],this.errorCallbacks=[],this.statusCallbacks=[],this.transcriptCallbacks=[],this.metricsCallbacks=[]}stopPlayback(){this.playback&&this.playback.flush(),this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening")}getInterruptionMode(){return"barge-in"}isBargeInActive(){return this.callLive}onLevel(t){this.levelCallbacks.push(t)}async deactivateBargeIn(){this.cleanup(),this.emitStatus("idle")}startCapture(t,e,i,s){let n=t.createMediaStreamSource(e);this.sourceNode=n;let l=t.createScriptProcessor(V,1,1);this.processor=l,l.onaudioprocess=c=>{if(s!==this.callGeneration||i.readyState!==WebSocket.OPEN)return;let u=c.inputBuffer.getChannelData(0),d=new Int16Array(u.length),a=0;for(let o=0;o<u.length;o++){let h=Math.max(-1,Math.min(1,u[o]));a+=h*h,d[o]=h<0?h*32768:h*32767}if(i.send(d.buffer),this.levelCallbacks.length>0){let o=Math.sqrt(a/u.length),h=Math.max(0,Math.min(1,o*R));for(let k of this.levelCallbacks)k(h)}},n.connect(l),l.connect(t.destination)}handleMessage(t,e){if(e!==this.callGeneration)return;if(t.data instanceof ArrayBuffer){this.handleAudioFrame(t.data,e);return}let i;try{i=JSON.parse(t.data)}catch{return}switch(i.type){case"transcript_interim":this.emitStatus("listening"),this.emitTranscript("user",i.text??"",!1);break;case"transcript_final":{let s=i.role==="assistant"?"assistant":"user";this.emitStatus(s==="user"?"processing":"speaking"),this.emitTranscript(s,i.text??"",!0);break}case"audio_end":this.playback?this.playback.markStreamEnd():(this.isSpeaking=!1,this.emitStatus("listening"));break;case"metrics":this.emitMetrics({llmMs:i.llm_ms,ttsMs:i.tts_ms,firstAudioMs:i.first_audio_ms,totalMs:i.total_ms});break;case"error":this.emitError(new Error(i.error||"Voice error")),this.emitStatus("error");break}}handleAudioFrame(t,e){if(e!==this.callGeneration||!this.playback)return;let i=F(t);i.length!==0&&(this.isSpeaking||(this.isSpeaking=!0,this.emitStatus("speaking")),this.playback.enqueue(i))}cleanup(){if(this.callGeneration+=1,this.callLive=!1,this.isSpeaking=!1,this.releaseAttached(),this.processor&&(this.processor.onaudioprocess=null,this.processor.disconnect(),this.processor=null),this.sourceNode&&(this.sourceNode.disconnect(),this.sourceNode=null),this.mediaStream&&(this.mediaStream.getTracks().forEach(t=>t.stop()),this.mediaStream=null),this.captureContext&&(this.captureContext.close().catch(()=>{}),this.captureContext=null),this.playback&&(this.playback.destroy(),this.playback=null),this.ws){this.intentionalClose=!0;try{this.ws.close(1e3,"client ended call")}catch{}this.ws=null}}onResult(t){this.resultCallbacks.push(t)}onError(t){this.errorCallbacks.push(t)}onStatusChange(t){this.statusCallbacks.push(t)}onTranscript(t){this.transcriptCallbacks.push(t)}onMetrics(t){this.metricsCallbacks.push(t)}emitStatus(t){this.statusCallbacks.forEach(e=>e(t))}emitError(t){this.errorCallbacks.forEach(e=>e(t))}emitTranscript(t,e,i){this.transcriptCallbacks.forEach(s=>s(t,e,i))}emitMetrics(t){this.metricsCallbacks.forEach(e=>e(t))}};var p=class{constructor(t={}){this.config=t;this.type="browser";this.recognition=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.isListening=!1;this.w=typeof window<"u"?window:void 0}async connect(){this.statusCallbacks.forEach(t=>t("connected"))}async startListening(){try{if(this.isListening)throw new Error("Already listening");if(!this.w)throw new Error("Window object not available");let t=this.w.SpeechRecognition||this.w.webkitSpeechRecognition;if(!t)throw new Error("Browser speech recognition not supported");this.recognition=new t,this.recognition.lang=this.config?.language||"en-US",this.recognition.continuous=this.config?.continuous||!1,this.recognition.interimResults=!0,this.recognition.onresult=e=>{let i=Array.from(e.results).map(n=>n[0]).map(n=>n.transcript).join(""),s=e.results[e.results.length-1].isFinal;this.resultCallbacks.forEach(n=>n({text:i,confidence:s?.8:.5,provider:"browser"})),s&&!this.config?.continuous&&this.stopListening()},this.recognition.onerror=e=>{this.errorCallbacks.forEach(i=>i(new Error(e.error))),this.statusCallbacks.forEach(i=>i("error"))},this.recognition.onstart=()=>{this.isListening=!0,this.statusCallbacks.forEach(e=>e("listening"))},this.recognition.onend=()=>{this.isListening=!1,this.statusCallbacks.forEach(e=>e("idle"))},this.recognition.start()}catch(t){throw this.errorCallbacks.forEach(e=>e(t)),this.statusCallbacks.forEach(e=>e("error")),t}}async stopListening(){this.recognition&&(this.recognition.stop(),this.recognition=null),this.isListening=!1,this.statusCallbacks.forEach(t=>t("idle"))}onResult(t){this.resultCallbacks.push(t)}onError(t){this.errorCallbacks.push(t)}onStatusChange(t){this.statusCallbacks.push(t)}async disconnect(){await this.stopListening(),this.statusCallbacks.forEach(t=>t("disconnected"))}static isSupported(){return"SpeechRecognition"in window||"webkitSpeechRecognition"in window}};function m(r){switch(r.type){case"runtype":if(!r.runtype)throw new Error("Runtype voice provider requires configuration");return new f(r.runtype);case"browser":if(!p.isSupported())throw new Error("Browser speech recognition not supported");return new p(r.browser||{});case"custom":{let t=r.custom;if(!t)throw new Error("Custom voice provider requires a `custom` provider instance or factory");let e=typeof t=="function"?t():t;if(!e||typeof e.startListening!="function")throw new Error("Custom voice provider `custom` must be a VoiceProvider (or a factory returning one)");return e}default:throw new Error(`Unknown voice provider type: ${r.type}`)}}function b(r){if(r?.type==="custom"&&r.custom)return m({type:"custom",custom:r.custom});if(r?.type==="runtype"&&r.runtype)return m({type:"runtype",runtype:r.runtype});if(p.isSupported())return m({type:"browser",browser:r?.browser||{language:"en-US"}});throw new Error("No supported voice providers available")}function S(r){try{return b(r),!0}catch{return!1}}0&&(module.exports={BrowserVoiceProvider,RuntypeVoiceProvider,createBestAvailableVoiceProvider,createVoiceProvider,isVoiceSupported});
|
package/dist/voice-runtime.d.cts
CHANGED
|
@@ -77,6 +77,10 @@ type VoiceConfig = {
|
|
|
77
77
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
78
78
|
*/
|
|
79
79
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
80
|
+
/** See `voiceRecognition.provider.runtype.prewarmMode`. @default 'request' */
|
|
81
|
+
prewarmMode?: 'request' | 'attach';
|
|
82
|
+
/** See `voiceRecognition.provider.runtype.attachIdleMs`. @default 30000 */
|
|
83
|
+
attachIdleMs?: number;
|
|
80
84
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
81
85
|
pauseDuration?: number;
|
|
82
86
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -137,6 +141,13 @@ interface VoiceProvider {
|
|
|
137
141
|
deactivateBargeIn?(): Promise<void>;
|
|
138
142
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
139
143
|
stopPlayback?(): void;
|
|
144
|
+
/**
|
|
145
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
146
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
147
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
148
|
+
* repeatedly; throttle as needed.
|
|
149
|
+
*/
|
|
150
|
+
prewarm?(): void;
|
|
140
151
|
}
|
|
141
152
|
|
|
142
153
|
declare function createVoiceProvider(config: VoiceConfig): VoiceProvider;
|
|
@@ -157,6 +168,8 @@ declare class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
157
168
|
private isSpeaking;
|
|
158
169
|
private callGeneration;
|
|
159
170
|
private intentionalClose;
|
|
171
|
+
private lastPrewarmAt;
|
|
172
|
+
private attached;
|
|
160
173
|
private resultCallbacks;
|
|
161
174
|
private errorCallbacks;
|
|
162
175
|
private statusCallbacks;
|
|
@@ -165,8 +178,28 @@ declare class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
165
178
|
constructor(config: VoiceConfig["runtype"]);
|
|
166
179
|
/** No-op: the WS session opens lazily in `startListening` (the "call"). */
|
|
167
180
|
connect(): Promise<void>;
|
|
181
|
+
/**
|
|
182
|
+
* Warm the voice path ahead of the click. Fire-and-forget and throttled to
|
|
183
|
+
* once per {@link PREWARM_THROTTLE_MS}; never emits an error or a status.
|
|
184
|
+
*/
|
|
185
|
+
prewarm(): void;
|
|
186
|
+
private requestPrewarm;
|
|
187
|
+
private attach;
|
|
188
|
+
private armAttachIdle;
|
|
189
|
+
/** Close and forget the prewarmed socket, if any. Silent by design. */
|
|
190
|
+
private releaseAttached;
|
|
191
|
+
/**
|
|
192
|
+
* Hand the prewarmed socket to the call, waiting up to
|
|
193
|
+
* {@link ATTACH_HANDSHAKE_TIMEOUT_MS} for a handshake still in flight.
|
|
194
|
+
* Resolves `null` when there is none, it stalled, or it dropped, so the
|
|
195
|
+
* caller opens a fresh socket.
|
|
196
|
+
*/
|
|
197
|
+
private takeAttachedSocket;
|
|
198
|
+
private voiceSocketUrl;
|
|
168
199
|
/** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
|
|
169
200
|
startListening(): Promise<void>;
|
|
201
|
+
/** Route a call socket's frames, errors, and close into this call. */
|
|
202
|
+
private bindCallSocket;
|
|
170
203
|
/** End the call (hang up). */
|
|
171
204
|
stopListening(): Promise<void>;
|
|
172
205
|
/** Tear down the call and drop all callbacks (used by `cleanupVoice`). */
|
package/dist/voice-runtime.d.ts
CHANGED
|
@@ -77,6 +77,10 @@ type VoiceConfig = {
|
|
|
77
77
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
78
78
|
*/
|
|
79
79
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
80
|
+
/** See `voiceRecognition.provider.runtype.prewarmMode`. @default 'request' */
|
|
81
|
+
prewarmMode?: 'request' | 'attach';
|
|
82
|
+
/** See `voiceRecognition.provider.runtype.attachIdleMs`. @default 30000 */
|
|
83
|
+
attachIdleMs?: number;
|
|
80
84
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
81
85
|
pauseDuration?: number;
|
|
82
86
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -137,6 +141,13 @@ interface VoiceProvider {
|
|
|
137
141
|
deactivateBargeIn?(): Promise<void>;
|
|
138
142
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
139
143
|
stopPlayback?(): void;
|
|
144
|
+
/**
|
|
145
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
146
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
147
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
148
|
+
* repeatedly; throttle as needed.
|
|
149
|
+
*/
|
|
150
|
+
prewarm?(): void;
|
|
140
151
|
}
|
|
141
152
|
|
|
142
153
|
declare function createVoiceProvider(config: VoiceConfig): VoiceProvider;
|
|
@@ -157,6 +168,8 @@ declare class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
157
168
|
private isSpeaking;
|
|
158
169
|
private callGeneration;
|
|
159
170
|
private intentionalClose;
|
|
171
|
+
private lastPrewarmAt;
|
|
172
|
+
private attached;
|
|
160
173
|
private resultCallbacks;
|
|
161
174
|
private errorCallbacks;
|
|
162
175
|
private statusCallbacks;
|
|
@@ -165,8 +178,28 @@ declare class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
165
178
|
constructor(config: VoiceConfig["runtype"]);
|
|
166
179
|
/** No-op: the WS session opens lazily in `startListening` (the "call"). */
|
|
167
180
|
connect(): Promise<void>;
|
|
181
|
+
/**
|
|
182
|
+
* Warm the voice path ahead of the click. Fire-and-forget and throttled to
|
|
183
|
+
* once per {@link PREWARM_THROTTLE_MS}; never emits an error or a status.
|
|
184
|
+
*/
|
|
185
|
+
prewarm(): void;
|
|
186
|
+
private requestPrewarm;
|
|
187
|
+
private attach;
|
|
188
|
+
private armAttachIdle;
|
|
189
|
+
/** Close and forget the prewarmed socket, if any. Silent by design. */
|
|
190
|
+
private releaseAttached;
|
|
191
|
+
/**
|
|
192
|
+
* Hand the prewarmed socket to the call, waiting up to
|
|
193
|
+
* {@link ATTACH_HANDSHAKE_TIMEOUT_MS} for a handshake still in flight.
|
|
194
|
+
* Resolves `null` when there is none, it stalled, or it dropped, so the
|
|
195
|
+
* caller opens a fresh socket.
|
|
196
|
+
*/
|
|
197
|
+
private takeAttachedSocket;
|
|
198
|
+
private voiceSocketUrl;
|
|
168
199
|
/** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
|
|
169
200
|
startListening(): Promise<void>;
|
|
201
|
+
/** Route a call socket's frames, errors, and close into this call. */
|
|
202
|
+
private bindCallSocket;
|
|
170
203
|
/** End the call (hang up). */
|
|
171
204
|
stopListening(): Promise<void>;
|
|
172
205
|
/** Tear down the call and drop all callbacks (used by `cleanupVoice`). */
|
package/dist/voice-runtime.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
var m=class{constructor(e=24e3,t={}){this.ctx=null;this.nextStartTime=0;this.activeSources=[];this.finishedCallbacks=[];this.startedCallbacks=[];this.playing=!1;this.streamEnded=!1;this.pendingCount=0;this.started=!1;this.userPaused=!1;this.pendingBuffers=[];this.pendingSamples=0;this.remainder=null;this.sampleRate=e;let i=Math.max(0,t.prebufferMs??0);this.waterlineSamples=Math.round(e*i/1e3),this.buffering=this.waterlineSamples>0}ensureContext(){if(!this.ctx){let t=typeof window<"u"?window:void 0;if(!t)throw new Error("AudioPlaybackManager requires a browser environment");let i=t.AudioContext||t.webkitAudioContext;this.ctx=new i({sampleRate:this.sampleRate})}let e=this.ctx;return e.state==="suspended"&&!this.userPaused&&e.resume(),e}enqueue(e){if(e.length===0)return;let t=e;if(this.remainder){let s=new Uint8Array(this.remainder.length+e.length);s.set(this.remainder),s.set(e,this.remainder.length),t=s,this.remainder=null}if(t.length%2!==0&&(this.remainder=new Uint8Array([t[t.length-1]]),t=t.subarray(0,t.length-1)),t.length===0)return;let i=this.pcmToFloat32(t);i.length!==0&&(this.buffering?(this.pendingBuffers.push(i),this.pendingSamples+=i.length,this.pendingSamples>=this.waterlineSamples&&this.releaseBuffer()):this.scheduleSamples(i))}markStreamEnd(){this.pendingBuffers.length>0&&this.releaseBuffer(),this.streamEnded=!0,this.checkFinished()}flush(){for(let e of this.activeSources)try{e.stop(),e.disconnect()}catch{}this.activeSources=[],this.pendingCount=0,this.nextStartTime=0,this.playing=!1,this.streamEnded=!1,this.finishedCallbacks=[],this.startedCallbacks=[],this.remainder=null,this.pendingBuffers=[],this.pendingSamples=0,this.buffering=this.waterlineSamples>0,this.started=!1}isPlaying(){return this.playing}onFinished(e){this.finishedCallbacks.push(e)}onStarted(e){this.startedCallbacks.push(e)}pause(){this.userPaused=!0,this.ctx&&this.ctx.state==="running"&&this.ctx.suspend()}resume(){this.userPaused=!1,this.ctx&&this.ctx.state==="suspended"&&this.ctx.resume()}async destroy(){this.flush(),this.ctx&&(await this.ctx.close(),this.ctx=null)}releaseBuffer(){this.buffering=!1;let e=this.pendingBuffers;this.pendingBuffers=[],this.pendingSamples=0;for(let t of e)this.scheduleSamples(t)}scheduleSamples(e){if(e.length===0)return;let t=this.ensureContext(),i=t.createBuffer(1,e.length,this.sampleRate);i.getChannelData(0).set(e);let s=t.createBufferSource();s.buffer=i,s.connect(t.destination);let n=t.currentTime;if(this.nextStartTime===0?this.nextStartTime=n:this.nextStartTime<n&&(this.nextStartTime=n,this.waterlineSamples>0&&(this.buffering=!0)),s.start(this.nextStartTime),this.nextStartTime+=i.duration,this.activeSources.push(s),this.pendingCount++,this.playing=!0,!this.started){this.started=!0;let a=this.startedCallbacks.slice();this.startedCallbacks=[];for(let l of a)l()}s.onended=()=>{let a=this.activeSources.indexOf(s);a!==-1&&this.activeSources.splice(a,1),this.pendingCount--,this.checkFinished()}}checkFinished(){if(this.streamEnded&&this.pendingCount<=0&&this.pendingBuffers.length===0){this.playing=!1,this.streamEnded=!1;let e=this.finishedCallbacks.slice();this.finishedCallbacks=[];for(let t of e)t()}}pcmToFloat32(e){let t=Math.floor(e.length/2),i=new Float32Array(t),s=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let n=0;n<t;n++){let a=s.getInt16(n*2,!0);i[n]=a/32768}return i}};var g=16e3,w=24e3,S=4096,C=4,k=1380533830;function E(r){return r.byteLength>=44&&new DataView(r).getUint32(0,!1)===k?new Uint8Array(r,44):new Uint8Array(r)}function x(r){let e=r.replace(/\/+$/,"");return/^wss?:\/\//i.test(e)?e:/^https?:\/\//i.test(e)?e.replace(/^http/i,"ws"):`${typeof window<"u"&&window.location?.protocol==="https:"?"wss:":"ws:"}//${e}`}var d=class{constructor(e){this.config=e;this.type="runtype";this.ws=null;this.captureContext=null;this.levelCallbacks=[];this.mediaStream=null;this.sourceNode=null;this.processor=null;this.playback=null;this.callLive=!1;this.isSpeaking=!1;this.callGeneration=0;this.intentionalClose=!1;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.transcriptCallbacks=[];this.metricsCallbacks=[]}async connect(){}async startListening(){if(this.callLive)return;let e=this.config?.agentId,t=this.config?.clientToken,i=this.config?.host;if(!e)throw new Error("Runtype voice requires an agentId");if(!t)throw new Error("Runtype voice requires a clientToken");if(!i)throw new Error("Runtype voice requires a host (or widget apiUrl)");let s=++this.callGeneration;this.intentionalClose=!1,this.callLive=!0;try{let n=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:g,channelCount:1,echoCancellation:!0}});if(s!==this.callGeneration){n.getTracks().forEach(o=>o.stop());return}this.mediaStream=n;let a=window.AudioContext||window.webkitAudioContext,l=new a({sampleRate:g});l.state==="suspended"&&await l.resume().catch(()=>{}),this.captureContext=l;let h=this.config?.createPlaybackEngine?await this.config.createPlaybackEngine():new m(w);if(s!==this.callGeneration){h.destroy(),n.getTracks().forEach(o=>o.stop()),l.close().catch(()=>{});return}this.playback=h,h.onFinished(()=>{s===this.callGeneration&&(this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening"))});let f=`${x(i)}/ws/agents/${encodeURIComponent(e)}/voice`,c=new WebSocket(f,["runtype.bearer",t]);c.binaryType="arraybuffer",this.ws=c,c.onopen=()=>{s===this.callGeneration&&(this.emitStatus("listening"),this.startCapture(l,n,c,s))},c.onmessage=o=>this.handleMessage(o,s),c.onerror=()=>{s===this.callGeneration&&(this.emitError(new Error("Voice connection failed")),this.emitStatus("error"),this.cleanup())},c.onclose=o=>{if(this.intentionalClose){this.intentionalClose=!1;return}if(s===this.callGeneration){if(o.code!==1e3){let u=o.code?` (code ${o.code})`:"";this.emitError(new Error(`Voice connection closed${u}`)),this.emitStatus("error")}else this.emitStatus("idle");this.cleanup()}}}catch(n){throw this.cleanup(),this.emitError(n),this.emitStatus("error"),n}}async stopListening(){this.cleanup(),this.emitStatus("idle")}async disconnect(){this.cleanup(),this.emitStatus("disconnected"),this.resultCallbacks=[],this.errorCallbacks=[],this.statusCallbacks=[],this.transcriptCallbacks=[],this.metricsCallbacks=[]}stopPlayback(){this.playback&&this.playback.flush(),this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening")}getInterruptionMode(){return"barge-in"}isBargeInActive(){return this.callLive}onLevel(e){this.levelCallbacks.push(e)}async deactivateBargeIn(){this.cleanup(),this.emitStatus("idle")}startCapture(e,t,i,s){let n=e.createMediaStreamSource(t);this.sourceNode=n;let a=e.createScriptProcessor(S,1,1);this.processor=a,a.onaudioprocess=l=>{if(s!==this.callGeneration||i.readyState!==WebSocket.OPEN)return;let h=l.inputBuffer.getChannelData(0),f=new Int16Array(h.length),c=0;for(let o=0;o<h.length;o++){let u=Math.max(-1,Math.min(1,h[o]));c+=u*u,f[o]=u<0?u*32768:u*32767}if(i.send(f.buffer),this.levelCallbacks.length>0){let o=Math.sqrt(c/h.length),u=Math.max(0,Math.min(1,o*C));for(let y of this.levelCallbacks)y(u)}},n.connect(a),a.connect(e.destination)}handleMessage(e,t){if(t!==this.callGeneration)return;if(e.data instanceof ArrayBuffer){this.handleAudioFrame(e.data,t);return}let i;try{i=JSON.parse(e.data)}catch{return}switch(i.type){case"transcript_interim":this.emitStatus("listening"),this.emitTranscript("user",i.text??"",!1);break;case"transcript_final":{let s=i.role==="assistant"?"assistant":"user";this.emitStatus(s==="user"?"processing":"speaking"),this.emitTranscript(s,i.text??"",!0);break}case"audio_end":this.playback?this.playback.markStreamEnd():(this.isSpeaking=!1,this.emitStatus("listening"));break;case"metrics":this.emitMetrics({llmMs:i.llm_ms,ttsMs:i.tts_ms,firstAudioMs:i.first_audio_ms,totalMs:i.total_ms});break;case"error":this.emitError(new Error(i.error||"Voice error")),this.emitStatus("error");break}}handleAudioFrame(e,t){if(t!==this.callGeneration||!this.playback)return;let i=E(e);i.length!==0&&(this.isSpeaking||(this.isSpeaking=!0,this.emitStatus("speaking")),this.playback.enqueue(i))}cleanup(){if(this.callGeneration+=1,this.callLive=!1,this.isSpeaking=!1,this.processor&&(this.processor.onaudioprocess=null,this.processor.disconnect(),this.processor=null),this.sourceNode&&(this.sourceNode.disconnect(),this.sourceNode=null),this.mediaStream&&(this.mediaStream.getTracks().forEach(e=>e.stop()),this.mediaStream=null),this.captureContext&&(this.captureContext.close().catch(()=>{}),this.captureContext=null),this.playback&&(this.playback.destroy(),this.playback=null),this.ws){this.intentionalClose=!0;try{this.ws.close(1e3,"client ended call")}catch{}this.ws=null}}onResult(e){this.resultCallbacks.push(e)}onError(e){this.errorCallbacks.push(e)}onStatusChange(e){this.statusCallbacks.push(e)}onTranscript(e){this.transcriptCallbacks.push(e)}onMetrics(e){this.metricsCallbacks.push(e)}emitStatus(e){this.statusCallbacks.forEach(t=>t(e))}emitError(e){this.errorCallbacks.forEach(t=>t(e))}emitTranscript(e,t,i){this.transcriptCallbacks.forEach(s=>s(e,t,i))}emitMetrics(e){this.metricsCallbacks.forEach(t=>t(e))}};var p=class{constructor(e={}){this.config=e;this.type="browser";this.recognition=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.isListening=!1;this.w=typeof window<"u"?window:void 0}async connect(){this.statusCallbacks.forEach(e=>e("connected"))}async startListening(){try{if(this.isListening)throw new Error("Already listening");if(!this.w)throw new Error("Window object not available");let e=this.w.SpeechRecognition||this.w.webkitSpeechRecognition;if(!e)throw new Error("Browser speech recognition not supported");this.recognition=new e,this.recognition.lang=this.config?.language||"en-US",this.recognition.continuous=this.config?.continuous||!1,this.recognition.interimResults=!0,this.recognition.onresult=t=>{let i=Array.from(t.results).map(n=>n[0]).map(n=>n.transcript).join(""),s=t.results[t.results.length-1].isFinal;this.resultCallbacks.forEach(n=>n({text:i,confidence:s?.8:.5,provider:"browser"})),s&&!this.config?.continuous&&this.stopListening()},this.recognition.onerror=t=>{this.errorCallbacks.forEach(i=>i(new Error(t.error))),this.statusCallbacks.forEach(i=>i("error"))},this.recognition.onstart=()=>{this.isListening=!0,this.statusCallbacks.forEach(t=>t("listening"))},this.recognition.onend=()=>{this.isListening=!1,this.statusCallbacks.forEach(t=>t("idle"))},this.recognition.start()}catch(e){throw this.errorCallbacks.forEach(t=>t(e)),this.statusCallbacks.forEach(t=>t("error")),e}}async stopListening(){this.recognition&&(this.recognition.stop(),this.recognition=null),this.isListening=!1,this.statusCallbacks.forEach(e=>e("idle"))}onResult(e){this.resultCallbacks.push(e)}onError(e){this.errorCallbacks.push(e)}onStatusChange(e){this.statusCallbacks.push(e)}async disconnect(){await this.stopListening(),this.statusCallbacks.forEach(e=>e("disconnected"))}static isSupported(){return"SpeechRecognition"in window||"webkitSpeechRecognition"in window}};function v(r){switch(r.type){case"runtype":if(!r.runtype)throw new Error("Runtype voice provider requires configuration");return new d(r.runtype);case"browser":if(!p.isSupported())throw new Error("Browser speech recognition not supported");return new p(r.browser||{});case"custom":{let e=r.custom;if(!e)throw new Error("Custom voice provider requires a `custom` provider instance or factory");let t=typeof e=="function"?e():e;if(!t||typeof t.startListening!="function")throw new Error("Custom voice provider `custom` must be a VoiceProvider (or a factory returning one)");return t}default:throw new Error(`Unknown voice provider type: ${r.type}`)}}function b(r){if(r?.type==="custom"&&r.custom)return v({type:"custom",custom:r.custom});if(r?.type==="runtype"&&r.runtype)return v({type:"runtype",runtype:r.runtype});if(p.isSupported())return v({type:"browser",browser:r?.browser||{language:"en-US"}});throw new Error("No supported voice providers available")}function A(r){try{return b(r),!0}catch{return!1}}export{p as BrowserVoiceProvider,d as RuntypeVoiceProvider,b as createBestAvailableVoiceProvider,v as createVoiceProvider,A as isVoiceSupported};
|
|
1
|
+
var m=class{constructor(t=24e3,e={}){this.ctx=null;this.nextStartTime=0;this.activeSources=[];this.finishedCallbacks=[];this.startedCallbacks=[];this.playing=!1;this.streamEnded=!1;this.pendingCount=0;this.started=!1;this.userPaused=!1;this.pendingBuffers=[];this.pendingSamples=0;this.remainder=null;this.sampleRate=t;let i=Math.max(0,e.prebufferMs??0);this.waterlineSamples=Math.round(t*i/1e3),this.buffering=this.waterlineSamples>0}ensureContext(){if(!this.ctx){let e=typeof window<"u"?window:void 0;if(!e)throw new Error("AudioPlaybackManager requires a browser environment");let i=e.AudioContext||e.webkitAudioContext;this.ctx=new i({sampleRate:this.sampleRate})}let t=this.ctx;return t.state==="suspended"&&!this.userPaused&&t.resume(),t}enqueue(t){if(t.length===0)return;let e=t;if(this.remainder){let s=new Uint8Array(this.remainder.length+t.length);s.set(this.remainder),s.set(t,this.remainder.length),e=s,this.remainder=null}if(e.length%2!==0&&(this.remainder=new Uint8Array([e[e.length-1]]),e=e.subarray(0,e.length-1)),e.length===0)return;let i=this.pcmToFloat32(e);i.length!==0&&(this.buffering?(this.pendingBuffers.push(i),this.pendingSamples+=i.length,this.pendingSamples>=this.waterlineSamples&&this.releaseBuffer()):this.scheduleSamples(i))}markStreamEnd(){this.pendingBuffers.length>0&&this.releaseBuffer(),this.streamEnded=!0,this.checkFinished()}flush(){for(let t of this.activeSources)try{t.stop(),t.disconnect()}catch{}this.activeSources=[],this.pendingCount=0,this.nextStartTime=0,this.playing=!1,this.streamEnded=!1,this.finishedCallbacks=[],this.startedCallbacks=[],this.remainder=null,this.pendingBuffers=[],this.pendingSamples=0,this.buffering=this.waterlineSamples>0,this.started=!1}isPlaying(){return this.playing}onFinished(t){this.finishedCallbacks.push(t)}onStarted(t){this.startedCallbacks.push(t)}pause(){this.userPaused=!0,this.ctx&&this.ctx.state==="running"&&this.ctx.suspend()}resume(){this.userPaused=!1,this.ctx&&this.ctx.state==="suspended"&&this.ctx.resume()}async destroy(){this.flush(),this.ctx&&(await this.ctx.close(),this.ctx=null)}releaseBuffer(){this.buffering=!1;let t=this.pendingBuffers;this.pendingBuffers=[],this.pendingSamples=0;for(let e of t)this.scheduleSamples(e)}scheduleSamples(t){if(t.length===0)return;let e=this.ensureContext(),i=e.createBuffer(1,t.length,this.sampleRate);i.getChannelData(0).set(t);let s=e.createBufferSource();s.buffer=i,s.connect(e.destination);let n=e.currentTime;if(this.nextStartTime===0?this.nextStartTime=n:this.nextStartTime<n&&(this.nextStartTime=n,this.waterlineSamples>0&&(this.buffering=!0)),s.start(this.nextStartTime),this.nextStartTime+=i.duration,this.activeSources.push(s),this.pendingCount++,this.playing=!0,!this.started){this.started=!0;let l=this.startedCallbacks.slice();this.startedCallbacks=[];for(let c of l)c()}s.onended=()=>{let l=this.activeSources.indexOf(s);l!==-1&&this.activeSources.splice(l,1),this.pendingCount--,this.checkFinished()}}checkFinished(){if(this.streamEnded&&this.pendingCount<=0&&this.pendingBuffers.length===0){this.playing=!1,this.streamEnded=!1;let t=this.finishedCallbacks.slice();this.finishedCallbacks=[];for(let e of t)e()}}pcmToFloat32(t){let e=Math.floor(t.length/2),i=new Float32Array(e),s=new DataView(t.buffer,t.byteOffset,t.byteLength);for(let n=0;n<e;n++){let l=s.getInt16(n*2,!0);i[n]=l/32768}return i}};var g=16e3,S=24e3,k=4096,C=4,E=1380533830,A=3e4,x=5e3,P=3e4,T=3e4,M=6e5;function V(r){return r.byteLength>=44&&new DataView(r).getUint32(0,!1)===E?new Uint8Array(r,44):new Uint8Array(r)}function b(r){let t=r.replace(/\/+$/,"");return/^wss?:\/\//i.test(t)?t:/^https?:\/\//i.test(t)?t.replace(/^http/i,"ws"):`${typeof window<"u"&&window.location?.protocol==="https:"?"wss:":"ws:"}//${t}`}function R(r){return b(r).replace(/^ws/i,"http")}function _(r){return typeof r!="number"||!Number.isFinite(r)?P:Math.min(M,Math.max(T,Math.round(r)))}var f=class{constructor(t){this.config=t;this.type="runtype";this.ws=null;this.captureContext=null;this.levelCallbacks=[];this.mediaStream=null;this.sourceNode=null;this.processor=null;this.playback=null;this.callLive=!1;this.isSpeaking=!1;this.callGeneration=0;this.intentionalClose=!1;this.lastPrewarmAt=Number.NEGATIVE_INFINITY;this.attached=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.transcriptCallbacks=[];this.metricsCallbacks=[]}async connect(){}prewarm(){if(this.callLive||this.attached)return;let t=this.config?.agentId,e=this.config?.clientToken,i=this.config?.host;if(!t||!e||!i)return;let s=Date.now();if(!(s-this.lastPrewarmAt<A)){this.lastPrewarmAt=s;try{this.config?.prewarmMode==="attach"?this.attach(i,t,e):this.requestPrewarm(i,t,e)}catch{}}}requestPrewarm(t,e,i){if(typeof fetch!="function")return;let s=`${R(t)}/v1/client/agents/${encodeURIComponent(e)}/voice/prewarm`;fetch(s,{method:"POST",headers:{Authorization:`Bearer ${i}`},keepalive:!0}).catch(()=>{})}attach(t,e,i){let s=new URLSearchParams({clientCapabilities:"attach"}),n=this.config?.attachIdleMs,l=_(n);n!==void 0&&s.set("attachIdleMs",String(l));let c=new WebSocket(`${this.voiceSocketUrl(t,e)}?${s}`,["runtype.bearer","runtype.attach",i]);c.binaryType="arraybuffer";let u=()=>{},d=new Promise(o=>{u=o}),a={ws:c,state:"connecting",ready:d,settle:u,openedAt:0,idleTimer:void 0};this.attached=a,c.onopen=()=>{this.attached===a&&(a.state=c.protocol==="runtype.attach"?"attached":"live",a.openedAt=Date.now(),this.armAttachIdle(a,l),a.state==="attached"&&c.send('{"type":"ping"}'),a.settle())},c.onmessage=o=>{if(this.attached!==a||typeof o.data!="string")return;let h;try{h=JSON.parse(o.data)}catch{return}h.type==="attached"&&typeof h.idleMs=="number"&&this.armAttachIdle(a,h.idleMs-(Date.now()-a.openedAt))},c.onerror=()=>{},c.onclose=()=>{this.attached===a&&this.releaseAttached()}}armAttachIdle(t,e){clearTimeout(t.idleTimer),t.idleTimer=setTimeout(()=>{this.attached===t&&this.releaseAttached()},Math.max(0,e))}releaseAttached(){let t=this.attached;if(!t)return;this.attached=null,clearTimeout(t.idleTimer);let{ws:e}=t;if(e.onopen=null,e.onmessage=null,e.onerror=null,e.onclose=null,t.settle(),e.readyState===WebSocket.CONNECTING||e.readyState===WebSocket.OPEN)try{e.close(1e3,"prewarm released")}catch{}}async takeAttachedSocket(){let t=this.attached;if(!t)return null;if(t.state==="connecting"){let e;await Promise.race([t.ready,new Promise(i=>{e=setTimeout(i,x)})]),clearTimeout(e)}return this.attached!==t?null:t.state==="connecting"||t.ws.readyState!==WebSocket.OPEN?(this.releaseAttached(),null):(this.attached=null,clearTimeout(t.idleTimer),{ws:t.ws,live:t.state==="live"})}voiceSocketUrl(t,e){return`${b(t)}/ws/agents/${encodeURIComponent(e)}/voice`}async startListening(){if(this.callLive)return;let t=this.config?.agentId,e=this.config?.clientToken,i=this.config?.host;if(!t)throw new Error("Runtype voice requires an agentId");if(!e)throw new Error("Runtype voice requires a clientToken");if(!i)throw new Error("Runtype voice requires a host (or widget apiUrl)");let s=++this.callGeneration;this.intentionalClose=!1,this.callLive=!0;try{let n=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:g,channelCount:1,echoCancellation:!0}});if(s!==this.callGeneration){n.getTracks().forEach(o=>o.stop());return}this.mediaStream=n;let l=window.AudioContext||window.webkitAudioContext,c=new l({sampleRate:g});c.state==="suspended"&&await c.resume().catch(()=>{}),this.captureContext=c;let u=this.config?.createPlaybackEngine?await this.config.createPlaybackEngine():new m(S);if(s!==this.callGeneration){u.destroy(),n.getTracks().forEach(o=>o.stop()),c.close().catch(()=>{});return}this.playback=u,u.onFinished(()=>{s===this.callGeneration&&(this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening"))});let d=await this.takeAttachedSocket();if(s!==this.callGeneration){d?.ws.close(1e3,"client ended call");return}if(d){let{ws:o}=d;this.ws=o,this.bindCallSocket(o,s),d.live||o.send('{"type":"start"}'),this.emitStatus("listening"),this.startCapture(c,n,o,s);return}let a=new WebSocket(this.voiceSocketUrl(i,t),["runtype.bearer",e]);a.binaryType="arraybuffer",this.ws=a,a.onopen=()=>{s===this.callGeneration&&(this.emitStatus("listening"),this.startCapture(c,n,a,s))},this.bindCallSocket(a,s)}catch(n){throw this.cleanup(),this.emitError(n),this.emitStatus("error"),n}}bindCallSocket(t,e){t.onmessage=i=>this.handleMessage(i,e),t.onerror=()=>{e===this.callGeneration&&(this.emitError(new Error("Voice connection failed")),this.emitStatus("error"),this.cleanup())},t.onclose=i=>{if(this.intentionalClose){this.intentionalClose=!1;return}if(e===this.callGeneration){if(i.code!==1e3){let s=i.code?` (code ${i.code})`:"";this.emitError(new Error(`Voice connection closed${s}`)),this.emitStatus("error")}else this.emitStatus("idle");this.cleanup()}}}async stopListening(){this.cleanup(),this.emitStatus("idle")}async disconnect(){this.cleanup(),this.emitStatus("disconnected"),this.resultCallbacks=[],this.errorCallbacks=[],this.statusCallbacks=[],this.transcriptCallbacks=[],this.metricsCallbacks=[]}stopPlayback(){this.playback&&this.playback.flush(),this.isSpeaking=!1,this.ws&&this.ws.readyState===WebSocket.OPEN&&this.emitStatus("listening")}getInterruptionMode(){return"barge-in"}isBargeInActive(){return this.callLive}onLevel(t){this.levelCallbacks.push(t)}async deactivateBargeIn(){this.cleanup(),this.emitStatus("idle")}startCapture(t,e,i,s){let n=t.createMediaStreamSource(e);this.sourceNode=n;let l=t.createScriptProcessor(k,1,1);this.processor=l,l.onaudioprocess=c=>{if(s!==this.callGeneration||i.readyState!==WebSocket.OPEN)return;let u=c.inputBuffer.getChannelData(0),d=new Int16Array(u.length),a=0;for(let o=0;o<u.length;o++){let h=Math.max(-1,Math.min(1,u[o]));a+=h*h,d[o]=h<0?h*32768:h*32767}if(i.send(d.buffer),this.levelCallbacks.length>0){let o=Math.sqrt(a/u.length),h=Math.max(0,Math.min(1,o*C));for(let w of this.levelCallbacks)w(h)}},n.connect(l),l.connect(t.destination)}handleMessage(t,e){if(e!==this.callGeneration)return;if(t.data instanceof ArrayBuffer){this.handleAudioFrame(t.data,e);return}let i;try{i=JSON.parse(t.data)}catch{return}switch(i.type){case"transcript_interim":this.emitStatus("listening"),this.emitTranscript("user",i.text??"",!1);break;case"transcript_final":{let s=i.role==="assistant"?"assistant":"user";this.emitStatus(s==="user"?"processing":"speaking"),this.emitTranscript(s,i.text??"",!0);break}case"audio_end":this.playback?this.playback.markStreamEnd():(this.isSpeaking=!1,this.emitStatus("listening"));break;case"metrics":this.emitMetrics({llmMs:i.llm_ms,ttsMs:i.tts_ms,firstAudioMs:i.first_audio_ms,totalMs:i.total_ms});break;case"error":this.emitError(new Error(i.error||"Voice error")),this.emitStatus("error");break}}handleAudioFrame(t,e){if(e!==this.callGeneration||!this.playback)return;let i=V(t);i.length!==0&&(this.isSpeaking||(this.isSpeaking=!0,this.emitStatus("speaking")),this.playback.enqueue(i))}cleanup(){if(this.callGeneration+=1,this.callLive=!1,this.isSpeaking=!1,this.releaseAttached(),this.processor&&(this.processor.onaudioprocess=null,this.processor.disconnect(),this.processor=null),this.sourceNode&&(this.sourceNode.disconnect(),this.sourceNode=null),this.mediaStream&&(this.mediaStream.getTracks().forEach(t=>t.stop()),this.mediaStream=null),this.captureContext&&(this.captureContext.close().catch(()=>{}),this.captureContext=null),this.playback&&(this.playback.destroy(),this.playback=null),this.ws){this.intentionalClose=!0;try{this.ws.close(1e3,"client ended call")}catch{}this.ws=null}}onResult(t){this.resultCallbacks.push(t)}onError(t){this.errorCallbacks.push(t)}onStatusChange(t){this.statusCallbacks.push(t)}onTranscript(t){this.transcriptCallbacks.push(t)}onMetrics(t){this.metricsCallbacks.push(t)}emitStatus(t){this.statusCallbacks.forEach(e=>e(t))}emitError(t){this.errorCallbacks.forEach(e=>e(t))}emitTranscript(t,e,i){this.transcriptCallbacks.forEach(s=>s(t,e,i))}emitMetrics(t){this.metricsCallbacks.forEach(e=>e(t))}};var p=class{constructor(t={}){this.config=t;this.type="browser";this.recognition=null;this.resultCallbacks=[];this.errorCallbacks=[];this.statusCallbacks=[];this.isListening=!1;this.w=typeof window<"u"?window:void 0}async connect(){this.statusCallbacks.forEach(t=>t("connected"))}async startListening(){try{if(this.isListening)throw new Error("Already listening");if(!this.w)throw new Error("Window object not available");let t=this.w.SpeechRecognition||this.w.webkitSpeechRecognition;if(!t)throw new Error("Browser speech recognition not supported");this.recognition=new t,this.recognition.lang=this.config?.language||"en-US",this.recognition.continuous=this.config?.continuous||!1,this.recognition.interimResults=!0,this.recognition.onresult=e=>{let i=Array.from(e.results).map(n=>n[0]).map(n=>n.transcript).join(""),s=e.results[e.results.length-1].isFinal;this.resultCallbacks.forEach(n=>n({text:i,confidence:s?.8:.5,provider:"browser"})),s&&!this.config?.continuous&&this.stopListening()},this.recognition.onerror=e=>{this.errorCallbacks.forEach(i=>i(new Error(e.error))),this.statusCallbacks.forEach(i=>i("error"))},this.recognition.onstart=()=>{this.isListening=!0,this.statusCallbacks.forEach(e=>e("listening"))},this.recognition.onend=()=>{this.isListening=!1,this.statusCallbacks.forEach(e=>e("idle"))},this.recognition.start()}catch(t){throw this.errorCallbacks.forEach(e=>e(t)),this.statusCallbacks.forEach(e=>e("error")),t}}async stopListening(){this.recognition&&(this.recognition.stop(),this.recognition=null),this.isListening=!1,this.statusCallbacks.forEach(t=>t("idle"))}onResult(t){this.resultCallbacks.push(t)}onError(t){this.errorCallbacks.push(t)}onStatusChange(t){this.statusCallbacks.push(t)}async disconnect(){await this.stopListening(),this.statusCallbacks.forEach(t=>t("disconnected"))}static isSupported(){return"SpeechRecognition"in window||"webkitSpeechRecognition"in window}};function v(r){switch(r.type){case"runtype":if(!r.runtype)throw new Error("Runtype voice provider requires configuration");return new f(r.runtype);case"browser":if(!p.isSupported())throw new Error("Browser speech recognition not supported");return new p(r.browser||{});case"custom":{let t=r.custom;if(!t)throw new Error("Custom voice provider requires a `custom` provider instance or factory");let e=typeof t=="function"?t():t;if(!e||typeof e.startListening!="function")throw new Error("Custom voice provider `custom` must be a VoiceProvider (or a factory returning one)");return e}default:throw new Error(`Unknown voice provider type: ${r.type}`)}}function y(r){if(r?.type==="custom"&&r.custom)return v({type:"custom",custom:r.custom});if(r?.type==="runtype"&&r.runtype)return v({type:"runtype",runtype:r.runtype});if(p.isSupported())return v({type:"browser",browser:r?.browser||{language:"en-US"}});throw new Error("No supported voice providers available")}function L(r){try{return y(r),!0}catch{return!1}}export{p as BrowserVoiceProvider,f as RuntypeVoiceProvider,y as createBestAvailableVoiceProvider,v as createVoiceProvider,L as isVoiceSupported};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@runtypelabs/persona",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.25.0",
|
|
4
4
|
"description": "Themeable, pluggable streaming agent widget for websites, in plain JS with support for voice input and reasoning / tool output.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.cjs",
|
package/src/index-core.ts
CHANGED
package/src/session.ts
CHANGED
|
@@ -910,6 +910,33 @@ export class AgentWidgetSession {
|
|
|
910
910
|
}
|
|
911
911
|
}
|
|
912
912
|
|
|
913
|
+
/**
|
|
914
|
+
* Warm the voice path ahead of the mic click (`voiceRecognition.prewarm`).
|
|
915
|
+
* Session-owned providers (`runtype`, `custom`) get `provider.prewarm()` once
|
|
916
|
+
* the lazy voice runtime has installed them; the browser (Web Speech) path
|
|
917
|
+
* warms the client-token session instead. Fire-and-forget and silent.
|
|
918
|
+
*/
|
|
919
|
+
public prewarmVoice(): void {
|
|
920
|
+
if (!usesSessionVoice(this.config.voiceRecognition?.provider)) {
|
|
921
|
+
this.warmClientSession();
|
|
922
|
+
return;
|
|
923
|
+
}
|
|
924
|
+
const generation = this.voiceSetupGeneration;
|
|
925
|
+
const run = () => {
|
|
926
|
+
const provider = this.voiceProvider;
|
|
927
|
+
if (!provider?.prewarm || generation !== this.voiceSetupGeneration) return;
|
|
928
|
+
if (this.voiceActive || this.isBargeInActive()) return;
|
|
929
|
+
void Promise.resolve()
|
|
930
|
+
.then(() => provider.prewarm?.())
|
|
931
|
+
.catch(() => {});
|
|
932
|
+
};
|
|
933
|
+
if (!this.voiceProvider && this.voiceSetupPromise) {
|
|
934
|
+
void this.voiceSetupPromise.then(run, () => {});
|
|
935
|
+
return;
|
|
936
|
+
}
|
|
937
|
+
run();
|
|
938
|
+
}
|
|
939
|
+
|
|
913
940
|
/**
|
|
914
941
|
* Cleanup voice resources
|
|
915
942
|
*/
|
|
@@ -975,7 +1002,9 @@ export class AgentWidgetSession {
|
|
|
975
1002
|
clientToken: providerConfig.runtype?.clientToken ?? this.config.clientToken,
|
|
976
1003
|
host: providerConfig.runtype?.host ?? this.config.apiUrl,
|
|
977
1004
|
voiceId: providerConfig.runtype?.voiceId,
|
|
978
|
-
createPlaybackEngine: providerConfig.runtype?.createPlaybackEngine
|
|
1005
|
+
createPlaybackEngine: providerConfig.runtype?.createPlaybackEngine,
|
|
1006
|
+
prewarmMode: providerConfig.runtype?.prewarmMode,
|
|
1007
|
+
attachIdleMs: providerConfig.runtype?.attachIdleMs
|
|
979
1008
|
}
|
|
980
1009
|
};
|
|
981
1010
|
|
|
@@ -13,7 +13,8 @@ const h = vi.hoisted(() => {
|
|
|
13
13
|
| ((role: 'user' | 'assistant', text: string, isFinal: boolean) => void)
|
|
14
14
|
| null;
|
|
15
15
|
metricsCb: ((m: VoiceMetrics) => void) | null;
|
|
16
|
-
|
|
16
|
+
prewarms: number;
|
|
17
|
+
} = { transcriptCb: null, metricsCb: null, prewarms: 0 };
|
|
17
18
|
|
|
18
19
|
const fakeProvider = {
|
|
19
20
|
type: 'runtype' as const,
|
|
@@ -30,6 +31,9 @@ const h = vi.hoisted(() => {
|
|
|
30
31
|
onMetrics: (cb: typeof state.metricsCb) => {
|
|
31
32
|
state.metricsCb = cb;
|
|
32
33
|
},
|
|
34
|
+
prewarm: () => {
|
|
35
|
+
state.prewarms += 1;
|
|
36
|
+
},
|
|
33
37
|
};
|
|
34
38
|
|
|
35
39
|
return { state, fakeProvider };
|
|
@@ -212,3 +216,70 @@ describe('AgentWidgetSession - Runtype TTS config', () => {
|
|
|
212
216
|
}
|
|
213
217
|
});
|
|
214
218
|
});
|
|
219
|
+
|
|
220
|
+
describe('AgentWidgetSession - prewarmVoice', () => {
|
|
221
|
+
const callbacks = {
|
|
222
|
+
onMessagesChanged: () => {},
|
|
223
|
+
onStatusChanged: () => {},
|
|
224
|
+
onStreamingChanged: () => {},
|
|
225
|
+
onError: () => {},
|
|
226
|
+
};
|
|
227
|
+
const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
|
|
228
|
+
const runtypeSession = () =>
|
|
229
|
+
new AgentWidgetSession(
|
|
230
|
+
{
|
|
231
|
+
apiUrl: 'http://localhost:8000',
|
|
232
|
+
voiceRecognition: {
|
|
233
|
+
enabled: true,
|
|
234
|
+
provider: { type: 'runtype', runtype: { agentId: 'a1' } },
|
|
235
|
+
},
|
|
236
|
+
},
|
|
237
|
+
callbacks,
|
|
238
|
+
);
|
|
239
|
+
|
|
240
|
+
beforeEach(() => {
|
|
241
|
+
h.state.prewarms = 0;
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it("calls the provider's prewarm once setup has installed it", async () => {
|
|
245
|
+
const session = runtypeSession();
|
|
246
|
+
session.setupVoice();
|
|
247
|
+
await flush();
|
|
248
|
+
session.prewarmVoice();
|
|
249
|
+
await flush();
|
|
250
|
+
expect(h.state.prewarms).toBe(1);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it('waits for a setup still loading the voice runtime', async () => {
|
|
254
|
+
const session = runtypeSession();
|
|
255
|
+
session.setupVoice();
|
|
256
|
+
session.prewarmVoice();
|
|
257
|
+
expect(h.state.prewarms).toBe(0);
|
|
258
|
+
await flush();
|
|
259
|
+
expect(h.state.prewarms).toBe(1);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it('drops a prewarm whose setup was torn down meanwhile', async () => {
|
|
263
|
+
const session = runtypeSession();
|
|
264
|
+
session.setupVoice();
|
|
265
|
+
session.prewarmVoice();
|
|
266
|
+
session.cleanupVoice();
|
|
267
|
+
await flush();
|
|
268
|
+
expect(h.state.prewarms).toBe(0);
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
it('warms the client session on the browser (Web Speech) path', () => {
|
|
272
|
+
const session = new AgentWidgetSession(
|
|
273
|
+
{
|
|
274
|
+
apiUrl: 'http://localhost:8000',
|
|
275
|
+
clientToken: 'ct_test',
|
|
276
|
+
voiceRecognition: { enabled: true, provider: { type: 'browser' } },
|
|
277
|
+
},
|
|
278
|
+
callbacks,
|
|
279
|
+
);
|
|
280
|
+
const warm = vi.spyOn(session, 'warmClientSession').mockImplementation(() => {});
|
|
281
|
+
session.prewarmVoice();
|
|
282
|
+
expect(warm).toHaveBeenCalledOnce();
|
|
283
|
+
expect(h.state.prewarms).toBe(0);
|
|
284
|
+
});
|
|
285
|
+
});
|