@streamoji/aitwin 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aitwin.cjs +2 -2
- package/dist/aitwin.js +5 -3
- package/dist/index.d.ts +7 -0
- package/package.json +1 -1
package/dist/aitwin.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const z=require("react/jsx-runtime"),w=require("react"),Li=require("@pipecat-ai/client-js"),Cn=require("@pipecat-ai/websocket-transport"),he=require("pixi.js");var wn=typeof document<"u"?document.currentScript:null;const Oi="https://aitwin.bubu.social",Br={publicBase:Oi},Ni="0.5.5",Vi={version:Ni},Ui={},ot="https://ai.aitwin.me",Rn=`${Br.publicBase}/custom-faces`,Wi=`${Br.publicBase}/aitwin-workers/v${Vi.version}/visemeDiffPreview.worker.js`;function Hi(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:wn&&wn.tagName.toUpperCase()==="SCRIPT"&&wn.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?Ui?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():Wi}const $i="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",gn=3e3,Pt=16,ar=3e3,zi=4e3,Ki=.85,ji=8,Dn=400,Bn="eng_a7f2b9c1",tn="eng_d4e8f3a2",ct="eng_c9b1e6d4",st=ct,Fr=new Set([tn,ct]),Lr={[Bn]:"oculus_direct",[ct]:"oculus_direct",[tn]:"inworld_word"},or={google:Bn,inworld:tn,cartesia:ct};function Or(e){return Lr[e]}function qi(e){return e===ct}function Gi(e){if(e.ttsEngineId&&e.ttsEngineId in Lr)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in or?or[t]:st}class Gt extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function Nr(e,t=$i){const n=new URL(t);n.searchParams.set("id",e);const r=await fetch(n.toString());if(r.status===404)throw new Gt;if(!r.ok)throw new Error(`getAiTwin failed (${r.status})`);const i=await r.json();if(!i.success)throw new Gt(i.error??"AI twin not found");const{tts:s,token:o,...a}=i.data,l=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:Gi(i.data),authToken:l}}const Yi="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",Qi="client_6TNvp3SCs4Og0a1ijm9TommXLql1",Xi="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",Ji="6TNvp3SCs4Og0a1ijm9TommXLql1",Zi="Swaraj Mali";async function Mt(){const e=await fetch(Yi,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":Qi,"Client-Secret":Xi},body:JSON.stringify({userId:Ji,userName:Zi})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),n=t.authToken??t.token??"";if(!n.trim())throw new Error("getAuthToken returned no token");return n}async function es(e,t=ot){const n=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!n.ok)throw new Error(`/api/session-value failed (${n.status})`);const r=await n.json();if(!r.value)throw new Error("Session value response missing value");return r}function ts(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const n=new Uint8Array(t.length/2);for(let r=0;r<t.length;r+=2)n[r/2]=Number.parseInt(t.slice(r,r+2),16);return n}function Ht(e){return Uint8Array.from(e)}function ns(e,t){const n=new Uint8Array(e.length+t.length);return n.set(e,0),n.set(t,e.length),n}async function rs(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const n=new Uint8Array(e),r=Ht(n.subarray(0,12)),i=Ht(n.subarray(12,28)),s=Ht(n.subarray(28)),o=Ht(ns(s,i)),a=await crypto.subtle.importKey("raw",ts(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:r,tagLength:128},a,o)}let bn=null;const Tn=new Set;function $t(e){return e.includes("/custom-faces/")}function Yt(e){const t=e.binBase.trim().replace(/\/$/,""),n=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||$t(t)||$t(n)))return{twinBase:n,binBase:t,encrypted:!1};const i=$t(t)?t:$t(n)?n:t;return{twinBase:i,binBase:i,encrypted:!0}}function ut(e){return Yt(e).encrypted}function kn(e,t=Rn){const n=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:n,binBase:n,encrypted:!0}}function Vr(e,t){return(e??t)?.trim()||void 0}function Ur(e){bn=Yt(e);for(const t of Tn)t()}function is(e,t){const n=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!n)return Yt(e);const r=i=>{try{return new URL(i,n).href.replace(/\/$/,"")}catch{return i}};return Yt({twinBase:r(e.twinBase),binBase:r(e.binBase),encrypted:e.encrypted})}function lt(){if(!bn)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return bn}function Wr(e){return Tn.add(e),()=>Tn.delete(e)}function Qt(){const e=lt();return ut(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function ss(){const e=lt();return ut(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function Hr(){const e=lt();return ut(e)?e.binBase:e.twinBase}function Dt(){return ut(lt())}function $r(){const e=lt();return ut(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function zr(){const e=lt();return ut(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function Kr(e){const t=e.split(/\r?\n/);let n="",r="";for(const s of t)s.startsWith("event:")?n=s.slice(6).trim():s.startsWith("data:")&&(r=s.slice(5).trim());if(!n)return null;let i={};if(r)try{i=JSON.parse(r)}catch{i={raw:r}}return{event:n,data:i}}const jr=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],as=new Map(jr.map(e=>[e.toLowerCase(),e]));function nn(e){const t=(e??"").trim();return t?as.get(t.toLowerCase())??"sil":"sil"}const os={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function cs(e){if(!e)return["sil"];const t=e.toLowerCase();return os[t]??["sil"]}function qr(e,t){const n=e.word.length;if(n<=0)return 0;const r=e.wduration;if(r<=0)return 0;const i=Math.max(0,Math.min(1,(t-e.wtime)/r));return Math.min(n-1,Math.floor(i*n))}function yt(e,t){const n=Math.max(0,Math.min(e.length-1,t));return e[n]??""}function rn(e,t){const n=e.toLowerCase().replace(/[^a-z']/g,"");return!n||yt(n,t)!=="w"||t>0&&yt(n,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(n)}function us(e,t){const n=e.toLowerCase(),r=yt(n,t),i=n.slice(Math.max(0,t-1),t+2);return r==="i"||r==="y"?"I":i.includes("ee")||r==="e"||n.includes("ea")?"E":"aa"}function ls(e,t){const n=e.toLowerCase(),r=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));if(/oo|ou|uw/.test(r))return"U";if(rn(n,t))return"FF";const i=yt(n,t);return i==="u"||i==="w"?"U":"O"}function fs(e,t){const n=e.toLowerCase();if(rn(n,t))return"FF";const r=n.slice(Math.max(0,t-2),t+1);if(/qu/.test(r))return"U";const i=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));return/wo|wh/.test(i)||yt(n,t)==="o"?"O":"U"}const zt={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function ds(e,t){const n=e.toLowerCase(),r=yt(n,t);if(zt[r])return zt[r];for(let i=0;i<n.length;i++){const s=n[i];if(zt[s])return zt[s]}return"DD"}function Gr(e,t,n){const r=cs(e);if(r.length===1)return r[0];const i=t.trim();if(!i)return r[0];const s=(e??"").toLowerCase(),o=Math.max(0,n);switch(s){case"aei":return us(i,o);case"o":return ls(i,o);case"qw":return fs(i,o);case"cdgknstxyz":return ds(i,o);default:return r[0]}}const Fn=16,xn=47,Yr=new Set(["aa","E","I"]),Qr=new Set(["DD","SS","kk","CH"]),ms=[Yr,Qr];function hs(e){for(const t of ms)if(t.has(e))return t;return null}function ps(e){return hs(e)!=null}function Xr(e,t=Fn){return t<=0?e:Math.round(e/t)*t}function Ln(e,t=Fn){return e.map(n=>({...n,vtime:Xr(n.vtime,t)}))}function Jr(e){const t=new Map;for(const n of e)t.has(n.vtime)||t.set(n.vtime,n);return[...t.values()].sort((n,r)=>n.vtime-r.vtime)}function On(e){if(e.length<=2)return[...e];const t=[e[0]];for(let n=1;n<e.length-1;n++){const r=e[n],i=e[n+1],s=t[t.length-1];if(!ps(r.viseme)){t.push(r);continue}const o=r.vtime-s.vtime,a=i.vtime-r.vtime;o<xn||a<xn||t.push(r)}return t.push(e[e.length-1]),t}function sn(e){return On(Jr(Ln(e)))}function Zr(e,t){let n="sil";for(const r of e){if(r.vtime>t)break;r.viseme!=="sil"&&(n=r.viseme)}return n}function an(e){const t=Jr(Ln(e)),n=On(t),r=new Set(n.map(s=>s.vtime)),i=[];for(const s of t)r.has(s.vtime)||i.push({viseme:s.viseme,vtime:s.vtime});return i}function ws(e){return an(e).map(t=>t.viseme)}const gs=32,cr=48;function ei(e){let t=0;for(const n of e)t=Math.max(t,n.vtime+n.vduration);return t}function ti(e){let t=null,n=-1;for(const r of e){if(r.viseme==="sil")continue;const i=r.vtime+r.vduration;i>n&&(t=r,n=i)}return t}function ys(e){return sn(e).find(n=>n.viseme!=="sil")??null}function rt(e){const t=ei(e);return ti(e)?t+Dn:t}function _s(e){const t=[],n=ys(e);n&&n.vtime>0&&t.push({from:"sil",to:n.viseme,fromVtime:0,toVtime:n.vtime});const r=ti(e),i=ei(e);return r&&t.push({from:r.viseme,to:"sil",fromVtime:i,toVtime:i+Dn}),t}function ni(e){const t=sn(e),n=[];for(let r=0;r<t.length-1;r++){const i=t[r].viseme,s=t[r+1].viseme,o=t[r].vtime,a=t[r+1].vtime;i===s||a<=o||n.push({from:i,to:s,fromVtime:o,toVtime:a})}return n}function ri(e){return e.length===0?[]:[..._s(e),...ni(e)]}function vs(e,t){let n=null;for(const r of e){const{transStart:i,transEnd:s}=ii(r.fromVtime,r.toVtime,r.from,r.to);t>=i&&t<s&&(n=r)}return n}function ii(e,t,n,r){const i=Math.max(0,t-e);if((n==="PP"||r==="PP")&&i>cr){const f=cr,u=t,c=u-f;return{gapMs:i,effectiveGapMs:f,holdEnd:c,transStart:c,transEnd:u}}const o=i>0?Math.max(i,gs):0,a=e,l=e+o;return{gapMs:i,effectiveGapMs:o,holdEnd:e,transStart:a,transEnd:l}}function si(e,t,n,r,i,s){r.forEach(o=>{const l=String(o.word??"").trim();if(!l)return;const f=Math.round((o.start??0)*1e3),u=o.duration!=null?Math.round(o.duration*1e3):o.end!=null?Math.max(1,Math.round((o.end-(o.start??0))*1e3)):0;t.push({word:l,wtime:i+f,wduration:u,queueIndex:t.length,gesture:o.gesture??null,emphasis:o.emphasis===!0})}),n.forEach(o=>{const a=String(o.symbol??o.viseme??"").trim();if(!a)return;const l=Math.round((o.start??0)*1e3),f=Math.max(1,Math.round((o.duration??0)*1e3)),u=i+l,c=ci(t,u),d=c?qr(c,u):0;let p;Or(s)==="oculus_direct"?(p=nn(a),p==="U"&&c&&rn(c.word,d)&&(p="FF")):p=Gr(a,c?.word??"",d),e.push({viseme:p,weight:1,vtime:u,vduration:f})})}function ai(e,t){let n=t;for(const r of e){const i=Math.round((r.start??0)*1e3),s=Math.max(1,Math.round((r.duration??0)*1e3));n=Math.max(n,t+i+s)}return n}const Nn=16e3;function oi(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:Nn}function Ss(e){let t=e.trim();const n=t.indexOf(",");t.startsWith("data:")&&n>=0&&(t=t.slice(n+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const r=t.length%4;return r!==0&&(t+="=".repeat(4-r)),t}function bs(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function ur(e,t,n=Nn){const r=t.byteOffset%2===0?t:t.slice(),i=new Int16Array(r.buffer,r.byteOffset,r.byteLength/2),s=new Float32Array(i.length);for(let a=0;a<i.length;a++)s[a]=i[a]>=32768?-(65536-i[a])/32768:i[a]/32767;const o=e.createBuffer(1,s.length,n);return o.copyToChannel(s,0),o}function Ts(e){const t=Ss(e),n=window.atob(t),r=new Uint8Array(n.length);for(let i=0;i<n.length;i++)r[i]=n.charCodeAt(i);return r}async function lr(e,t,n=Nn){const r=Ts(t),i=oi(n);if(bs(r)){const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return e.decodeAudioData(s)}if(r.length>=2&&r.length%2===0)return ur(e,r,i);try{const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return await e.decodeAudioData(s)}catch{return ur(e,r,i)}}function ci(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function ui(e,t){const n=[],r=[];let i=null,s=!1,o=!1,a=!1,l=!1,f=0,u=0,c=0,d=null,p=1,T=t,m=!1,A=null,C=0,x=!1,g=!1,E=0,N=null,I=null,te=null;const de=()=>({visemes:[],words:[],segmentOffsetMs:0}),P=()=>{g=!1,E=0},W=()=>{d!=null&&(clearTimeout(d),d=null)},K=()=>{for(const S of r){try{S.onended=null,S.stop(0)}catch{}try{S.disconnect()}catch{}}r.length=0},X=()=>{for(const S of r)S.playbackRate.value=p},Y=()=>{if(!(m||!l||s||g||r.length>0||n.length>0)){if(a&&i&&u>0){const S=(i.currentTime-u)*1e3;if(S<C){W();const $=p>0?(C-S)/p:C-S;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,$));return}}o=!1,W(),e.onSpeakingDone(),e.onNotify()}},pe=S=>{const $=rt(S);if($<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,f=performance.now(),e.onSpeakingStart(),e.onNotify(),W();const b=p>0?$/p:$;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},b)},j=(S,$,b,L)=>{$.length!==0&&(c=b,$.forEach(B=>{const _=String(B.word??"").trim();if(!_)return;const v=Math.round((B.start??0)*1e3),k=B.duration!=null?Math.round(B.duration*1e3):B.end!=null?Math.max(1,Math.round((B.end-(B.start??0))*1e3)):0;S.push({word:_,wtime:c+v,wduration:k,queueIndex:S.length,gesture:B.gesture??null,emphasis:B.emphasis===!0})}))},Z=(S,$,b,L,B,O)=>{b.length!==0&&(c=L,b.forEach(_=>{const v=String(_.symbol??_.viseme??"").trim();if(!v)return;const k=Math.round((_.start??0)*1e3),H=Math.max(1,Math.round((_.duration??0)*1e3)),J=c+k,ie=ci($,J),xe=ie?qr(ie,J):0;let be;Or(O)==="oculus_direct"?(be=nn(v),be==="U"&&ie&&rn(ie.word,xe)&&(be="FF")):be=Gr(v,ie?.word??"",xe),S.push({viseme:be,weight:1,vtime:J,vduration:H})}),e.onNotify())},M=(S,$,b,L,B,O,_)=>{j($,L,B),Z(S,$,b,B,O,_)},G=(S,$,b)=>{te===S&&S.visemes===$||($.length=0,b.length=0,S.visemes=$,S.words=b,S.segmentOffsetMs=0,te=S,c=0,C=0)},V=async()=>{const S=window.AudioContext||window.webkitAudioContext;return i||(i=new S),i.state==="suspended"&&await i.resume(),A||(A=i.createMediaStreamDestination()),i},ue=(S,$)=>{if(!i)return 0;const b=i,L=b.currentTime,B=L+.02;if($.isFirstAudio){u<=0&&(u=B);const _=(B-L)*1e3;f=performance.now()+_}$.onBeforeStart?.(B);const O=b.createBufferSource();return O.buffer=S,O.playbackRate.value=p,O.connect(b.destination),A&&O.connect(A),r.push(O),m?(r.pop(),B):($.isFirstAudio&&(o=!0,e.onSpeakingStart()),O.onended=()=>{const _=r.indexOf(O);_>=0&&r.splice(_,1),e.onNotify(),!m&&(N&&I?ne(N,I):Y())},O.start(B),B)},ne=async(S,$)=>{if(m||s||r.length>0)return;if(n.length===0){Y();return}s=!0;let b=!1;try{const L=await V();if(m)return;const B=n[0];if(!B)return;const O=!o;if(x&&(e.shouldHoldForWarmup?.()??!1)&&(g||O&&B.newStream)||g){G(B.utterance,S,$);for(const k of n){if(m)return;k.audioBuffer||(k.audioBuffer=await lr(L,k.audio,k.sampleRate)),a=!0,g||(g=!0,E=0,k.newStream&&k.onNewStream?.()),k.visemesQueued||(M(S,$,k.visemes,k.words,E,!0,T),k.visemesQueued=!0,E+=Math.round(k.audioBuffer.duration*1e3),C=Math.max(C,rt(S)))}e.onNotify();return}const v=n.shift();if(!v||(v.audioBuffer||(v.audioBuffer=await lr(L,v.audio,v.sampleRate)),m))return;a=!0,W(),G(v.utterance,S,$),v.visemesQueued=!1,v.newStream&&v.onNewStream?.(),ue(v.audioBuffer,{isFirstAudio:!o,isFirstOfStream:!1,onBeforeStart:k=>{u=k,c=0,f=performance.now()+(k-L.currentTime)*1e3,S.length=0,$.length=0,M(S,$,v.visemes,v.words,0,!0,T),v.visemesQueued=!0,v.utterance.segmentOffsetMs=0}}),b=r.length>0,C=rt(S),e.onNotify()}catch(L){console.error("[avatarChunkPlayback] audio chunk failed:",L)}finally{if(s=!1,m)return;if(g){n.some(B=>!B.audioBuffer||!B.visemesQueued)&&ne(S,$);return}!b&&r.length===0&&n.length>0?ne(S,$):r.length===0&&n.length===0&&Y()}};return{setStreamTtsEngineId:S=>{T=S},setPlaybackSpeed:S=>{p=Math.max(.1,Math.min(1,S)),X()},getPlaybackSpeed:()=>p,setLipsyncWarmupHoldEnabled:S=>{x=S,S||P()},isWarmingUp:()=>g,releaseWarmupHold:()=>{if(g){g=!1,E=0;for(const S of n)S.newStream=!1;N&&I&&ne(N,I),e.onNotify()}},resetForNewStream:S=>{m=!1,l=!1,T=S,C=0},resumeBotStream:()=>{m=!1,l=!1},clearAborted:()=>{m=!1},primeAudioContext:async()=>{const S=window.AudioContext||window.webkitAudioContext;i||(i=new S),i.state==="suspended"&&await i.resume(),i&&!A&&(A=i.createMediaStreamDestination())},stop:()=>{m=!0,W(),K(),n.length=0,s=!1,o=!1,a=!1,l=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},markStreamComplete:S=>{if(C=Math.max(C,rt(S)),l=!0,!a&&S.length>0){pe(S);return}Y()},playAudioChunk:async(S,$,b,L,B,O,_)=>{if(m)return;N=S,I=$;const v=oi(_?.sampleRate),k=_?.newStream===!0,H=de();if(!b?.trim()){if(L.length===0&&B.length===0)return;G(H,S,$),k&&_?.onNewStream?.(),M(S,$,L,B,0,!0,T),C=rt(S),!o&&C>0&&pe(S),e.onNotify();return}n.push({audio:b,audioBuffer:null,utterance:H,visemes:L,words:B,isNewSegment:O,sampleRate:v,newStream:k,onNewStream:_?.onNewStream,visemesQueued:!1}),await ne(S,$)},queueTimingsOnly:(S,$,b,L,B,O)=>{if(m)return;const _=de();G(_,S,$),M(S,$,b,L,0,!0,T),_.segmentOffsetMs=0,C=rt(S)},getPlaybackElapsedMs:()=>a&&i&&u>0?Math.max(0,(i.currentTime-u)*1e3*p):o?Math.max(0,(performance.now()-f)*p):0,isSpeaking:()=>o?!0:!a||!i||u<=0?!1:(i.currentTime-u)*1e3<C,getVoiceStream:()=>A?A.stream:null,getPlaybackStartPerf:()=>f,bargeIn:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},interrupt:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()}}}function Bt(e,t){let n=null;for(const r of e)t<r.vtime||t>=r.vtime+r.vduration||(!n||r.vtime>=n.vtime)&&(n=r);return n?n.viseme:"sil"}function li(e,t,n=120){const r=Bt(e,t);if(r!=="sil")return r;let i=null;for(const s of e){if(s.vtime>t)break;const o=s.vtime+s.vduration;o<=t&&t-o<=n&&(i=s)}return i?.viseme??"sil"}function ks(e,t,n=400){const r=t-n;let i=0;for(const s of e)if(s.vtime+s.vduration<r)i+=1;else break;i>0&&e.splice(0,i)}function xs(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function fr(e,t){const n=e.trim();if(!n)return null;try{const r=n.replace(/\s+/g,""),i=atob(r).length,s=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return i/2/s*1e3}catch{return null}}const on=.5,cn=1.5,Vn="Olivia",Un="f786b574-daa5-4673-aa0c-cbe3e8534c02",Ft=.85;function En(e,t=ot,n={}){const r=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,i=[],s=[],o=[],a=[];let l=0;const f=[],u=[];let c="",d="neutral",p="",T=st;const m=new Set;let A=0,C=null,x=0,g=null;const E=()=>m.forEach(j=>j());let N=()=>!1;const I=ui({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:E,shouldHoldForWarmup:()=>N()},st),te=()=>I.getPlaybackElapsedMs(),de=(j,Z=c)=>{if(!n.onLipsyncDebug)return;const M=te(),G=i,V=an(o).map(ue=>({viseme:ue.viseme,vtimeMs:ue.vtime}));n.onLipsyncDebug({event:j,spokenText:Z,mood:d,playbackAnchorPerf:I.getPlaybackStartPerf(),playbackElapsedMs:M,queueLength:G.length,activeViseme:Bt(G,M),segments:u.map(ue=>({...ue,visemes:[...ue.visemes]})),fullQueueLength:o.length,compactionSkipped:V})},P=(j,Z)=>{n.onLipsyncDebug&&(C={event:j,spokenText:Z},A===0&&(A=requestAnimationFrame(()=>{A=0;const M=C;C=null,M&&de(M.event,M.spokenText)})))},W=(j,Z,M,G,V)=>{if(!n.onLipsyncDebug)return;const ue=i.slice(Z),ne=M.filter(ae=>String(ae.symbol??ae.viseme??"").trim());u.push({spokenText:j,audioOffsetMs:ue[0]?.vtime??0,isNewSegment:G,chunkDurationMs:V,receivedAtMs:performance.now(),visemes:ue.map((ae,we)=>{const oe=ne[we];return{rawSymbol:String(oe?.symbol??oe?.viseme??ae.viseme).trim(),viseme:ae.viseme,startSec:Number(oe?.start??0),durationSec:Number(oe?.duration??0),vtimeMs:ae.vtime,vdurationMs:ae.vduration}})})},K=()=>{o.length=0,a.length=0,l=0},X=(j,Z,M,G,V)=>{if(j.length===0&&Z.length===0)return;si(o,a,j,Z,l,V);const ue=ai(j,l),ne=M?fr(M,G):null;l=Math.max(ue,ne!=null?l+ne:l)},Y=()=>{x+=1,g?.abort(),g=null,I.stop(),i.length=0,s.length=0,K(),f.length=0,d="neutral",u.length=0,c="",e("idle"),P("reset"),E()};return{setDeveloperToken:j=>{p=j},setTtsEngineId:j=>{T=j},getTtsEngineId:()=>T,speak:async(j,Z)=>{Y();const M=x;g=new AbortController,e("loading"),c=j,u.length=0,K();const G=p.trim();if(!G)throw new Error("Developer token required for avatar_ttsWithPoses");const V=Z?.ttsEngineId??T??st;I.resetForNewStream(V),I.setStreamTtsEngineId(V);const ue=Z?.voiceId??Vn,ne=Z?.speakingRate??Ft,ae=Math.max(on,Math.min(cn,ne)),we={user_query:j,ttsEngineId:V,speakingRate:ae,tenant:"aiTwin"};Fr.has(V)&&(we.voice_id=ue);try{const oe=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${G}`},body:JSON.stringify(we),signal:g.signal});if(!oe.ok)throw new Error(`avatar_ttsWithPoses failed (${oe.status})`);const S=oe.body;if(!S)throw new Error("No response body");const $=S.getReader(),b=new TextDecoder;let L="",B=!0;const O=async _=>{if(M!==x)return;const v=Kr(_);if(v){if(v.event==="audio"){const k=v.data.chunk,H=v.data.visemes??[],J=v.data.words??[],ie=v.data.is_new_segment??!1,xe=v.data.sample_rate,be=Number(v.data.audio_offset_ms??0),Re=i.length;if(X(H,J,k,xe,V),k){const De=B;B=!1,await I.playAudioChunk(i,s,k,H,J,ie,{sampleRate:xe,newStream:De,audioOffsetMs:be}),W(j,Re,H,ie,fr(k,xe)),P("audio_chunk",j)}else(H.length>0||J.length>0)&&(I.queueTimingsOnly(i,s,H,J,ie,{audioOffsetMs:be,newStream:B}),W(j,Re,H,ie,null),P("audio_chunk",j))}else if(v.event==="metadata"){const k=v.data.mood;typeof k=="string"&&k.trim()&&(d=k.trim());const H=v.data.sentence_emotions;if(Array.isArray(H)){f.length=0;for(const J of H){if(!J||typeof J!="object")continue;const ie=J;f.push({sentence_index:Number(ie.sentence_index??0),text:String(ie.text??""),sentiment:typeof ie.sentiment=="string"?ie.sentiment:void 0,emotion:String(ie.emotion??d),start_word:Number(ie.start_word??0),end_word:Number(ie.end_word??0)})}}E(),P("metadata",j)}else if(v.event==="error")throw new Error(String(v.data.message??"avatar_ttsWithPoses stream error"))}};for(;M===x;){const{done:_,value:v}=await $.read();v&&(L+=b.decode(v,{stream:!0}));const k=L.split(`
|
|
1
|
+
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const z=require("react/jsx-runtime"),w=require("react"),Li=require("@pipecat-ai/client-js"),Cn=require("@pipecat-ai/websocket-transport"),he=require("pixi.js");var wn=typeof document<"u"?document.currentScript:null;const Oi="https://aitwin.bubu.social",Br={publicBase:Oi},Ni="0.5.7",Vi={version:Ni},Ui={},ot="https://ai.aitwin.me",Rn=`${Br.publicBase}/custom-faces`,Wi=`${Br.publicBase}/aitwin-workers/v${Vi.version}/visemeDiffPreview.worker.js`;function Hi(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:wn&&wn.tagName.toUpperCase()==="SCRIPT"&&wn.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?Ui?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():Wi}const $i="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",gn=3e3,Pt=16,ar=3e3,zi=4e3,Ki=.85,ji=8,Dn=400,Bn="eng_a7f2b9c1",tn="eng_d4e8f3a2",ct="eng_c9b1e6d4",st=ct,Fr=new Set([tn,ct]),Lr={[Bn]:"oculus_direct",[ct]:"oculus_direct",[tn]:"inworld_word"},or={google:Bn,inworld:tn,cartesia:ct};function Or(e){return Lr[e]}function qi(e){return e===ct}function Gi(e){if(e.ttsEngineId&&e.ttsEngineId in Lr)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in or?or[t]:st}class Gt extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function Nr(e,t=$i){const n=new URL(t);n.searchParams.set("id",e);const r=await fetch(n.toString());if(r.status===404)throw new Gt;if(!r.ok)throw new Error(`getAiTwin failed (${r.status})`);const i=await r.json();if(!i.success)throw new Gt(i.error??"AI twin not found");const{tts:s,token:o,...a}=i.data,l=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:Gi(i.data),authToken:l}}const Yi="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",Qi="client_6TNvp3SCs4Og0a1ijm9TommXLql1",Xi="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",Ji="6TNvp3SCs4Og0a1ijm9TommXLql1",Zi="Swaraj Mali";async function Mt(){const e=await fetch(Yi,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":Qi,"Client-Secret":Xi},body:JSON.stringify({userId:Ji,userName:Zi})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),n=t.authToken??t.token??"";if(!n.trim())throw new Error("getAuthToken returned no token");return n}async function es(e,t=ot){const n=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!n.ok)throw new Error(`/api/session-value failed (${n.status})`);const r=await n.json();if(!r.value)throw new Error("Session value response missing value");return r}function ts(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const n=new Uint8Array(t.length/2);for(let r=0;r<t.length;r+=2)n[r/2]=Number.parseInt(t.slice(r,r+2),16);return n}function Ht(e){return Uint8Array.from(e)}function ns(e,t){const n=new Uint8Array(e.length+t.length);return n.set(e,0),n.set(t,e.length),n}async function rs(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const n=new Uint8Array(e),r=Ht(n.subarray(0,12)),i=Ht(n.subarray(12,28)),s=Ht(n.subarray(28)),o=Ht(ns(s,i)),a=await crypto.subtle.importKey("raw",ts(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:r,tagLength:128},a,o)}let bn=null;const Tn=new Set;function $t(e){return e.includes("/custom-faces/")}function Yt(e){const t=e.binBase.trim().replace(/\/$/,""),n=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||$t(t)||$t(n)))return{twinBase:n,binBase:t,encrypted:!1};const i=$t(t)?t:$t(n)?n:t;return{twinBase:i,binBase:i,encrypted:!0}}function ut(e){return Yt(e).encrypted}function kn(e,t=Rn){const n=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:n,binBase:n,encrypted:!0}}function Vr(e,t){return(e??t)?.trim()||void 0}function Ur(e){bn=Yt(e);for(const t of Tn)t()}function is(e,t){const n=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!n)return Yt(e);const r=i=>{try{return new URL(i,n).href.replace(/\/$/,"")}catch{return i}};return Yt({twinBase:r(e.twinBase),binBase:r(e.binBase),encrypted:e.encrypted})}function lt(){if(!bn)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return bn}function Wr(e){return Tn.add(e),()=>Tn.delete(e)}function Qt(){const e=lt();return ut(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function ss(){const e=lt();return ut(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function Hr(){const e=lt();return ut(e)?e.binBase:e.twinBase}function Dt(){return ut(lt())}function $r(){const e=lt();return ut(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function zr(){const e=lt();return ut(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function Kr(e){const t=e.split(/\r?\n/);let n="",r="";for(const s of t)s.startsWith("event:")?n=s.slice(6).trim():s.startsWith("data:")&&(r=s.slice(5).trim());if(!n)return null;let i={};if(r)try{i=JSON.parse(r)}catch{i={raw:r}}return{event:n,data:i}}const jr=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],as=new Map(jr.map(e=>[e.toLowerCase(),e]));function nn(e){const t=(e??"").trim();return t?as.get(t.toLowerCase())??"sil":"sil"}const os={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function cs(e){if(!e)return["sil"];const t=e.toLowerCase();return os[t]??["sil"]}function qr(e,t){const n=e.word.length;if(n<=0)return 0;const r=e.wduration;if(r<=0)return 0;const i=Math.max(0,Math.min(1,(t-e.wtime)/r));return Math.min(n-1,Math.floor(i*n))}function yt(e,t){const n=Math.max(0,Math.min(e.length-1,t));return e[n]??""}function rn(e,t){const n=e.toLowerCase().replace(/[^a-z']/g,"");return!n||yt(n,t)!=="w"||t>0&&yt(n,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(n)}function us(e,t){const n=e.toLowerCase(),r=yt(n,t),i=n.slice(Math.max(0,t-1),t+2);return r==="i"||r==="y"?"I":i.includes("ee")||r==="e"||n.includes("ea")?"E":"aa"}function ls(e,t){const n=e.toLowerCase(),r=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));if(/oo|ou|uw/.test(r))return"U";if(rn(n,t))return"FF";const i=yt(n,t);return i==="u"||i==="w"?"U":"O"}function fs(e,t){const n=e.toLowerCase();if(rn(n,t))return"FF";const r=n.slice(Math.max(0,t-2),t+1);if(/qu/.test(r))return"U";const i=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));return/wo|wh/.test(i)||yt(n,t)==="o"?"O":"U"}const zt={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function ds(e,t){const n=e.toLowerCase(),r=yt(n,t);if(zt[r])return zt[r];for(let i=0;i<n.length;i++){const s=n[i];if(zt[s])return zt[s]}return"DD"}function Gr(e,t,n){const r=cs(e);if(r.length===1)return r[0];const i=t.trim();if(!i)return r[0];const s=(e??"").toLowerCase(),o=Math.max(0,n);switch(s){case"aei":return us(i,o);case"o":return ls(i,o);case"qw":return fs(i,o);case"cdgknstxyz":return ds(i,o);default:return r[0]}}const Fn=16,xn=47,Yr=new Set(["aa","E","I"]),Qr=new Set(["DD","SS","kk","CH"]),ms=[Yr,Qr];function hs(e){for(const t of ms)if(t.has(e))return t;return null}function ps(e){return hs(e)!=null}function Xr(e,t=Fn){return t<=0?e:Math.round(e/t)*t}function Ln(e,t=Fn){return e.map(n=>({...n,vtime:Xr(n.vtime,t)}))}function Jr(e){const t=new Map;for(const n of e)t.has(n.vtime)||t.set(n.vtime,n);return[...t.values()].sort((n,r)=>n.vtime-r.vtime)}function On(e){if(e.length<=2)return[...e];const t=[e[0]];for(let n=1;n<e.length-1;n++){const r=e[n],i=e[n+1],s=t[t.length-1];if(!ps(r.viseme)){t.push(r);continue}const o=r.vtime-s.vtime,a=i.vtime-r.vtime;o<xn||a<xn||t.push(r)}return t.push(e[e.length-1]),t}function sn(e){return On(Jr(Ln(e)))}function Zr(e,t){let n="sil";for(const r of e){if(r.vtime>t)break;r.viseme!=="sil"&&(n=r.viseme)}return n}function an(e){const t=Jr(Ln(e)),n=On(t),r=new Set(n.map(s=>s.vtime)),i=[];for(const s of t)r.has(s.vtime)||i.push({viseme:s.viseme,vtime:s.vtime});return i}function ws(e){return an(e).map(t=>t.viseme)}const gs=32,cr=48;function ei(e){let t=0;for(const n of e)t=Math.max(t,n.vtime+n.vduration);return t}function ti(e){let t=null,n=-1;for(const r of e){if(r.viseme==="sil")continue;const i=r.vtime+r.vduration;i>n&&(t=r,n=i)}return t}function ys(e){return sn(e).find(n=>n.viseme!=="sil")??null}function rt(e){const t=ei(e);return ti(e)?t+Dn:t}function _s(e){const t=[],n=ys(e);n&&n.vtime>0&&t.push({from:"sil",to:n.viseme,fromVtime:0,toVtime:n.vtime});const r=ti(e),i=ei(e);return r&&t.push({from:r.viseme,to:"sil",fromVtime:i,toVtime:i+Dn}),t}function ni(e){const t=sn(e),n=[];for(let r=0;r<t.length-1;r++){const i=t[r].viseme,s=t[r+1].viseme,o=t[r].vtime,a=t[r+1].vtime;i===s||a<=o||n.push({from:i,to:s,fromVtime:o,toVtime:a})}return n}function ri(e){return e.length===0?[]:[..._s(e),...ni(e)]}function vs(e,t){let n=null;for(const r of e){const{transStart:i,transEnd:s}=ii(r.fromVtime,r.toVtime,r.from,r.to);t>=i&&t<s&&(n=r)}return n}function ii(e,t,n,r){const i=Math.max(0,t-e);if((n==="PP"||r==="PP")&&i>cr){const f=cr,u=t,c=u-f;return{gapMs:i,effectiveGapMs:f,holdEnd:c,transStart:c,transEnd:u}}const o=i>0?Math.max(i,gs):0,a=e,l=e+o;return{gapMs:i,effectiveGapMs:o,holdEnd:e,transStart:a,transEnd:l}}function si(e,t,n,r,i,s){r.forEach(o=>{const l=String(o.word??"").trim();if(!l)return;const f=Math.round((o.start??0)*1e3),u=o.duration!=null?Math.round(o.duration*1e3):o.end!=null?Math.max(1,Math.round((o.end-(o.start??0))*1e3)):0;t.push({word:l,wtime:i+f,wduration:u,queueIndex:t.length,gesture:o.gesture??null,emphasis:o.emphasis===!0})}),n.forEach(o=>{const a=String(o.symbol??o.viseme??"").trim();if(!a)return;const l=Math.round((o.start??0)*1e3),f=Math.max(1,Math.round((o.duration??0)*1e3)),u=i+l,c=ci(t,u),d=c?qr(c,u):0;let p;Or(s)==="oculus_direct"?(p=nn(a),p==="U"&&c&&rn(c.word,d)&&(p="FF")):p=Gr(a,c?.word??"",d),e.push({viseme:p,weight:1,vtime:u,vduration:f})})}function ai(e,t){let n=t;for(const r of e){const i=Math.round((r.start??0)*1e3),s=Math.max(1,Math.round((r.duration??0)*1e3));n=Math.max(n,t+i+s)}return n}const Nn=16e3;function oi(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:Nn}function Ss(e){let t=e.trim();const n=t.indexOf(",");t.startsWith("data:")&&n>=0&&(t=t.slice(n+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const r=t.length%4;return r!==0&&(t+="=".repeat(4-r)),t}function bs(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function ur(e,t,n=Nn){const r=t.byteOffset%2===0?t:t.slice(),i=new Int16Array(r.buffer,r.byteOffset,r.byteLength/2),s=new Float32Array(i.length);for(let a=0;a<i.length;a++)s[a]=i[a]>=32768?-(65536-i[a])/32768:i[a]/32767;const o=e.createBuffer(1,s.length,n);return o.copyToChannel(s,0),o}function Ts(e){const t=Ss(e),n=window.atob(t),r=new Uint8Array(n.length);for(let i=0;i<n.length;i++)r[i]=n.charCodeAt(i);return r}async function lr(e,t,n=Nn){const r=Ts(t),i=oi(n);if(bs(r)){const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return e.decodeAudioData(s)}if(r.length>=2&&r.length%2===0)return ur(e,r,i);try{const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return await e.decodeAudioData(s)}catch{return ur(e,r,i)}}function ci(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function ui(e,t){const n=[],r=[];let i=null,s=!1,o=!1,a=!1,l=!1,f=0,u=0,c=0,d=null,p=1,T=t,m=!1,A=null,C=0,x=!1,g=!1,E=0,N=null,I=null,te=null;const de=()=>({visemes:[],words:[],segmentOffsetMs:0}),P=()=>{g=!1,E=0},W=()=>{d!=null&&(clearTimeout(d),d=null)},K=()=>{for(const S of r){try{S.onended=null,S.stop(0)}catch{}try{S.disconnect()}catch{}}r.length=0},X=()=>{for(const S of r)S.playbackRate.value=p},Y=()=>{if(!(m||!l||s||g||r.length>0||n.length>0)){if(a&&i&&u>0){const S=(i.currentTime-u)*1e3;if(S<C){W();const $=p>0?(C-S)/p:C-S;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,$));return}}o=!1,W(),e.onSpeakingDone(),e.onNotify()}},pe=S=>{const $=rt(S);if($<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,f=performance.now(),e.onSpeakingStart(),e.onNotify(),W();const b=p>0?$/p:$;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},b)},j=(S,$,b,L)=>{$.length!==0&&(c=b,$.forEach(B=>{const _=String(B.word??"").trim();if(!_)return;const v=Math.round((B.start??0)*1e3),k=B.duration!=null?Math.round(B.duration*1e3):B.end!=null?Math.max(1,Math.round((B.end-(B.start??0))*1e3)):0;S.push({word:_,wtime:c+v,wduration:k,queueIndex:S.length,gesture:B.gesture??null,emphasis:B.emphasis===!0})}))},Z=(S,$,b,L,B,O)=>{b.length!==0&&(c=L,b.forEach(_=>{const v=String(_.symbol??_.viseme??"").trim();if(!v)return;const k=Math.round((_.start??0)*1e3),H=Math.max(1,Math.round((_.duration??0)*1e3)),J=c+k,ie=ci($,J),xe=ie?qr(ie,J):0;let be;Or(O)==="oculus_direct"?(be=nn(v),be==="U"&&ie&&rn(ie.word,xe)&&(be="FF")):be=Gr(v,ie?.word??"",xe),S.push({viseme:be,weight:1,vtime:J,vduration:H})}),e.onNotify())},M=(S,$,b,L,B,O,_)=>{j($,L,B),Z(S,$,b,B,O,_)},G=(S,$,b)=>{te===S&&S.visemes===$||($.length=0,b.length=0,S.visemes=$,S.words=b,S.segmentOffsetMs=0,te=S,c=0,C=0)},V=async()=>{const S=window.AudioContext||window.webkitAudioContext;return i||(i=new S),i.state==="suspended"&&await i.resume(),A||(A=i.createMediaStreamDestination()),i},ue=(S,$)=>{if(!i)return 0;const b=i,L=b.currentTime,B=L+.02;if($.isFirstAudio){u<=0&&(u=B);const _=(B-L)*1e3;f=performance.now()+_}$.onBeforeStart?.(B);const O=b.createBufferSource();return O.buffer=S,O.playbackRate.value=p,O.connect(b.destination),A&&O.connect(A),r.push(O),m?(r.pop(),B):($.isFirstAudio&&(o=!0,e.onSpeakingStart()),O.onended=()=>{const _=r.indexOf(O);_>=0&&r.splice(_,1),e.onNotify(),!m&&(N&&I?ne(N,I):Y())},O.start(B),B)},ne=async(S,$)=>{if(m||s||r.length>0)return;if(n.length===0){Y();return}s=!0;let b=!1;try{const L=await V();if(m)return;const B=n[0];if(!B)return;const O=!o;if(x&&(e.shouldHoldForWarmup?.()??!1)&&(g||O&&B.newStream)||g){G(B.utterance,S,$);for(const k of n){if(m)return;k.audioBuffer||(k.audioBuffer=await lr(L,k.audio,k.sampleRate)),a=!0,g||(g=!0,E=0,k.newStream&&k.onNewStream?.()),k.visemesQueued||(M(S,$,k.visemes,k.words,E,!0,T),k.visemesQueued=!0,E+=Math.round(k.audioBuffer.duration*1e3),C=Math.max(C,rt(S)))}e.onNotify();return}const v=n.shift();if(!v||(v.audioBuffer||(v.audioBuffer=await lr(L,v.audio,v.sampleRate)),m))return;a=!0,W(),G(v.utterance,S,$),v.visemesQueued=!1,v.newStream&&v.onNewStream?.(),ue(v.audioBuffer,{isFirstAudio:!o,isFirstOfStream:!1,onBeforeStart:k=>{u=k,c=0,f=performance.now()+(k-L.currentTime)*1e3,S.length=0,$.length=0,M(S,$,v.visemes,v.words,0,!0,T),v.visemesQueued=!0,v.utterance.segmentOffsetMs=0}}),b=r.length>0,C=rt(S),e.onNotify()}catch(L){console.error("[avatarChunkPlayback] audio chunk failed:",L)}finally{if(s=!1,m)return;if(g){n.some(B=>!B.audioBuffer||!B.visemesQueued)&&ne(S,$);return}!b&&r.length===0&&n.length>0?ne(S,$):r.length===0&&n.length===0&&Y()}};return{setStreamTtsEngineId:S=>{T=S},setPlaybackSpeed:S=>{p=Math.max(.1,Math.min(1,S)),X()},getPlaybackSpeed:()=>p,setLipsyncWarmupHoldEnabled:S=>{x=S,S||P()},isWarmingUp:()=>g,releaseWarmupHold:()=>{if(g){g=!1,E=0;for(const S of n)S.newStream=!1;N&&I&&ne(N,I),e.onNotify()}},resetForNewStream:S=>{m=!1,l=!1,T=S,C=0},resumeBotStream:()=>{m=!1,l=!1},clearAborted:()=>{m=!1},primeAudioContext:async()=>{const S=window.AudioContext||window.webkitAudioContext;i||(i=new S),i.state==="suspended"&&await i.resume(),i&&!A&&(A=i.createMediaStreamDestination())},stop:()=>{m=!0,W(),K(),n.length=0,s=!1,o=!1,a=!1,l=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},markStreamComplete:S=>{if(C=Math.max(C,rt(S)),l=!0,!a&&S.length>0){pe(S);return}Y()},playAudioChunk:async(S,$,b,L,B,O,_)=>{if(m)return;N=S,I=$;const v=oi(_?.sampleRate),k=_?.newStream===!0,H=de();if(!b?.trim()){if(L.length===0&&B.length===0)return;G(H,S,$),k&&_?.onNewStream?.(),M(S,$,L,B,0,!0,T),C=rt(S),!o&&C>0&&pe(S),e.onNotify();return}n.push({audio:b,audioBuffer:null,utterance:H,visemes:L,words:B,isNewSegment:O,sampleRate:v,newStream:k,onNewStream:_?.onNewStream,visemesQueued:!1}),await ne(S,$)},queueTimingsOnly:(S,$,b,L,B,O)=>{if(m)return;const _=de();G(_,S,$),M(S,$,b,L,0,!0,T),_.segmentOffsetMs=0,C=rt(S)},getPlaybackElapsedMs:()=>a&&i&&u>0?Math.max(0,(i.currentTime-u)*1e3*p):o?Math.max(0,(performance.now()-f)*p):0,isSpeaking:()=>o?!0:!a||!i||u<=0?!1:(i.currentTime-u)*1e3<C,getVoiceStream:()=>A?A.stream:null,getPlaybackStartPerf:()=>f,bargeIn:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},interrupt:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()}}}function Bt(e,t){let n=null;for(const r of e)t<r.vtime||t>=r.vtime+r.vduration||(!n||r.vtime>=n.vtime)&&(n=r);return n?n.viseme:"sil"}function li(e,t,n=120){const r=Bt(e,t);if(r!=="sil")return r;let i=null;for(const s of e){if(s.vtime>t)break;const o=s.vtime+s.vduration;o<=t&&t-o<=n&&(i=s)}return i?.viseme??"sil"}function ks(e,t,n=400){const r=t-n;let i=0;for(const s of e)if(s.vtime+s.vduration<r)i+=1;else break;i>0&&e.splice(0,i)}function xs(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function fr(e,t){const n=e.trim();if(!n)return null;try{const r=n.replace(/\s+/g,""),i=atob(r).length,s=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return i/2/s*1e3}catch{return null}}const on=.5,cn=1.5,Vn="Olivia",Un="f786b574-daa5-4673-aa0c-cbe3e8534c02",Ft=.85;function En(e,t=ot,n={}){const r=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,i=[],s=[],o=[],a=[];let l=0;const f=[],u=[];let c="",d="neutral",p="",T=st;const m=new Set;let A=0,C=null,x=0,g=null;const E=()=>m.forEach(j=>j());let N=()=>!1;const I=ui({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:E,shouldHoldForWarmup:()=>N()},st),te=()=>I.getPlaybackElapsedMs(),de=(j,Z=c)=>{if(!n.onLipsyncDebug)return;const M=te(),G=i,V=an(o).map(ue=>({viseme:ue.viseme,vtimeMs:ue.vtime}));n.onLipsyncDebug({event:j,spokenText:Z,mood:d,playbackAnchorPerf:I.getPlaybackStartPerf(),playbackElapsedMs:M,queueLength:G.length,activeViseme:Bt(G,M),segments:u.map(ue=>({...ue,visemes:[...ue.visemes]})),fullQueueLength:o.length,compactionSkipped:V})},P=(j,Z)=>{n.onLipsyncDebug&&(C={event:j,spokenText:Z},A===0&&(A=requestAnimationFrame(()=>{A=0;const M=C;C=null,M&&de(M.event,M.spokenText)})))},W=(j,Z,M,G,V)=>{if(!n.onLipsyncDebug)return;const ue=i.slice(Z),ne=M.filter(ae=>String(ae.symbol??ae.viseme??"").trim());u.push({spokenText:j,audioOffsetMs:ue[0]?.vtime??0,isNewSegment:G,chunkDurationMs:V,receivedAtMs:performance.now(),visemes:ue.map((ae,we)=>{const oe=ne[we];return{rawSymbol:String(oe?.symbol??oe?.viseme??ae.viseme).trim(),viseme:ae.viseme,startSec:Number(oe?.start??0),durationSec:Number(oe?.duration??0),vtimeMs:ae.vtime,vdurationMs:ae.vduration}})})},K=()=>{o.length=0,a.length=0,l=0},X=(j,Z,M,G,V)=>{if(j.length===0&&Z.length===0)return;si(o,a,j,Z,l,V);const ue=ai(j,l),ne=M?fr(M,G):null;l=Math.max(ue,ne!=null?l+ne:l)},Y=()=>{x+=1,g?.abort(),g=null,I.stop(),i.length=0,s.length=0,K(),f.length=0,d="neutral",u.length=0,c="",e("idle"),P("reset"),E()};return{setDeveloperToken:j=>{p=j},setTtsEngineId:j=>{T=j},getTtsEngineId:()=>T,speak:async(j,Z)=>{Y();const M=x;g=new AbortController,e("loading"),c=j,u.length=0,K();const G=p.trim();if(!G)throw new Error("Developer token required for avatar_ttsWithPoses");const V=Z?.ttsEngineId??T??st;I.resetForNewStream(V),I.setStreamTtsEngineId(V);const ue=Z?.voiceId??Vn,ne=Z?.speakingRate??Ft,ae=Math.max(on,Math.min(cn,ne)),we={user_query:j,ttsEngineId:V,speakingRate:ae,tenant:"aiTwin"};Fr.has(V)&&(we.voice_id=ue);try{const oe=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${G}`},body:JSON.stringify(we),signal:g.signal});if(!oe.ok)throw new Error(`avatar_ttsWithPoses failed (${oe.status})`);const S=oe.body;if(!S)throw new Error("No response body");const $=S.getReader(),b=new TextDecoder;let L="",B=!0;const O=async _=>{if(M!==x)return;const v=Kr(_);if(v){if(v.event==="audio"){const k=v.data.chunk,H=v.data.visemes??[],J=v.data.words??[],ie=v.data.is_new_segment??!1,xe=v.data.sample_rate,be=Number(v.data.audio_offset_ms??0),Re=i.length;if(X(H,J,k,xe,V),k){const De=B;B=!1,await I.playAudioChunk(i,s,k,H,J,ie,{sampleRate:xe,newStream:De,audioOffsetMs:be}),W(j,Re,H,ie,fr(k,xe)),P("audio_chunk",j)}else(H.length>0||J.length>0)&&(I.queueTimingsOnly(i,s,H,J,ie,{audioOffsetMs:be,newStream:B}),W(j,Re,H,ie,null),P("audio_chunk",j))}else if(v.event==="metadata"){const k=v.data.mood;typeof k=="string"&&k.trim()&&(d=k.trim());const H=v.data.sentence_emotions;if(Array.isArray(H)){f.length=0;for(const J of H){if(!J||typeof J!="object")continue;const ie=J;f.push({sentence_index:Number(ie.sentence_index??0),text:String(ie.text??""),sentiment:typeof ie.sentiment=="string"?ie.sentiment:void 0,emotion:String(ie.emotion??d),start_word:Number(ie.start_word??0),end_word:Number(ie.end_word??0)})}}E(),P("metadata",j)}else if(v.event==="error")throw new Error(String(v.data.message??"avatar_ttsWithPoses stream error"))}};for(;M===x;){const{done:_,value:v}=await $.read();v&&(L+=b.decode(v,{stream:!0}));const k=L.split(`
|
|
2
2
|
|
|
3
|
-
`);L=k.pop()??"";for(const H of k)await O(H);if(_){L.trim()&&await O(L.trim()),I.markStreamComplete(i),P("bot_stopped",j);break}}}catch(oe){if(M!==x||oe instanceof DOMException&&oe.name==="AbortError")return;throw console.error("[avatarTtsLipsync]",oe),Y(),e("error"),oe}},stop:Y,getVisemeQueue:()=>i,getWordQueue:()=>s,getFullVisemeQueue:()=>o,getFullWordQueue:()=>a,getSentenceEmotions:()=>f,getStreamMood:()=>d,getPlaybackElapsedMs:()=>I.getPlaybackElapsedMs(),isSpeaking:()=>I.isSpeaking(),isWarmingUp:()=>I.isWarmingUp(),releaseWarmupHold:()=>I.releaseWarmupHold(),setLipsyncWarmupHold:(j,Z)=>{Z&&(N=Z),I.setLipsyncWarmupHoldEnabled(j)},getVoiceStream:()=>I.getVoiceStream(),setPlaybackSpeed:j=>{I.setPlaybackSpeed(j)},getPlaybackSpeed:()=>I.getPlaybackSpeed(),subscribe:j=>(m.add(j),()=>m.delete(j))}}const Wn=16e3,Hn=16e3;class Es extends Cn.ProtobufFrameSerializer{async deserialize(t){try{return await super.deserialize(t)}catch(n){if(n instanceof Error&&n.message==="Unknown frame kind")return{type:"raw",message:null};throw n}}}class Is extends Cn.DailyMediaManager{constructor(){super(!0,!0,void 0,void 0,512,Wn,Hn)}bufferBotAudio(t,n){}}const As=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function Ps(e){return As.test(e.trim())}function Ms(e){const t=e.replace(/\/$/,"");return t.startsWith("https://")?`wss://${t.slice(8)}`:t.startsWith("http://")?`ws://${t.slice(7)}`:t}function fi(e){if(e.wsUrl?.trim())return e.wsUrl.trim();const t=e.authToken?.trim();if(!t)throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");const n=e.apiBase??ot,r=new URL(`${Ms(n)}/ws/voice`);r.searchParams.set("authToken",t),r.searchParams.set("tenant","aiTwin");const i=e.voiceId?.trim();i&&Ps(i)&&r.searchParams.set("voiceId",i);const s=e.personaId?.trim();s&&r.searchParams.set("personaId",s);const o=e.speakingRate??Ft;
|
|
3
|
+
`);L=k.pop()??"";for(const H of k)await O(H);if(_){L.trim()&&await O(L.trim()),I.markStreamComplete(i),P("bot_stopped",j);break}}}catch(oe){if(M!==x||oe instanceof DOMException&&oe.name==="AbortError")return;throw console.error("[avatarTtsLipsync]",oe),Y(),e("error"),oe}},stop:Y,getVisemeQueue:()=>i,getWordQueue:()=>s,getFullVisemeQueue:()=>o,getFullWordQueue:()=>a,getSentenceEmotions:()=>f,getStreamMood:()=>d,getPlaybackElapsedMs:()=>I.getPlaybackElapsedMs(),isSpeaking:()=>I.isSpeaking(),isWarmingUp:()=>I.isWarmingUp(),releaseWarmupHold:()=>I.releaseWarmupHold(),setLipsyncWarmupHold:(j,Z)=>{Z&&(N=Z),I.setLipsyncWarmupHoldEnabled(j)},getVoiceStream:()=>I.getVoiceStream(),setPlaybackSpeed:j=>{I.setPlaybackSpeed(j)},getPlaybackSpeed:()=>I.getPlaybackSpeed(),subscribe:j=>(m.add(j),()=>m.delete(j))}}const Wn=16e3,Hn=16e3;class Es extends Cn.ProtobufFrameSerializer{async deserialize(t){try{return await super.deserialize(t)}catch(n){if(n instanceof Error&&n.message==="Unknown frame kind")return{type:"raw",message:null};throw n}}}class Is extends Cn.DailyMediaManager{constructor(){super(!0,!0,void 0,void 0,512,Wn,Hn)}bufferBotAudio(t,n){}}const As=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function Ps(e){return As.test(e.trim())}function Ms(e){const t=e.replace(/\/$/,"");return t.startsWith("https://")?`wss://${t.slice(8)}`:t.startsWith("http://")?`ws://${t.slice(7)}`:t}function fi(e){if(e.wsUrl?.trim())return e.wsUrl.trim();const t=e.authToken?.trim();if(!t)throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");const n=e.apiBase??ot,r=new URL(`${Ms(n)}/ws/voice`);r.searchParams.set("authToken",t),r.searchParams.set("tenant","aiTwin");const i=e.voiceId?.trim();i&&Ps(i)&&r.searchParams.set("voiceId",i);const s=e.personaId?.trim();s&&r.searchParams.set("personaId",s);const o=e.speakingRate??Ft;r.searchParams.set("speaking_rate",String(Math.max(on,Math.min(cn,o)))),e.ttsStream===!1?r.searchParams.set("tts_stream","false"):r.searchParams.set("tts_stream","true"),e.speakOnlyWhenSpokenTo&&r.searchParams.set("speakOnlyWhenSpokenTo","true");const a=e.avatarName?.trim();return a&&r.searchParams.set("avatarName",a),r.toString()}function yn(e,t){if(!Array.isArray(e))return[];const n=[];for(const r of e)!r||typeof r!="object"||n.push({sentence_index:Number(r.sentence_index??0),text:String(r.text??""),sentiment:typeof r.sentiment=="string"?r.sentiment:void 0,emotion:String(r.emotion??t),start_word:Number(r.start_word??0),end_word:Number(r.end_word??0)});return n}function Cs(e){if(!e||typeof e!="object")return null;const t=e,r=(t.data&&typeof t.data=="object"&&!Array.isArray(t.data)?t.data:null)||t;return{...r,type:String(r.type??t.type??"server-message")}}function dr(e){return e.is_new_segment===!0||Number(e.audio_offset_ms??-1)===0?!0:Number(e.chunk_index??-1)===0}function Rs(e){return e?.transport?._mediaManager?._wavStreamPlayer??null}function di(e,t={}){const n=[],r=[],i=[],s=[];let o=0;const a=[],l=[];let f="",u="neutral",c=ct,d=!1,p=null,T=null,m={...t},A=0,C=null;const x=new Set,g=[];let E=!1,N=null,I=!0,te="default";const de=new Set;let P=null,W=0,K=!1,X=!1,Y=null,pe=!1;const j=()=>{Y!=null&&(clearTimeout(Y),Y=null)},Z=()=>{j(),Y=setTimeout(()=>{Y=null,I&&(V.markStreamComplete(n),oe("bot_stopped"))},450)},M=()=>x.forEach(h=>h());let G=()=>!1;const V=ui({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>{pe=!1,X=!1,e("done")},onNotify:M,shouldHoldForWarmup:()=>G()},c),ue=()=>{if(N)return N;const h=new MessageChannel;return h.port1.onmessage=()=>{const R=g.shift();R!==void 0&&Re(R),g.length>0?h.port2.postMessage(null):E=!1},N=h,h},ne=h=>{g.push(h),!E&&(E=!0,ue().port2.postMessage(null))},ae=()=>V.getPlaybackElapsedMs(),we=(h,R=f)=>{if(!m.onLipsyncDebug)return;const ee=ae(),le=n,ve=an(i).map(ge=>({viseme:ge.viseme,vtimeMs:ge.vtime}));m.onLipsyncDebug({event:h,spokenText:R,mood:u,playbackAnchorPerf:V.getPlaybackStartPerf(),playbackElapsedMs:ee,queueLength:le.length,activeViseme:Bt(le,ee),segments:l.map(ge=>({...ge,visemes:[...ge.visemes]})),fullQueueLength:i.length,compactionSkipped:ve})},oe=(h,R)=>{m.onLipsyncDebug&&(C={event:h,spokenText:R},A===0&&(A=requestAnimationFrame(()=>{A=0;const ee=C;C=null,ee&&we(ee.event,ee.spokenText)})))},S=()=>{I=!0,de.clear(),V.clearAborted()},$=()=>{const h=Rs(p);if(h){te=`bot-${Date.now()}`,h.interruptedTrackIds={[te]:!0};try{h.interrupt?.()}catch{}}const R=T;try{R?.interrupt?.()}catch{}},b=h=>{if(pe&&V.isSpeaking())return!0;const R=h.context_id?.trim();return R&&de.has(R)?!0:I?!1:!(h.type==="avatar_metadata"||dr(h))},L=h=>{I=!1,pe=!1;const R=h.context_id?.trim();R&&de.add(R),j(),K=!1,X=!1,g.length=0,E=!1,P?.abort(),P=null,W+=1,$(),V.bargeIn(),B(),e("idle"),oe("interrupt"),M()},B=()=>{n.length=0,r.length=0,i.length=0,s.length=0,o=0,l.length=0,f="",u="neutral",a.length=0},O=()=>{j(),X=!1,K=!1,V.stop(),B(),oe("reset"),M()},_=()=>{O(),e("idle"),M()},v=(h,R)=>{typeof h.mood=="string"&&h.mood.trim()&&(u=h.mood.trim()),Array.isArray(h.sentence_emotions)&&h.sentence_emotions.length>0&&(R&&(a.length=0),a.push(...yn(h.sentence_emotions,u)))},k=h=>{if(b({...h,type:"avatar_metadata"}))return;S(),K=!0;const R=String(h.spoken_text??h.text??"").trim();R&&(f=R),V.resetForNewStream(c),typeof h.mood=="string"&&h.mood.trim()&&(u=h.mood.trim()),a.length=0,a.push(...yn(h.sentence_emotions,u)),Array.isArray(h.gestures)&&h.gestures,oe("metadata",R),M()},H=(h,R,ee,le)=>{if(!m.onLipsyncDebug)return;const ve=n.slice(ee),ge=le.filter(ye=>String(ye.symbol??ye.viseme??"").trim());l.push({spokenText:R,audioOffsetMs:ve[0]?.vtime??0,isNewSegment:!!h.is_new_segment,chunkDurationMs:h.chunk_duration_ms!=null?Number(h.chunk_duration_ms):null,receivedAtMs:performance.now(),visemes:ve.map((ye,Be)=>{const _e=ge[Be];return{rawSymbol:String(_e?.symbol??_e?.viseme??ye.viseme).trim(),viseme:ye.viseme,startSec:Number(_e?.start??0),durationSec:Number(_e?.duration??0),vtimeMs:ye.vtime,vdurationMs:ye.vduration}})})},J=(h,R,ee,le)=>{if(h.length===0&&R.length===0)return;si(i,s,h,R,o,c);const ve=ai(h,o);let ge=null;const ye=ee.trim();if(ye)try{const Be=atob(ye.replace(/\s+/g,"")).length,_e=Number.isFinite(le)&&(le??0)>0?Number(le):16e3;ge=Be/2/_e*1e3}catch{ge=null}o=Math.max(ve,ge!=null?o+ge:o)},ie=h=>{if(b(h))return;const R=dr(h);R&&(j(),S());const ee=String(h.spoken_text??h.text??"").trim(),le=h.visemes??[],ve=h.words??[],ge=String(h.chunk??"").trim(),ye=K||R||!X;K&&(K=!1),ye&&(X=!0),ee&&(f=ee),v(h,R),ye?(V.resetForNewStream(c),i.length=0,s.length=0,o=0):R&&V.resumeBotStream(),J(le,ve,ge,h.sample_rate);const Be=n.length;V.playAudioChunk(n,r,ge,le,ve,R,{sampleRate:h.sample_rate,newStream:ye,audioOffsetMs:Number(h.audio_offset_ms??0),onNewStream:()=>{l.length=0}}).then(()=>{H(h,ee,Be,le),oe("audio_chunk",ee),Z()})},xe=h=>{if(b(h))return;(h.is_new_segment||Number(h.audio_offset_ms??-1)===0)&&S(),console.warn("[realtimeVoice] avatar_lipsync is deprecated — expect avatar_audio_chunk");const R=String(h.spoken_text??h.text??"").trim();R&&(f=R),v(h,!!h.is_new_segment),J(h.visemes??[],h.words??[],"",void 0);const ee=n.length;V.queueTimingsOnly(n,r,h.visemes??[],h.words??[],!!h.is_new_segment,{audioOffsetMs:Number(h.audio_offset_ms??0)}),H(h,R,ee,h.visemes??[]),oe("lipsync",R),M()},be=async h=>{const R=String(h.text??"").trim();if(!R)return;const ee=m.authToken?.trim();if(!ee){console.error("[realtimeVoice] avatar_speak requires authToken");return}P?.abort();const le=new AbortController;P=le;const ve=++W;S(),V.resetForNewStream(c),f=R;const ye=`${(m.apiBase??ot).replace(/\/$/,"")}/avatar_ttsWithPoses`,Be=m.voiceId?.trim()||Un,_e=m.speakingRate??Ft,ze=Math.max(on,Math.min(cn,_e)),Xe={user_query:R,ttsEngineId:c,speakingRate:ze,tenant:"aiTwin"};Fr.has(c)&&(Xe.voice_id=Be);try{const Fe=await fetch(ye,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${ee}`},body:JSON.stringify(Xe),signal:le.signal});if(!Fe.ok)throw new Error(`avatar_ttsWithPoses failed (${Fe.status})`);const Tt=Fe.body;if(!Tt)throw new Error("No response body");const Je=Tt.getReader(),Ue=new TextDecoder;let Ee="",Ze=!0;const et=async tt=>{if(ve!==W||le.signal.aborted||!I)return;const Se=Kr(tt);if(Se){if(Se.event==="audio"){const re=Se.data.chunk,Ne=Se.data.visemes??[],ft=Se.data.words??[],kt=Se.data.is_new_segment??!1,dt=Se.data.sample_rate,xt=Number(Se.data.audio_offset_ms??0);if(re){const qe=Ze;Ze=!1,await V.playAudioChunk(n,r,re,Ne,ft,kt,{sampleRate:dt,newStream:qe,audioOffsetMs:xt})}else(Ne.length>0||ft.length>0)&&V.queueTimingsOnly(n,r,Ne,ft,kt,{audioOffsetMs:xt,newStream:Ze})}else if(Se.event==="metadata"){const re=Se.data.mood;typeof re=="string"&&re.trim()&&(u=re.trim());const Ne=Se.data.sentence_emotions;Array.isArray(Ne)&&(a.length=0,a.push(...yn(Ne,u))),M()}else if(Se.event==="error")throw new Error(String(Se.data.message??"avatar_ttsWithPoses stream error"))}};for(;!(ve!==W||le.signal.aborted||!I);){const{done:tt,value:Se}=await Je.read();Se&&(Ee+=Ue.decode(Se,{stream:!0}));const re=Ee.split(`
|
|
4
4
|
|
|
5
5
|
`);Ee=re.pop()??"";for(const Ne of re)await et(Ne);if(tt){Ee.trim()&&await et(Ee.trim()),ve===W&&!le.signal.aborted&&I&&V.markStreamComplete(n);break}}}catch(Fe){if(le.signal.aborted||!I)return;console.error("[realtimeVoice] avatar_speak failed:",Fe),e("error"),M()}finally{P===le&&(P=null)}},Re=h=>{const R=Cs(h);if(!R){console.warn("[realtimeVoice] server message with no payload",h);return}const ee=String(R.type??"unknown");switch(ee!=="avatar_metadata"&&ee!=="avatar_audio_chunk"&&ee!=="avatar_lipsync"&&ee!=="avatar_interrupt"&&ee!=="error"&&console.debug("[realtimeVoice] server message",ee,R),ee){case"avatar_metadata":k(R);break;case"avatar_audio_chunk":ie(R);break;case"avatar_lipsync":xe(R);break;case"avatar_interrupt":L(R);break;case"error":{const le=String(R.message??"Voice session error");console.error("[realtimeVoice] server error:",le),e("error"),M();break}case"avatar_speak":be(R);break}},De=()=>p||(T=new Is,p=new Li.PipecatClient({transport:new Cn.WebSocketTransport({mediaManager:T,serializer:new Es,recorderSampleRate:Wn,playerSampleRate:Hn}),enableCam:!1,enableMic:!0,callbacks:{onBotReady:()=>{d=!0,e("idle"),M()},onDisconnected:()=>{d=!1,_()},onError:h=>{const R=typeof h=="object"&&h!==null&&"data"in h?String(h.data??h):String(h);console.error("[realtimeVoice]",R,h),e("error"),M()},onTransportStateChanged:h=>{h==="error"&&(e("error"),M())},onServerMessage:h=>{ne(h)},onUserStartedSpeaking:()=>{},onBotStartedSpeaking:()=>{S(),oe("bot_started")},onBotStoppedSpeaking:()=>{I&&Z()},onUserTranscript:h=>{String(h.text??""),m.onUserTranscript?.(h.text,!!h.final)},onBotOutput:h=>{m.onBotOutput?.(h.text)}}}),p);return{setDeveloperToken:h=>{},setTtsEngineId:h=>{c=h,V.setStreamTtsEngineId(h)},getTtsEngineId:()=>c,speak:async(h,R)=>{const ee=h.trim();if(ee){if(!p||!d)throw new Error("Realtime voice is not connected. Call connect() first.");O(),await p.sendText(ee)}},stop:()=>{P?.abort(),P=null,W+=1,S(),O(),e("idle"),M()},connect:async h=>{if(h&&(m={...m,...h}),!m.authToken?.trim())throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");e("loading");const R=De();await R.initDevices();const ee=fi(m);console.info("[realtimeVoice] connecting (avatar_audio_chunk RTVI):",ee);try{await R.connect({wsUrl:ee})}catch(le){d=!1,e("error"),M();const ve=le instanceof Error?le.message:"Realtime voice connect failed (WebSocket 101 only means upgrade OK — check authToken/origin/RTVI handshake)";throw new Error(`${ve} — ${ee}`)}d=!0,e("idle"),M()},disconnect:async()=>{if(j(),K=!1,X=!1,g.length=0,E=!1,P?.abort(),P=null,W+=1,S(),p)try{await p.disconnect()}catch(h){console.error("[realtimeVoice] disconnect failed:",h)}d=!1,_()},isConnected:()=>d,unlockAudio:async()=>{await V.primeAudioContext()},getVisemeQueue:()=>n,getWordQueue:()=>r,getFullVisemeQueue:()=>i,getFullWordQueue:()=>s,getSentenceEmotions:()=>a,getStreamMood:()=>u,getPlaybackElapsedMs:ae,isSpeaking:()=>V.isSpeaking(),isWarmingUp:()=>V.isWarmingUp(),releaseWarmupHold:()=>V.releaseWarmupHold(),setLipsyncWarmupHold:(h,R)=>{R&&(G=R),V.setLipsyncWarmupHoldEnabled(h)},getVoiceStream:()=>V.getVoiceStream(),setPlaybackSpeed:h=>{V.setPlaybackSpeed(h)},getPlaybackSpeed:()=>V.getPlaybackSpeed(),subscribe:h=>(x.add(h),()=>x.delete(h))}}function mi(e,t,n){return{width:e.crop?.source_width??t,height:e.crop?.source_height??n}}function hi(e,t){const n=e.crop;return{x:t?.source_x??n?.x??0,y:t?.source_y??n?.y??0,width:t?.source_w??n?.width??e.frame_width??0,height:t?.source_h??n?.height??e.frame_height??0}}function pi(e){return e?(e.w??0)>0&&(e.h??0)>0:!1}async function bt(e,t){const n=await fetch(e);if(!n.ok)throw new Error(`Failed to load encrypted asset: ${e}`);const r=await n.arrayBuffer();return rs(r,t)}async function wi(e,t){if(!t)throw new Error("Missing asset key for encrypted JSON");const n=await bt(e,t),r=new TextDecoder().decode(n);return JSON.parse(r)}async function Ds(e){const t=new Blob([e],{type:"image/webp"});return createImageBitmap(t)}async function Bs(e){const t=new Blob([e],{type:"image/webp"}),n=URL.createObjectURL(t);return new Promise((r,i)=>{const s=new Image;s.decoding="async",s.onload=()=>{URL.revokeObjectURL(n),r(s)},s.onerror=()=>{URL.revokeObjectURL(n),i(new Error("Failed to decode decrypted image"))},s.src=n})}async function Fs(e){const t=new Blob([e],{type:"image/png"});return createImageBitmap(t)}async function Ls(e){const t=new Blob([e],{type:"image/png"}),n=URL.createObjectURL(t);return new Promise((r,i)=>{const s=new Image;s.decoding="async",s.onload=()=>{URL.revokeObjectURL(n),r(s)},s.onerror=()=>{URL.revokeObjectURL(n),i(new Error("Failed to decode decrypted PNG"))},s.src=n})}function Os(e){const t=new Blob([e],{type:"video/mp4"});return URL.createObjectURL(t)}async function Ns(e){const t=ss();if(!Dt())return t;if(!e)throw new Error("Missing asset key for encrypted idle video");const n=await bt(t,e);return Os(n)}function gt(e,t){return`${e}/${t}_minus_sil.png`}const gi=5;function un(e){const t=e?.composite_threshold;if(typeof t=="number"&&t>0)return t;const n=e?.crop?.content_threshold;return typeof n=="number"&&n>0?n:gi}const Vs=["aa","E","I","O","U","PP","FF","DD","SS","TH","CH","RR","kk","nn"],In=["t01_0.25","t02_0.50","t03_0.75"],Us=In.length,mr=new Set(["aa__kk","aa__nn","aa__sil","CH__aa","CH__DD","CH__E","CH__FF","CH__I","CH__kk","CH__nn","CH__O","CH__PP","CH__RR","CH__sil","CH__SS","CH__TH","CH__U","DD__aa","DD__E","DD__FF","DD__I","DD__kk","DD__nn","DD__O","DD__PP","DD__RR","DD__sil","DD__SS","DD__TH","DD__U","E__aa","E__FF","E__I","E__kk","E__nn","E__O","E__PP","E__RR","E__sil","E__SS","E__TH","E__U","FF__aa","FF__I","FF__kk","FF__nn","FF__O","FF__PP","FF__RR","FF__sil","FF__SS","FF__TH","FF__U","I__aa","I__kk","I__nn","I__O","I__PP","I__RR","I__sil","I__SS","I__TH","I__U","kk__nn","kk__sil","nn__sil","O__aa","O__kk","O__nn","O__PP","O__RR","O__sil","O__SS","O__TH","O__U","PP__aa","PP__kk","PP__nn","PP__RR","PP__sil","PP__SS","PP__TH","PP__U","RR__aa","RR__kk","RR__nn","RR__sil","RR__SS","RR__TH","RR__U","SS__aa","SS__kk","SS__nn","SS__sil","SS__TH","SS__U","TH__aa","TH__kk","TH__nn","TH__sil","TH__U","U__aa","U__kk","U__nn","U__sil"]);function hr(e,t){return`${e}__${t}`}function pr(e,t){return`pairs/${e}/${t}_minus_sil.png`}function _t(e){return`${e}_minus_sil.png`}function $n(e,t){if(t.has(e))return e;const n=/^pairs\/([^/]+)\//.exec(e);if(!n)return null;const r=n[1].split("__")[1];if(!r||r==="sil")return null;const i=_t(r);return t.has(i)?i:null}function yi(e,t){return e.filter(n=>t.has(n))}function Ws(e,t,n){const r=zn(e,t);return yi(r,n).length>0}function zn(e,t){if(e===t)return[];const n=[],r=hr(e,t);if(mr.has(r)){for(const s of In)n.push(pr(r,s));return n}const i=hr(t,e);if(mr.has(i)){for(let s=Us-1;s>=0;s--)n.push(pr(i,In[s]));return n}return[]}function Hs(e){return e<16?[1]:e<24?[1]:e<35?[0,1,2]:[0,1,2]}function $s(e,t){return Hs(t).map(r=>e[r]).filter(r=>r!=null)}function zs(e,t,n,r){if(r<=0)return 0;const s=Math.max(1,n-t)/r,o=e-t,a=Math.floor(o/s);return Math.min(r-1,Math.max(0,a))}function _i(e,t,n){if(!n)return`${e}/${t}`;const r=t.split("/").pop()??t,i=r.includes(".")?r.slice(0,r.lastIndexOf(".")):r;return`${e}/${i}.bin`}const Ks=[.32,.24,.16,.08,0],js=Ks.map(e=>`eyes_${e.toFixed(2)}`),qs=["eyes_0.32","eyes_0.16","eyes_0.00"],Gs=["eyes_0.32","eyes_0.16","eyes_0.08","eyes_0.00"],Ve=2,Ys=-20,Xt=16,vi={fear:-20,anger:-20,sad:-10,neutral:0,love:0,happy:0,disgust:10};function Ct(e){return`brow_${e}`}function at(e){return vi[e]}function $e(e){return Math.max(Ys,Math.min(Xt,e))}function Qs(e,t){if(e===t)return[];const n=e<t,r=n?Ve:-Ve,i=[];let s=e;if(s%Ve!==0){const o=n?Math.ceil(s/Ve)*Ve:Math.floor(s/Ve)*Ve,a=$e(o);if(n?a>=t:a<=t)return[t];s=a,i.push(s)}for(;;){const o=s+r;if(n?o>=t:o<=t){t!==s&&i.push(t);break}s=o,i.push(s)}return i}function wr(e){const t=$e(e);if(t%Ve===0)return[t];const n=Math.floor(t/Ve)*Ve,r=Math.min(Xt,n+Ve),i=$e(n);return t-i<=r-t?i===r?[i]:[i,r]:i===r?[r]:[r,i]}const Xs=["fear","anger","sad","neutral","love","happy","disgust"],Js={fear:"fear",afraid:"fear",anxiety:"fear",anger:"anger",angry:"anger",mad:"anger",sad:"sad",sadness:"sad",sorrow:"sad",neutral:"neutral",calm:"neutral",love:"love",loving:"love",affection:"love",happy:"happy",happiness:"happy",joy:"happy",joyful:"happy",excited:"happy",disgust:"disgust",disgusted:"disgust"};function je(e,t="neutral"){const n=String(e??"").trim().toLowerCase();return n?Js[n]??t:t}function Si(e){return e===0?null:gt(Ct(e),"eyes_standard")}function bi(e){return Si(at(e))}const Zs={[-20]:"anger",[-10]:"sad",0:"neutral",10:"disgust"};function Kn(e,t){if(!e)return null;if(t(e))return e;const n=e.match(/^brow_(-?\d+)\/(eyes_[^/]+)_minus_sil\.png$/);if(n){const i=Number(n[1]),s=n[2];for(const a of wr(i)){if(a===i)continue;const l=gt(Ct(a),s);if(t(l))return l}const o=Zs[i];if(o){const a=gt(o,s);if(t(a))return a}return null}const r=e.match(/^(fear|anger|sad|neutral|love|happy|disgust)\/(eyes_[^/]+)_minus_sil\.png$/);if(r){const i=r[1],s=r[2],o=at(i);for(const a of[o,...wr(o)]){const l=gt(Ct(a),s);if(t(l))return l}}return null}function gr(e,t){const n=Ct(at(e));return!t||t==="eyes_standard"?bi(e):e==="neutral"&&t==="eyes_standard"?null:gt(n,t)}function ea(e,t){const n=[];for(const r of t)n.push(gr(e,r));for(let r=t.length-2;r>=0;r--)n.push(gr(e,t[r]));return n.push(bi(e)),n}function ta(e){return gt(Ct(e),"eyes_standard")}function na(e,t){return Qs(e,t).map(ta)}const ra=6;function ia(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function sa(e){const t=$e(e);return t>=Xt?Xt:$e(t+ra)}function aa(e,t,n){const r=$e(n);return ia(t,e)?.emphasis?sa(r):r}function yr(e){if(!e)return 0;const t=e.match(/^brow_(-?\d+)\//);if(!t?.[1])return null;const n=Number.parseInt(t[1],10);return Number.isFinite(n)?$e(n):null}function wt(e,t){return e+Math.random()*(t-e)}function Kt(){return wt(14,32)}function oa(e={}){const{initialDelayMs:t=800+Math.random()*1200,intervalMinMs:n=2200,intervalMaxMs:r=5800,doubleBlinkChance:i=.18,doubleBlinkGapMinMs:s=120,doubleBlinkGapMaxMs:o=220,fullBlinkChance:a=.75,postEmotionChangeDelayMs:l=380,minDwellMs:f=550}=e;let u="neutral",c="neutral",d=0,p=0,T=0,m={kind:"idle",nextBlinkAt:performance.now()+t},A=null,C="__eyes_open__|neutral|brow:0";function x(P=u){C=`${A??"__eyes_open__"}|${P}|brow:${p}`}function g(P,W=u){A=P;const K=yr(P);K!=null?p=K:P==null&&(p=0),x(W)}function E(P){return Si(P)}function N(P,W,K){const X=$e(W),Y=$e(K);if(d=Y,X===Y){g(E(Y));return}const pe=[...na(X,Y),E(Y)];m={kind:"browBlend",paths:pe,index:0,holdUntil:P+Kt(),targetBrow:Y},g(pe[0]??null)}function I(P,W){m={kind:"idle",nextBlinkAt:P+wt(n,r)},c=W,g(E(d),W)}function te(P,W,K){const X=K?qs:Math.random()<a?js:Gs;m={kind:"playing",paths:ea(W,X),index:0,holdUntil:P+Kt(),after:"idle",blinkEmotion:W},c=W,g(m.paths[0]??null,W)}function de(P){if(m.kind==="browBlend"){for(;P>=m.holdUntil&&m.index<m.paths.length-1;)m.index+=1,m.holdUntil=P+Kt();const W=m.paths[m.index]??null;g(W),P>=m.holdUntil&&m.index>=m.paths.length-1&&(p=m.targetBrow,d=m.targetBrow,g(E(m.targetBrow)),m={kind:"idle",nextBlinkAt:P+l});return}if(m.kind==="idle"){g(E(d),c),P>=m.nextBlinkAt&&te(P,c,!1);return}if(m.kind==="doublePause"){g(E(d),m.blinkEmotion),P>=m.resumeAt&&te(P,m.blinkEmotion,!0);return}if(m.kind==="playing"){for(;P>=m.holdUntil&&m.index<m.paths.length-1;)m.index+=1,m.holdUntil=P+Kt();if(g(m.paths[m.index]??null,m.blinkEmotion),P>=m.holdUntil&&m.index>=m.paths.length-1){if(m.after==="resumeIdle"){I(P+l,u);return}if(m.after==="doublePause"){m={kind:"doublePause",resumeAt:P+wt(s,o),blinkEmotion:m.blinkEmotion},g(E(d),m.blinkEmotion);return}if(m.after==="idle"&&Math.random()<i){m={kind:"doublePause",resumeAt:P+wt(s,o),blinkEmotion:m.blinkEmotion},g(E(d),m.blinkEmotion);return}I(P,m.blinkEmotion)}}}return{advance:de,setTargetEmotion(P,W){const K=je(P,u);if(K===u)return;const X=performance.now();!W?.force&&f>0&&!(u==="neutral"&&K!=="neutral")&&X-T<f||(u=K,c=K,T=X)},setTargetEyebrow(P){const W=$e(P);if(W===d)return;const K=performance.now(),X=m.kind==="browBlend"?yr(A)??p:p;d=W,!(W===X&&m.kind!=="browBlend")&&(m.kind==="playing"||m.kind==="doublePause"||N(K,X,W))},snapToEmotion(P){const W=je(P),K=at(W);u=W,c=W,d=K,p=K,T=performance.now(),m={kind:"idle",nextBlinkAt:performance.now()+wt(4e3,7e3)},g(E(K),W)},getTargetEmotion:()=>u,getTargetEyebrow:()=>d,getExpressionPath:()=>A,getDrawKey:()=>C,reset(){u="neutral",c="neutral",d=0,p=0,T=0,I(performance.now()+wt(n,r),"neutral")}}}function ca(){const e=document.createElement("video");e.muted=!0,e.playsInline=!0,e.preload="auto",e.setAttribute("playsinline",""),e.loop=!0;let t=!1,n=null,r="";const i=()=>{n&&(URL.revokeObjectURL(n),n=null)};return{async load(s){t&&r===s||(i(),t=!1,r=s,s.startsWith("blob:")&&(n=s),await new Promise((o,a)=>{const l=()=>{const f=e.duration;if(!Number.isFinite(f)||f<=0){a(new Error(`Idle video has invalid duration: ${s}`));return}t=!0,e.currentTime=0,o()};e.addEventListener("loadeddata",l,{once:!0}),e.addEventListener("error",()=>a(new Error(`Failed to load idle video: ${s}`)),{once:!0}),e.src=s,e.load()}))},isReady(){return t},getVideo(){return e},setActive(s){t&&(s?e.paused&&e.play().catch(()=>{}):e.paused||e.pause())},restart(){t&&(e.currentTime=0)},dispose(){e.pause(),e.removeAttribute("src"),e.load(),i(),t=!1,r=""}}}const ua=50,Ti=400;function la(e,t,n){return e.filter(r=>{const i=r.queueIndex??-1;return i>=t&&i<=n})}function fa(e,t){const n=[];for(const r of e){const i=la(t,r.start_word,r.end_word);if(i.length===0)continue;const s=i[0],o=i[i.length-1];n.push({sentence:r,startMs:Math.max(0,s.wtime-ua),endMs:o.wtime+o.wduration+Ti})}return n.sort((r,i)=>r.startMs-i.startMs)}function da(e,t,n,r="neutral"){const i=je(r);if(t.length===0)return i;if(n.length===0||e<n[0].wtime)return je(t[0]?.emotion,i);const s=fa(t,n);let o=null;for(const f of s)e>=f.startMs&&e<=f.endMs&&(o=f);if(o)return je(o.sentence.emotion,i);const a=n[n.length-1],l=a.wtime+a.wduration;return e>l+Ti?je(t[t.length-1]?.emotion,i):i}function jn(e,t,n,r,i=gi){const s=new OffscreenCanvas(n,r),o=s.getContext("2d",{willReadFrequently:!0});o.drawImage(e,0,0,n,r);const a=o.getImageData(0,0,n,r),l=o.createImageData(n,r);l.data.set(a.data);const u=new OffscreenCanvas(n,r).getContext("2d",{willReadFrequently:!0});for(const c of t){u.clearRect(0,0,n,r),u.drawImage(c,0,0,n,r);const d=u.getImageData(0,0,n,r);for(let p=0;p<l.data.length;p+=4){const T=d.data[p],m=d.data[p+1],A=d.data[p+2];Math.max(T,m,A)>i&&(l.data[p]=T,l.data[p+1]=m,l.data[p+2]=A,l.data[p+3]=255)}}return o.putImageData(l,0,0),s}let it=null,vt=null;const Jt=new Map;function ma(){it=null,vt=null,Jt.clear()}Wr(ma);function ha(e){return e??vt??void 0}function qn(e){return new Promise((t,n)=>{const r=new Image;r.decoding="async",r.onload=()=>t(r),r.onerror=()=>n(new Error(`Failed to load image: ${e}`)),r.src=e})}async function pa(e){return Dt()?Ls(await bt(Qt(),e??"")):qn(Qt())}async function _r(e,t,n,r){const i=n?await wi(e,r):await fetch(e).then(f=>{if(!f.ok)throw new Error(`Failed to load ${e}`);return f.json()}),s=i.sheets?.[0];if(!s?.path)throw new Error(`${e} has no sheets[0].path`);const o=n?await Bs(await bt(_i(t,s.path,n),r??"")):await qn(`${t}/${s.path}`),a=new Map;for(const f of i.cells??[])f.path&&a.set(f.path,f);const l=new Map;for(const f of i.sheets??[])l.set(f.index,f);return{atlas:o,atlasMeta:i,cellByPath:a,sheetByIndex:l,diffBase:t}}async function An(e){const t=ha(e),n=Dt();if(n&&!t)throw new Error("Encrypted assets enabled but no key provided");if(it&&(!n||vt===(t??null))){const s=await it;return{...St(s),visemePaths:[...s.viseme.cellByPath.keys()]}}vt=t??null,Jt.clear();const r=Hr();it=(async()=>{const s=await pa(t),o=await _r($r(),r,n,t);let a=null;try{a=await _r(zr(),r,n,t)}catch{a=null}return{sil:s,viseme:o,expression:a}})();const i=await it;return{...St(i),visemePaths:[...i.viseme.cellByPath.keys()]}}async function Gn(e){if(await An(e),!it)throw new Error("Assets not loaded");return it}function St(e){return mi(e.viseme.atlasMeta,e.sil.naturalWidth,e.sil.naturalHeight)}function wa(e,t){const n=document.createElement("canvas");return n.width=t.w,n.height=t.h,n.getContext("2d",{willReadFrequently:!0}).drawImage(e.atlas,t.x,t.y,t.w,t.h,0,0,t.w,t.h),n}async function Zt(e,t,n){const r=`${t.diffBase}::${n}`,i=Jt.get(r);if(i)return i;const s=(async()=>{const o=t.cellByPath.get(n);if(!o)throw new Error(`No atlas cell for path: ${n}`);const a=hi(t.atlasMeta,o),l=St(e);if(pi(o)){const c=wa(t,o),d=document.createElement("canvas");d.width=l.width,d.height=l.height;const p=d.getContext("2d");return p.fillStyle="#000",p.fillRect(0,0,l.width,l.height),p.drawImage(c,a.x,a.y),d}const f=`${t.diffBase}/${n}`,u=await qn(f);if(u.naturalWidth===l.width&&u.naturalHeight===l.height)return u;throw new Error(`Diff PNG wrong size for ${n}`)})();return Jt.set(r,s),s}async function _n(e,t,n,r,i,s){if(i){const{width:p,height:T}=i;if(s&&!s()||t.readyState<HTMLMediaElement.HAVE_CURRENT_DATA)return;const m=document.createElement("canvas");m.width=p,m.height=T;const A=m.getContext("2d");if(A.clearRect(0,0,p,T),A.drawImage(t,0,0,p,T),s&&!s())return;(e.width!==p||e.height!==T)&&(e.width=p,e.height=T);const C=e.getContext("2d");C.clearRect(0,0,p,T),C.drawImage(m,0,0);return}const o=await Gn(),a=un(o.viseme.atlasMeta),{width:l,height:f}=St(o);(e.width!==l||e.height!==f)&&(e.width=l,e.height=f);const u=[],c=e.getContext("2d");if(c.clearRect(0,0,l,f),t.readyState<HTMLMediaElement.HAVE_CURRENT_DATA){const p=o.sil;c.drawImage(p,0,0,l,f);return}if(u.length===0){c.drawImage(t,0,0,l,f);return}const d=jn(t,u,l,f,a);c.drawImage(d,0,0)}async function Pn(e,t,n){const r=await Gn(vt??void 0),i=un(r.viseme.atlasMeta),{width:s,height:o}=St(r);(e.width!==s||e.height!==o)&&(e.width=s,e.height=o);const a=[];if(t.mouthPath){const u=new Set(r.viseme.cellByPath.keys()),c=$n(t.mouthPath,u)??(u.has(t.mouthPath)?t.mouthPath:null);c&&a.push(await Zt(r,r.viseme,c))}if(t.expressionPath&&r.expression){const u=Kn(t.expressionPath,c=>r.expression.cellByPath.has(c));u&&a.push(await Zt(r,r.expression,u))}const l=e.getContext("2d");if(l.clearRect(0,0,s,o),a.length===0){l.drawImage(r.sil,0,0,s,o);return}const f=jn(r.sil,a,s,o,i);l.drawImage(f,0,0)}async function ga(e,t){const n=await Gn(vt??void 0),r=un(n.viseme.atlasMeta),{width:i,height:s}=St(n),o=[];if(e.mouthPath){const u=new Set(n.viseme.cellByPath.keys()),c=$n(e.mouthPath,u)??(u.has(e.mouthPath)?e.mouthPath:null);c&&o.push(await Zt(n,n.viseme,c))}if(e.expressionPath&&n.expression){const u=Kn(e.expressionPath,c=>n.expression.cellByPath.has(c));u&&o.push(await Zt(n,n.expression,u))}const a=new OffscreenCanvas(i,s),l=a.getContext("2d");if(o.length===0)return l.drawImage(n.sil,0,0,i,s),a.transferToImageBitmap();const f=jn(n.sil,o,i,s,r);return l.drawImage(f,0,0),a.transferToImageBitmap()}async function ya(e){await Pn(e,{mouthPath:null,expressionPath:null})}const jt={fontFamily:"system-ui, -apple-system, sans-serif",fontSize:"clamp(0.875rem, 2.5vw, 1.125rem)",color:"rgba(255,255,255,0.85)"};function _a(e){return e.length===0?"":e.map(t=>t.word).join(" ")}function va(e){if(!e)return{enabled:!1,style:jt};const t=e===!0?{}:e;return{enabled:!0,style:{fontFamily:t.fontFamily??jt.fontFamily,fontSize:t.fontSize??jt.fontSize,color:t.color??jt.color}}}const Sa={position:"absolute",bottom:0,left:0,right:0,display:"flex",justifyContent:"center",padding:"0.75rem 1rem",pointerEvents:"none"},ba={display:"inline-block",maxWidth:"100%",padding:"0.45rem 0.85rem",borderRadius:"0.75rem",background:"rgba(0,0,0,0.55)",lineHeight:1.4,textAlign:"center"},Ta={display:"-webkit-box",WebkitLineClamp:2,WebkitBoxOrient:"vertical",overflow:"hidden",maxWidth:"100%",margin:0};function ka({text:e,style:t}){return z.jsx("div",{style:Sa,"aria-live":"polite","aria-atomic":"true",children:z.jsx("div",{style:{...ba,fontFamily:t.fontFamily,fontSize:t.fontSize,color:t.color},children:z.jsx("p",{style:Ta,children:e})})})}let ki=null;function vr(e){ki=e}function Qe(){return ki}function xa(e){return`${e.from}→${e.to}@${e.fromVtime}`}function Sr(){return{rafTicksSpeaking:0,rafTicksDeduped:0,drawsRequested:0,drawsDisplayed:0,queueSuperseded:0,workerStale:0,rifeFramesOmittedByGap:0,transitionFramesSampled:0,transitionFramesDisplayed:0,transitionFramesMissed:0,bufferTicks:0,bufferHits:0,bufferUnderruns:0,bufferPrefetched:0}}function Ea(){let e=Sr();const t=new Set,n=new Set,r=new Set;let i="",s=-1,o="";const a=()=>{e=Sr(),t.clear(),n.clear(),r.clear(),i="",s=-1,o=""},l=u=>{t.add(u)},f=u=>{u&&r.add(u)};return{reset:a,onRafTickSpeaking(){e.rafTicksSpeaking+=1},onDrawDeduped(){e.rafTicksDeduped+=1},onDrawRequested(){e.drawsRequested+=1},onQueueSuperseded(){e.queueSuperseded+=1},onWorkerStale(){e.workerStale+=1},onTransitionSample(u,c){const d=xa(u);e.transitionFramesSampled+=1,d!==i&&(i=d,s=-1,o!==d&&(o=d,e.rifeFramesOmittedByGap+=u.omittedByGap)),c&&n.add(c),s>=0&&u.frameIdx>s+1&&(e.transitionFramesMissed+=u.frameIdx-s-1),s=u.frameIdx},onBufferTick(){e.bufferTicks+=1},onBufferHit(){e.bufferHits+=1},onBufferUnderrun(){e.bufferUnderruns+=1},onBufferPrefetched(){e.bufferPrefetched+=1},onBufferReset(){},onDrawDisplayed(u,c){if(e.drawsDisplayed+=1,u.transitionMeta){e.transitionFramesDisplayed+=1;const p=u.transitionMeta;l(p.fromVtime),l(p.toVtime),f(u.mouthPath);return}if(u.label.includes("→")){e.transitionFramesDisplayed+=1;return}const d=u.label.trim();if(!(!d||d==="sil")){for(const p of c)if(p.viseme===d&&u.elapsedMs>=p.vtime&&u.elapsedMs<p.vtime+p.vduration){l(p.vtime);break}}},snapshot(u,c){let d=0,p=0;for(const x of u)if(x.viseme!=="sil"){if(x.vtime>=c)break;d+=1,t.has(x.vtime)&&(p+=1)}const T=d>0?Math.round(p/d*100):100,m=e.queueSuperseded+e.workerStale+Math.max(0,e.drawsRequested-e.drawsDisplayed),A=d-p,C=e.bufferTicks>0?Math.round(e.bufferHits/e.bufferTicks*100):100;return{...e,bufferHitRate:C,renderLoss:m,visemeSlotsTotal:d,visemeSlotsCovered:p,visemeCoveragePct:T,transitionFramesUniqueSampled:n.size,transitionFramesUniquePainted:r.size,drawsDeduped:e.rafTicksDeduped,renderSkips:e.rafTicksDeduped+m,visemesShown:p,visemesSkipped:A,rifeTransitionFramesSkipped:e.transitionFramesMissed}}}}const xi=Dn,Ia=200,br=300,Aa=100,Pa=600,Ei="__sil__";function Tr(e){return e.lastDrawnKey?e.lastDrawnKey.split("|")[0]===Ei:!1}function Yn(){return{lastDrawnKey:""}}function Ce(e){e.lastDrawnKey="",e.transitionCache=void 0}function Ma(e,t){if(t?.transitionCache&&t.transitionCache.queueLength===e.length)return t.transitionCache.transitions;const n=ri(e);return t&&(t.transitionCache={queueLength:e.length,transitions:n}),n}function Ca(e,t){let n="sil";for(const r of e)r.viseme!=="sil"&&(t!=null&&r.vtime>t||(n=r.viseme));return n}function Ra(e,t,n,r=120){if(n!=="sil")return n;const i=li(e,t,r);return i!=="sil"?i:Ca(e,t)}function Da(e,t){return e==="sil"?!1:t?Ws(e,"sil",t):zn(e,"sil").length>0}function Ba(){return xi+Pa}function en(e){return e??Ei}function Qn(e,t,n,r,i=Aa){if(e.length===0)return{diffPath:null,label:"sil"};if(t>=rt(e))return{diffPath:null,label:"sil"};const s=Ma(e,r),o=vs(s,t);if(o){const f=ii(o.fromVtime,o.toVtime,o.from,o.to);if(t>=f.transStart&&t<f.transEnd){let u=zn(o.from,o.to);n&&(u=yi(u,n));const c=$s(u,f.effectiveGapMs);if(c.length>0){const d=zs(t,f.transStart,f.transEnd,c.length);return{diffPath:c[d]??c[c.length-1],label:`${o.from}→${o.to}`,transitionMeta:{from:o.from,to:o.to,fromVtime:o.fromVtime,toVtime:o.toVtime,frameIdx:d,frameCount:c.length,fullFrameCount:u.length,omittedByGap:Math.max(0,u.length-c.length)}}}return{diffPath:_t(o.to),label:o.to}}}const a=sn(e);let l=Zr(a,t);return l==="sil"&&(l=li(e,t,i)),l==="sil"?{diffPath:null,label:"sil"}:{diffPath:_t(l),label:l}}function Rt(e,t){const n=t??en(e.expressionPath);return`${en(e.mouthPath)}|${n}`}function kr(e,t,n,r,i,s,o,a,l,f){const u=Qn(t,n,l,s),c={mouthPath:u.diffPath,expressionPath:r},d=Rt(c,i),p=Qe();if(u.transitionMeta&&p?.onTransitionSample(u.transitionMeta,u.diffPath),d===s.lastDrawnKey){p?.onDrawDeduped();return}p?.onDrawRequested(),a(u),s.lastDrawnKey=d,o(e,c,d,T=>{T===d&&(s.lastDrawnKey=d)})}function vn(e,t,n,r,i,s,o=null,a=null){const l={mouthPath:a,expressionPath:t},f=o!=null?`|idle@${Math.round(o*1e3)}`:"",u=Rt(l,n)+f;!(o!=null)&&u===r.lastDrawnKey||(s(o!=null?t?"idle · blink":"idle":t?"blink":"sil"),i(e,l,u,d=>{d===u&&(r.lastDrawnKey=u)}))}function Ii(e,t,n,r,i=Pt,s){if(n<=t||e.length===0)return[];const o=s??Yn(),a=[];let l="";for(let f=t;f<n;f+=i){const u=Qn(e,f,r,o),c=en(u.diffPath);c!==l&&(l=c,a.push({displayAtMs:f,mouthPath:u.diffPath,mouthKey:c,label:u.label}))}return a}const Fa=3;function Xn(e){let t=0;const n=[];let r=!1,i=!1;const s=async()=>{if(!r){r=!0;try{for(;n.length>0;){const a=n.shift(),l=++t;await e(a.diffs,l,a)&&l===t&&a.onDisplayed?.(a.drawKey)}}finally{r=!1,n.length>0&&s()}}},o=()=>{i||(i=!0,queueMicrotask(()=>{i=!1,s()}))};return{enqueue(a,l,f){const u=n.findIndex(c=>c.drawKey===l);u>=0?n[u]={diffs:a,drawKey:l,onDisplayed:f}:(n.length>=Fa&&(n.shift(),Qe()?.onQueueSuperseded()),n.push({diffs:a,drawKey:l,onDisplayed:f})),o()},reset(){t+=1,n.length=0},isLatestSeq(a){return a===t},getLatestSeq(){return t}}}function La(e,t=2){const n=[],r=new Set;let i=0,s=!1,o=0;const a=(f,u)=>`${f}|${u}`,l=async()=>{if(!s){s=!0;try{for(;i<t&&n.length>0;){const f=n.shift(),u=a(f.displayAtMs,f.drawKey);if(r.has(u))continue;r.add(u),i+=1;const c=o;try{await e(f)}finally{if(r.delete(u),i-=1,c!==o)return}}}finally{s=!1,n.length>0&&i<t&&l()}}};return{enqueue(f){const u=a(f.displayAtMs,f.drawKey);r.has(u)||n.some(c=>a(c.displayAtMs,c.drawKey)===u)||(n.push(f),l())},isQueuedOrInflight(f,u){const c=a(f,u);return r.has(c)?!0:n.some(d=>a(d.displayAtMs,d.drawKey)===c)},reset(){o+=1,n.length=0,r.clear(),i=0},pendingCount(){return n.length+i}}}function Oa(){const e=new Map,t=n=>{n.bitmap.close()};return{has(n,r){return e.has(r)},markRendered(n,r,i,s,o){const a=e.get(r);a&&t(a),e.set(r,{displayAtMs:n,mouthKey:r,label:i,mouthPath:s,bitmap:o})},findForMouthKey(n){return e.get(n)??null},removeThrough(n){},pruneBefore(n){},reset(){for(const n of e.values())t(n);e.clear()},size(){return e.size}}}function Na(e,t,n,r,i,s){const o=Ii(t,n,r,i,Pt,s);if(o.length===0)return{ready:0,total:0};let a=0;for(const l of o)e.has(l.displayAtMs,l.mouthKey)&&(a+=1);return{ready:a,total:o.length}}function Va(e){const t=Oa(),n=Yn(),r=new Set;let i="",s=null;const o=La(async c=>{try{if(!e.supportsBufferedPlayback)return;const d=await e.compositePrefetchFrame(c.diffs,c.mouthKey);if(!d)return;t.markRendered(c.displayAtMs,c.mouthKey,c.label,c.mouthPath,d),Qe()?.onBufferPrefetched()}finally{r.delete(c.mouthKey)}},ji),a=c=>{for(const d of c)t.has(d.displayAtMs,d.mouthKey)||r.has(d.mouthKey)||o.isQueuedOrInflight(0,d.mouthKey)||(r.add(d.mouthKey),o.enqueue({...d,displayAtMs:0,drawKey:d.mouthKey,diffs:{mouthPath:d.mouthPath,expressionPath:null}}))},l=(c,d,p,T=gn)=>{if(!e.supportsBufferedPlayback)return;const m=Math.max(0,d-Pt*4),A=Ii(c,m,d+T,p,Pt,n);a(A)},f=(c,d)=>{i=c,s=d},u=(c,d)=>!(i!==c||s!==d);return{reset(){t.reset(),o.reset(),r.clear(),Ce(n),i="",s=null,Qe()?.onBufferReset()},markExternallyDisplayed(c,d){f(c.split("|")[0]??c,d)},warmupTick(c){e.supportsBufferedPlayback&&l(c.queue,0,c.availablePaths,Math.max(ar,gn))},isWarmupReady(c,d=ar){if(!e.supportsBufferedPlayback)return!0;const{ready:p,total:T}=Na(t,c.queue,0,d,c.availablePaths,n);return T===0?!1:p/T>=Ki},getBufferedFrameCount:()=>t.size(),tick(c){const d=Qe();d?.onBufferTick(),l(c.queue,c.elapsedMs,c.availablePaths),t.pruneBefore(c.elapsedMs-gn);const p=Qn(c.queue,c.elapsedMs,c.availablePaths,n),T=en(p.diffPath),m=Rt({mouthPath:p.diffPath,expressionPath:c.expressionPath},c.expressionDrawKey),A=()=>(f(T,c.expressionPath),d?.onBufferHit(),{usedBuffer:!0,underrun:!1,held:!0});if(u(T,c.expressionPath)||c.playbackState.lastDrawnKey===m)return A();const C=t.findForMouthKey(T,c.elapsedMs,Pt);return C&&c.expressionPath!=null?(d?.onBufferHit(),{usedBuffer:!1,underrun:!1,held:!1}):C?(e.blitBitmap(c.canvas,C.bitmap),f(C.mouthKey,c.expressionPath),c.playbackState.lastDrawnKey=m,c.onStatus({diffPath:C.mouthPath,label:C.label}),c.onDisplayed(m,C.mouthPath,C.label),d?.onBufferHit(),{usedBuffer:!0,underrun:!1,held:!1}):(d?.onBufferUnderrun(),{usedBuffer:!1,underrun:!0,held:!1})}}}const Ua=`
|
|
6
6
|
in vec2 aPosition;
|
package/dist/aitwin.js
CHANGED
|
@@ -5,7 +5,7 @@ import { WebSocketTransport as Ei, DailyMediaManager as Ii, ProtobufFrameSeriali
|
|
|
5
5
|
import { Filter as Pi, GlProgram as Mi, Application as Ci, RendererType as Ri, Sprite as Rt, Texture as Me, VideoSource as Di, Rectangle as Bi } from "pixi.js";
|
|
6
6
|
const Fi = "https://aitwin.bubu.social", Rr = {
|
|
7
7
|
publicBase: Fi
|
|
8
|
-
}, Li = "0.5.
|
|
8
|
+
}, Li = "0.5.7", Oi = {
|
|
9
9
|
version: Li
|
|
10
10
|
}, Ni = {}, ut = "https://ai.aitwin.me", Pn = `${Rr.publicBase}/custom-faces`, Ui = `${Rr.publicBase}/aitwin-workers/v${Oi.version}/visemeDiffPreview.worker.js`;
|
|
11
11
|
function Vi(e) {
|
|
@@ -1190,10 +1190,12 @@ function Os(e) {
|
|
|
1190
1190
|
const s = e.personaId?.trim();
|
|
1191
1191
|
s && r.searchParams.set("personaId", s);
|
|
1192
1192
|
const o = e.speakingRate ?? an;
|
|
1193
|
-
|
|
1193
|
+
r.searchParams.set(
|
|
1194
1194
|
"speaking_rate",
|
|
1195
1195
|
String(Math.max(Ln, Math.min(On, o)))
|
|
1196
|
-
), e.ttsStream === !1 ? r.searchParams.set("tts_stream", "false") : r.searchParams.set("tts_stream", "true"), r.
|
|
1196
|
+
), e.ttsStream === !1 ? r.searchParams.set("tts_stream", "false") : r.searchParams.set("tts_stream", "true"), e.speakOnlyWhenSpokenTo && r.searchParams.set("speakOnlyWhenSpokenTo", "true");
|
|
1197
|
+
const a = e.avatarName?.trim();
|
|
1198
|
+
return a && r.searchParams.set("avatarName", a), r.toString();
|
|
1197
1199
|
}
|
|
1198
1200
|
function wn(e, t) {
|
|
1199
1201
|
if (!Array.isArray(e)) return [];
|
package/dist/index.d.ts
CHANGED
|
@@ -546,6 +546,13 @@ export declare type VoiceConnectOptions = {
|
|
|
546
546
|
speakingRate?: number;
|
|
547
547
|
/** Default true — server streams TTS + avatar messages. */
|
|
548
548
|
ttsStream?: boolean;
|
|
549
|
+
/**
|
|
550
|
+
* Recall meetings only: listen and store transcript until someone says `avatarName`,
|
|
551
|
+
* then reply once. Do not set for widget / normal twin Start Talk.
|
|
552
|
+
*/
|
|
553
|
+
speakOnlyWhenSpokenTo?: boolean;
|
|
554
|
+
/** Wake name used with `speakOnlyWhenSpokenTo` (e.g. twin.agentName). */
|
|
555
|
+
avatarName?: string;
|
|
549
556
|
/** HTTP API base; default `https://ai.aitwin.me`. */
|
|
550
557
|
apiBase?: string;
|
|
551
558
|
/** Full WebSocket URL override. Query params must be included. */
|