@streamoji/aitwin 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aitwin.cjs +4 -4
- package/dist/aitwin.js +2 -2
- package/package.json +1 -1
package/dist/aitwin.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const jt=require("react/jsx-runtime"),b=require("react"),zn=require("@pipecat-ai/client-js"),ur=require("@pipecat-ai/websocket-transport"),ne=require("pixi.js");var Yt=typeof document<"u"?document.currentScript:null;const jn="https://aitwin.bubu.social",an={publicBase:jn},Yn="0.4.3",Xn={version:Yn},Jn={},St="https://ai.aitwin.me",lr=`${an.publicBase}/custom-faces`,Qn=`${an.publicBase}/aitwin-workers/v${Xn.version}/visemeDiffPreview.worker.js`;function Zn(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:Yt&&Yt.tagName.toUpperCase()==="SCRIPT"&&Yt.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?Jn?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():Qn}const es="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",Xt=3e3,yt=16,Fr=3e3,ts=4e3,rs=.85,ns=8,fr=400,dr="eng_a7f2b9c1",Vt="eng_d4e8f3a2",Ge="eng_c9b1e6d4",qe=Ge,on=new Set([Vt,Ge]),cn={[dr]:"oculus_direct",[Ge]:"oculus_direct",[Vt]:"inworld_word"},Lr={google:dr,inworld:Vt,cartesia:Ge};function ss(e){return cn[e]}function is(e){return e===Ge}function as(e){if(e.ttsEngineId&&e.ttsEngineId in cn)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in Lr?Lr[t]:qe}class Bt extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function un(e,t=es){const r=new URL(t);r.searchParams.set("id",e);const n=await fetch(r.toString());if(n.status===404)throw new Bt;if(!n.ok)throw new Error(`getAiTwin failed (${n.status})`);const s=await n.json();if(!s.success)throw new Bt(s.error??"AI twin not found");const{tts:i,token:o,...a}=s.data,u=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:as(s.data),authToken:u}}const os="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",cs="client_6TNvp3SCs4Og0a1ijm9TommXLql1",us="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",ls="6TNvp3SCs4Og0a1ijm9TommXLql1",fs="Swaraj Mali";async function Ct(){const e=await fetch(os,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":cs,"Client-Secret":us},body:JSON.stringify({userId:ls,userName:fs})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),r=t.authToken??t.token??"";if(!r.trim())throw new Error("getAuthToken returned no token");return r}async function ds(e,t=St){const r=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!r.ok)throw new Error(`/api/session-value failed (${r.status})`);const n=await r.json();if(!n.value)throw new Error("Session value response missing value");return n}function ms(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const r=new Uint8Array(t.length/2);for(let n=0;n<t.length;n+=2)r[n/2]=Number.parseInt(t.slice(n,n+2),16);return r}function Pt(e){return Uint8Array.from(e)}function hs(e,t){const r=new Uint8Array(e.length+t.length);return r.set(e,0),r.set(t,e.length),r}async function ps(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const r=new Uint8Array(e),n=Pt(r.subarray(0,12)),s=Pt(r.subarray(12,28)),i=Pt(r.subarray(28)),o=Pt(hs(i,s)),a=await crypto.subtle.importKey("raw",ms(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:n,tagLength:128},a,o)}let er=null;const tr=new Set;function xt(e){return e.includes("/custom-faces/")}function Ft(e){const t=e.binBase.trim().replace(/\/$/,""),r=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||xt(t)||xt(r)))return{twinBase:r,binBase:t,encrypted:!1};const s=xt(t)?t:xt(r)?r:t;return{twinBase:s,binBase:s,encrypted:!0}}function ze(e){return Ft(e).encrypted}function rr(e,t=lr){const r=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:r,binBase:r,encrypted:!0}}function ln(e,t){return(e??t)?.trim()||void 0}function fn(e){er=Ft(e);for(const t of tr)t()}function ws(e,t){const r=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!r)return Ft(e);const n=s=>{try{return new URL(s,r).href.replace(/\/$/,"")}catch{return s}};return Ft({twinBase:n(e.twinBase),binBase:n(e.binBase),encrypted:e.encrypted})}function je(){if(!er)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return er}function dn(e){return tr.add(e),()=>tr.delete(e)}function Lt(){const e=je();return ze(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function gs(){const e=je();return ze(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function mn(){const e=je();return ze(e)?e.binBase:e.twinBase}function vt(){return ze(je())}function hn(){const e=je();return ze(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function pn(){const e=je();return ze(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function wn(e){const t=e.split(/\r?\n/);let r="",n="";for(const i of t)i.startsWith("event:")?r=i.slice(6).trim():i.startsWith("data:")&&(n=i.slice(5).trim());if(!r)return null;let s={};if(n)try{s=JSON.parse(n)}catch{s={raw:n}}return{event:r,data:s}}const gn=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],ys=new Map(gn.map(e=>[e.toLowerCase(),e]));function yn(e){const t=(e??"").trim();return t?ys.get(t.toLowerCase())??"sil":"sil"}const _s={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function Ss(e){if(!e)return["sil"];const t=e.toLowerCase();return _s[t]??["sil"]}function vs(e,t){const r=e.word.length;if(r<=0)return 0;const n=e.wduration;if(n<=0)return 0;const s=Math.max(0,Math.min(1,(t-e.wtime)/n));return Math.min(r-1,Math.floor(s*r))}function rt(e,t){const r=Math.max(0,Math.min(e.length-1,t));return e[r]??""}function mr(e,t){const r=e.toLowerCase().replace(/[^a-z']/g,"");return!r||rt(r,t)!=="w"||t>0&&rt(r,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(r)}function bs(e,t){const r=e.toLowerCase(),n=rt(r,t),s=r.slice(Math.max(0,t-1),t+2);return n==="i"||n==="y"?"I":s.includes("ee")||n==="e"||r.includes("ea")?"E":"aa"}function Ts(e,t){const r=e.toLowerCase(),n=r.slice(Math.max(0,t-1),Math.min(r.length,t+3));if(/oo|ou|uw/.test(n))return"U";if(mr(r,t))return"FF";const s=rt(r,t);return s==="u"||s==="w"?"U":"O"}function ks(e,t){const r=e.toLowerCase();if(mr(r,t))return"FF";const n=r.slice(Math.max(0,t-2),t+1);if(/qu/.test(n))return"U";const s=r.slice(Math.max(0,t-1),Math.min(r.length,t+3));return/wo|wh/.test(s)||rt(r,t)==="o"?"O":"U"}const Rt={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function Es(e,t){const r=e.toLowerCase(),n=rt(r,t);if(Rt[n])return Rt[n];for(let s=0;s<r.length;s++){const i=r[s];if(Rt[i])return Rt[i]}return"DD"}function As(e,t,r){const n=Ss(e);if(n.length===1)return n[0];const s=t.trim();if(!s)return n[0];const i=(e??"").toLowerCase(),o=Math.max(0,r);switch(i){case"aei":return bs(s,o);case"o":return Ts(s,o);case"qw":return ks(s,o);case"cdgknstxyz":return Es(s,o);default:return n[0]}}const Is=32,Or=48;function _n(e){let t=0;for(const r of e)t=Math.max(t,r.vtime+r.vduration);return t}function Sn(e){let t=null,r=-1;for(const n of e){if(n.viseme==="sil")continue;const s=n.vtime+n.vduration;s>r&&(t=n,r=s)}return t}function Ps(e){return vn(e).find(r=>r.viseme!=="sil")??null}function He(e){const t=_n(e);return Sn(e)?t+fr:t}function xs(e){const t=[],r=Ps(e);r&&r.vtime>0&&t.push({from:"sil",to:r.viseme,fromVtime:0,toVtime:r.vtime});const n=Sn(e),s=_n(e);return n&&t.push({from:n.viseme,to:"sil",fromVtime:s,toVtime:s+fr}),t}function vn(e){const t=new Map;for(const r of e)t.has(r.vtime)||t.set(r.vtime,r);return[...t.values()].sort((r,n)=>r.vtime-n.vtime)}function Rs(e){const t=vn(e),r=[];for(let n=0;n<t.length-1;n++){const s=t[n].viseme,i=t[n+1].viseme,o=t[n].vtime,a=t[n+1].vtime;s===i||a<=o||r.push({from:s,to:i,fromVtime:o,toVtime:a})}return r}function Ms(e){return e.length===0?[]:[...xs(e),...Rs(e)]}function Ds(e,t){let r=null;for(const n of e){const{transStart:s,transEnd:i}=bn(n.fromVtime,n.toVtime,n.from,n.to);t>=s&&t<i&&(r=n)}return r}function bn(e,t,r,n){const s=Math.max(0,t-e);if((r==="PP"||n==="PP")&&s>Or){const l=Or,f=t,c=f-l;return{gapMs:s,effectiveGapMs:l,holdEnd:c,transStart:c,transEnd:f}}const o=s>0?Math.max(s,Is):0,a=e,u=e+o;return{gapMs:s,effectiveGapMs:o,holdEnd:e,transStart:a,transEnd:u}}const hr=16e3;function Tn(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:hr}function Cs(e){let t=e.trim();const r=t.indexOf(",");t.startsWith("data:")&&r>=0&&(t=t.slice(r+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const n=t.length%4;return n!==0&&(t+="=".repeat(4-n)),t}function Bs(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function Nr(e,t,r=hr){const n=t.byteOffset%2===0?t:t.slice(),s=new Int16Array(n.buffer,n.byteOffset,n.byteLength/2),i=new Float32Array(s.length);for(let a=0;a<s.length;a++)i[a]=s[a]>=32768?-(65536-s[a])/32768:s[a]/32767;const o=e.createBuffer(1,i.length,r);return o.copyToChannel(i,0),o}function Fs(e){const t=Cs(e),r=window.atob(t),n=new Uint8Array(r.length);for(let s=0;s<r.length;s++)n[s]=r.charCodeAt(s);return n}async function Ur(e,t,r=hr){const n=Fs(t),s=Tn(r);if(Bs(n)){const i=n.buffer.slice(n.byteOffset,n.byteOffset+n.byteLength);return e.decodeAudioData(i)}if(n.length>=2&&n.length%2===0)return Nr(e,n,s);try{const i=n.buffer.slice(n.byteOffset,n.byteOffset+n.byteLength);return await e.decodeAudioData(i)}catch{return Nr(e,n,s)}}function Ls(e,t){for(const r of e)if(t>=r.wtime&&t<r.wtime+r.wduration)return r;return null}function kn(e,t){const r=[],n=[];let s=null,i=!1,o=!1,a=!1,u=!1,l=0,f=0,c=0,d=null,g=1,v=t,m=!1,R=null,_=0,D=!1,w=!1,T=0,O=null,B=null,G=null;const F=()=>({visemes:[],words:[],segmentOffsetMs:0}),k=()=>{w=!1,T=0},P=()=>{d!=null&&(clearTimeout(d),d=null)},C=()=>{for(const h of n){try{h.onended=null,h.stop(0)}catch{}try{h.disconnect()}catch{}}n.length=0},V=()=>{for(const h of n)h.playbackRate.value=g},N=()=>{if(!(m||!u||i||w||n.length>0||r.length>0)){if(a&&s&&f>0){const h=(s.currentTime-f)*1e3;if(h<_){P();const A=g>0?(_-h)/g:_-h;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,A));return}}o=!1,P(),e.onSpeakingDone(),e.onNotify()}},he=h=>{const A=He(h);if(A<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,l=performance.now(),e.onSpeakingStart(),e.onNotify(),P();const M=g>0?A/g:A;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},M)},X=(h,A,M,L)=>{A.length!==0&&(c=M,A.forEach(I=>{const Y=String(I.word??"").trim();if(!Y)return;const Q=Math.round((I.start??0)*1e3),Z=I.duration!=null?Math.round(I.duration*1e3):I.end!=null?Math.max(1,Math.round((I.end-(I.start??0))*1e3)):0;h.push({word:Y,wtime:c+Q,wduration:Z,queueIndex:h.length,gesture:I.gesture??null,emphasis:I.emphasis===!0})}))},z=(h,A,M,L,I,$)=>{M.length!==0&&(c=L,M.forEach(Y=>{const Q=String(Y.symbol??Y.viseme??"").trim();if(!Q)return;const Z=Math.round((Y.start??0)*1e3),Ie=Math.max(1,Math.round((Y.duration??0)*1e3)),be=c+Z,Be=Ls(A,be),at=Be?vs(Be,be):0;let Pe;ss($)==="oculus_direct"?(Pe=yn(Q),Pe==="U"&&Be&&mr(Be.word,at)&&(Pe="FF")):Pe=As(Q,Be?.word??"",at),h.push({viseme:Pe,weight:1,vtime:be,vduration:Ie})}),e.onNotify())},J=(h,A,M,L,I,$,Y)=>{X(A,L,I),z(h,A,M,I,$,Y)},se=(h,A,M)=>{G===h&&h.visemes===A||(A.length=0,M.length=0,h.visemes=A,h.words=M,h.segmentOffsetMs=0,G=h,c=0,_=0)},ee=async()=>{const h=window.AudioContext||window.webkitAudioContext;return s||(s=new h),s.state==="suspended"&&await s.resume(),R||(R=s.createMediaStreamDestination()),s},ge=(h,A)=>{if(!s)return 0;const M=s,L=M.currentTime,I=L+.02;if(A.isFirstAudio){f<=0&&(f=I);const Y=(I-L)*1e3;l=performance.now()+Y}A.onBeforeStart?.(I);const $=M.createBufferSource();return $.buffer=h,$.playbackRate.value=g,$.connect(M.destination),R&&$.connect(R),n.push($),m?(n.pop(),I):(A.isFirstAudio&&(o=!0,e.onSpeakingStart()),$.onended=()=>{const Y=n.indexOf($);Y>=0&&n.splice(Y,1),e.onNotify(),!m&&(O&&B?ce(O,B):N())},$.start(I),I)},ce=async(h,A)=>{if(m||i||n.length>0)return;if(r.length===0){N();return}i=!0;let M=!1;try{const L=await ee();if(m)return;const I=r[0];if(!I)return;const $=!o;if(D&&(e.shouldHoldForWarmup?.()??!1)&&(w||$&&I.newStream)||w){se(I.utterance,h,A);for(const Z of r){if(m)return;Z.audioBuffer||(Z.audioBuffer=await Ur(L,Z.audio,Z.sampleRate)),a=!0,w||(w=!0,T=0,Z.newStream&&Z.onNewStream?.()),Z.visemesQueued||(J(h,A,Z.visemes,Z.words,T,!0,v),Z.visemesQueued=!0,T+=Math.round(Z.audioBuffer.duration*1e3),_=Math.max(_,He(h)))}e.onNotify();return}const Q=r.shift();if(!Q||(Q.audioBuffer||(Q.audioBuffer=await Ur(L,Q.audio,Q.sampleRate)),m))return;a=!0,P(),se(Q.utterance,h,A),Q.visemesQueued=!1,Q.newStream&&Q.onNewStream?.(),ge(Q.audioBuffer,{isFirstAudio:!o,isFirstOfStream:!1,onBeforeStart:Z=>{f=Z,c=0,l=performance.now()+(Z-L.currentTime)*1e3,J(h,A,Q.visemes,Q.words,0,!0,v),Q.visemesQueued=!0,Q.utterance.segmentOffsetMs=0}}),M=n.length>0,_=He(h),e.onNotify()}catch(L){console.error("[avatarChunkPlayback] audio chunk failed:",L)}finally{if(i=!1,m)return;if(w){r.some(I=>!I.audioBuffer||!I.visemesQueued)&&ce(h,A);return}!M&&n.length===0&&r.length>0?ce(h,A):n.length===0&&r.length===0&&N()}};return{setStreamTtsEngineId:h=>{v=h},setPlaybackSpeed:h=>{g=Math.max(.1,Math.min(1,h)),V()},getPlaybackSpeed:()=>g,setLipsyncWarmupHoldEnabled:h=>{D=h,h||k()},isWarmingUp:()=>w,releaseWarmupHold:()=>{if(w){w=!1,T=0;for(const h of r)h.newStream=!1;O&&B&&ce(O,B),e.onNotify()}},resetForNewStream:h=>{m=!1,u=!1,v=h,_=0},resumeBotStream:()=>{m=!1,u=!1},clearAborted:()=>{m=!1},primeAudioContext:async()=>{const h=window.AudioContext||window.webkitAudioContext;s||(s=new h),s.state==="suspended"&&await s.resume(),s&&!R&&(R=s.createMediaStreamDestination())},stop:()=>{m=!0,P(),C(),r.length=0,i=!1,o=!1,a=!1,u=!1,c=0,f=0,l=0,_=0,O=null,B=null,G=null,k()},markStreamComplete:h=>{if(_=Math.max(_,He(h)),u=!0,!a&&h.length>0){he(h);return}N()},playAudioChunk:async(h,A,M,L,I,$,Y)=>{if(m)return;O=h,B=A;const Q=Tn(Y?.sampleRate),Z=Y?.newStream===!0,Ie=F();if(!M?.trim()){if(L.length===0&&I.length===0)return;se(Ie,h,A),Z&&Y?.onNewStream?.(),J(h,A,L,I,0,!0,v),_=He(h),!o&&_>0&&he(h),e.onNotify();return}r.push({audio:M,audioBuffer:null,utterance:Ie,visemes:L,words:I,isNewSegment:$,sampleRate:Q,newStream:Z,onNewStream:Y?.onNewStream,visemesQueued:!1}),await ce(h,A)},queueTimingsOnly:(h,A,M,L,I)=>{if(m)return;const $=F();se($,h,A),J(h,A,M,L,0,!0,v),$.segmentOffsetMs=0,_=He(h)},getPlaybackElapsedMs:()=>a&&s&&f>0?Math.max(0,(s.currentTime-f)*1e3):o?Math.max(0,(performance.now()-l)*g):0,isSpeaking:()=>o?!0:!a||!s||f<=0?!1:(s.currentTime-f)*1e3<_,getVoiceStream:()=>R?R.stream:null,getPlaybackStartPerf:()=>l,bargeIn:()=>{m=!0,u=!1,P(),C(),r.length=0,i=!1,o=!1,a=!1,c=0,f=0,l=0,_=0,O=null,B=null,G=null,k()},interrupt:()=>{m=!0,u=!1,P(),C(),r.length=0,i=!1,o=!1,a=!1,c=0,f=0,l=0,_=0,O=null,B=null,G=null,k()}}}function bt(e,t){let r=null;for(const n of e)t<n.vtime||t>=n.vtime+n.vduration||(!r||n.vtime>=r.vtime)&&(r=n);return r?r.viseme:"sil"}function En(e,t,r=120){const n=bt(e,t);if(n!=="sil")return n;let s=null;for(const i of e){if(i.vtime>t)break;const o=i.vtime+i.vduration;o<=t&&t-o<=r&&(s=i)}return s?.viseme??"sil"}function Os(e,t,r=400){const n=t-r;let s=0;for(const i of e)if(i.vtime+i.vduration<n)s+=1;else break;s>0&&e.splice(0,s)}function Ns(e,t){for(const r of e)if(t>=r.wtime&&t<r.wtime+r.wduration)return r;return null}function Us(e,t){const r=e.trim();if(!r)return null;try{const n=r.replace(/\s+/g,""),s=atob(n).length,i=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return s/2/i*1e3}catch{return null}}const Wt=.5,Ht=1.5,pr="Olivia",wr="f786b574-daa5-4673-aa0c-cbe3e8534c02",Tt=.85;function nr(e,t=St,r={}){const n=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,s=[],i=[],o=[],a=[];let u="",l="neutral",f="",c=qe;const d=new Set;let g=0,v=null;const m=()=>d.forEach(F=>F());let R=()=>!1;const _=kn({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:m,shouldHoldForWarmup:()=>R()},qe),D=()=>_.getPlaybackElapsedMs(),w=(F,k=u)=>{if(!r.onLipsyncDebug)return;const P=D(),C=s;r.onLipsyncDebug({event:F,spokenText:k,mood:l,playbackAnchorPerf:_.getPlaybackStartPerf(),playbackElapsedMs:P,queueLength:C.length,activeViseme:bt(C,P),segments:a.map(V=>({...V,visemes:[...V.visemes]}))})},T=(F,k)=>{r.onLipsyncDebug&&(v={event:F,spokenText:k},g===0&&(g=requestAnimationFrame(()=>{g=0;const P=v;v=null,P&&w(P.event,P.spokenText)})))},O=(F,k,P,C,V)=>{if(!r.onLipsyncDebug)return;const N=s.slice(k),he=P.filter(X=>String(X.symbol??X.viseme??"").trim());a.push({spokenText:F,audioOffsetMs:N[0]?.vtime??0,isNewSegment:C,chunkDurationMs:V,receivedAtMs:performance.now(),visemes:N.map((X,z)=>{const J=he[z];return{rawSymbol:String(J?.symbol??J?.viseme??X.viseme).trim(),viseme:X.viseme,startSec:Number(J?.start??0),durationSec:Number(J?.duration??0),vtimeMs:X.vtime,vdurationMs:X.vduration}})})},B=()=>{_.stop(),s.length=0,i.length=0,o.length=0,l="neutral",a.length=0,u="",e("idle"),T("reset"),m()};return{setDeveloperToken:F=>{f=F},setTtsEngineId:F=>{c=F},getTtsEngineId:()=>c,speak:async(F,k)=>{B(),e("loading"),u=F,a.length=0;const P=f.trim();if(!P)throw new Error("Developer token required for avatar_ttsWithPoses");const C=k?.ttsEngineId??c??qe;_.resetForNewStream(C),_.setStreamTtsEngineId(C);const V=k?.voiceId??pr,N=k?.speakingRate??Tt,he=Math.max(Wt,Math.min(Ht,N)),X={user_query:F,ttsEngineId:C,speakingRate:he,tenant:"aiTwin"};on.has(C)&&(X.voice_id=V);try{const z=await fetch(n,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${P}`},body:JSON.stringify(X)});if(!z.ok)throw new Error(`avatar_ttsWithPoses failed (${z.status})`);const J=z.body;if(!J)throw new Error("No response body");const se=J.getReader(),ee=new TextDecoder;let ge="",ce=!0;const fe=async S=>{const E=wn(S);if(E){if(E.event==="audio"){const h=E.data.chunk,A=E.data.visemes??[],M=E.data.words??[],L=E.data.is_new_segment??!1,I=E.data.sample_rate,$=s.length;if(h){const Y=ce;ce=!1,await _.playAudioChunk(s,i,h,A,M,L,{sampleRate:I,newStream:Y}),O(F,$,A,L,Us(h,I)),T("audio_chunk",F)}else(A.length>0||M.length>0)&&(_.queueTimingsOnly(s,i,A,M,L),O(F,$,A,L,null),T("audio_chunk",F))}else if(E.event==="metadata"){const h=E.data.mood;typeof h=="string"&&h.trim()&&(l=h.trim());const A=E.data.sentence_emotions;if(Array.isArray(A)){o.length=0;for(const M of A){if(!M||typeof M!="object")continue;const L=M;o.push({sentence_index:Number(L.sentence_index??0),text:String(L.text??""),sentiment:typeof L.sentiment=="string"?L.sentiment:void 0,emotion:String(L.emotion??l),start_word:Number(L.start_word??0),end_word:Number(L.end_word??0)})}}m(),T("metadata",F)}else if(E.event==="error")throw new Error(String(E.data.message??"avatar_ttsWithPoses stream error"))}};for(;;){const{done:S,value:E}=await se.read();E&&(ge+=ee.decode(E,{stream:!0}));const h=ge.split(`
|
|
1
|
+
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const jt=require("react/jsx-runtime"),v=require("react"),zn=require("@pipecat-ai/client-js"),ur=require("@pipecat-ai/websocket-transport"),ne=require("pixi.js");var Yt=typeof document<"u"?document.currentScript:null;const jn="https://aitwin.bubu.social",an={publicBase:jn},Yn="0.4.4",Xn={version:Yn},Jn={},St="https://ai.aitwin.me",lr=`${an.publicBase}/custom-faces`,Qn=`${an.publicBase}/aitwin-workers/v${Xn.version}/visemeDiffPreview.worker.js`;function Zn(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:Yt&&Yt.tagName.toUpperCase()==="SCRIPT"&&Yt.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?Jn?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():Qn}const es="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",Xt=3e3,yt=16,Fr=3e3,ts=4e3,rs=.85,ns=8,fr=400,dr="eng_a7f2b9c1",Vt="eng_d4e8f3a2",Ge="eng_c9b1e6d4",qe=Ge,on=new Set([Vt,Ge]),cn={[dr]:"oculus_direct",[Ge]:"oculus_direct",[Vt]:"inworld_word"},Lr={google:dr,inworld:Vt,cartesia:Ge};function ss(e){return cn[e]}function is(e){return e===Ge}function as(e){if(e.ttsEngineId&&e.ttsEngineId in cn)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in Lr?Lr[t]:qe}class Bt extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function un(e,t=es){const r=new URL(t);r.searchParams.set("id",e);const n=await fetch(r.toString());if(n.status===404)throw new Bt;if(!n.ok)throw new Error(`getAiTwin failed (${n.status})`);const s=await n.json();if(!s.success)throw new Bt(s.error??"AI twin not found");const{tts:i,token:o,...a}=s.data,u=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:as(s.data),authToken:u}}const os="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",cs="client_6TNvp3SCs4Og0a1ijm9TommXLql1",us="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",ls="6TNvp3SCs4Og0a1ijm9TommXLql1",fs="Swaraj Mali";async function Ct(){const e=await fetch(os,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":cs,"Client-Secret":us},body:JSON.stringify({userId:ls,userName:fs})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),r=t.authToken??t.token??"";if(!r.trim())throw new Error("getAuthToken returned no token");return r}async function ds(e,t=St){const r=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!r.ok)throw new Error(`/api/session-value failed (${r.status})`);const n=await r.json();if(!n.value)throw new Error("Session value response missing value");return n}function ms(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const r=new Uint8Array(t.length/2);for(let n=0;n<t.length;n+=2)r[n/2]=Number.parseInt(t.slice(n,n+2),16);return r}function Pt(e){return Uint8Array.from(e)}function hs(e,t){const r=new Uint8Array(e.length+t.length);return r.set(e,0),r.set(t,e.length),r}async function ps(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const r=new Uint8Array(e),n=Pt(r.subarray(0,12)),s=Pt(r.subarray(12,28)),i=Pt(r.subarray(28)),o=Pt(hs(i,s)),a=await crypto.subtle.importKey("raw",ms(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:n,tagLength:128},a,o)}let er=null;const tr=new Set;function xt(e){return e.includes("/custom-faces/")}function Ft(e){const t=e.binBase.trim().replace(/\/$/,""),r=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||xt(t)||xt(r)))return{twinBase:r,binBase:t,encrypted:!1};const s=xt(t)?t:xt(r)?r:t;return{twinBase:s,binBase:s,encrypted:!0}}function ze(e){return Ft(e).encrypted}function rr(e,t=lr){const r=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:r,binBase:r,encrypted:!0}}function ln(e,t){return(e??t)?.trim()||void 0}function fn(e){er=Ft(e);for(const t of tr)t()}function ws(e,t){const r=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!r)return Ft(e);const n=s=>{try{return new URL(s,r).href.replace(/\/$/,"")}catch{return s}};return Ft({twinBase:n(e.twinBase),binBase:n(e.binBase),encrypted:e.encrypted})}function je(){if(!er)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return er}function dn(e){return tr.add(e),()=>tr.delete(e)}function Lt(){const e=je();return ze(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function gs(){const e=je();return ze(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function mn(){const e=je();return ze(e)?e.binBase:e.twinBase}function vt(){return ze(je())}function hn(){const e=je();return ze(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function pn(){const e=je();return ze(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function wn(e){const t=e.split(/\r?\n/);let r="",n="";for(const i of t)i.startsWith("event:")?r=i.slice(6).trim():i.startsWith("data:")&&(n=i.slice(5).trim());if(!r)return null;let s={};if(n)try{s=JSON.parse(n)}catch{s={raw:n}}return{event:r,data:s}}const gn=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],ys=new Map(gn.map(e=>[e.toLowerCase(),e]));function yn(e){const t=(e??"").trim();return t?ys.get(t.toLowerCase())??"sil":"sil"}const _s={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function Ss(e){if(!e)return["sil"];const t=e.toLowerCase();return _s[t]??["sil"]}function vs(e,t){const r=e.word.length;if(r<=0)return 0;const n=e.wduration;if(n<=0)return 0;const s=Math.max(0,Math.min(1,(t-e.wtime)/n));return Math.min(r-1,Math.floor(s*r))}function rt(e,t){const r=Math.max(0,Math.min(e.length-1,t));return e[r]??""}function mr(e,t){const r=e.toLowerCase().replace(/[^a-z']/g,"");return!r||rt(r,t)!=="w"||t>0&&rt(r,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(r)}function bs(e,t){const r=e.toLowerCase(),n=rt(r,t),s=r.slice(Math.max(0,t-1),t+2);return n==="i"||n==="y"?"I":s.includes("ee")||n==="e"||r.includes("ea")?"E":"aa"}function Ts(e,t){const r=e.toLowerCase(),n=r.slice(Math.max(0,t-1),Math.min(r.length,t+3));if(/oo|ou|uw/.test(n))return"U";if(mr(r,t))return"FF";const s=rt(r,t);return s==="u"||s==="w"?"U":"O"}function ks(e,t){const r=e.toLowerCase();if(mr(r,t))return"FF";const n=r.slice(Math.max(0,t-2),t+1);if(/qu/.test(n))return"U";const s=r.slice(Math.max(0,t-1),Math.min(r.length,t+3));return/wo|wh/.test(s)||rt(r,t)==="o"?"O":"U"}const Rt={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function Es(e,t){const r=e.toLowerCase(),n=rt(r,t);if(Rt[n])return Rt[n];for(let s=0;s<r.length;s++){const i=r[s];if(Rt[i])return Rt[i]}return"DD"}function As(e,t,r){const n=Ss(e);if(n.length===1)return n[0];const s=t.trim();if(!s)return n[0];const i=(e??"").toLowerCase(),o=Math.max(0,r);switch(i){case"aei":return bs(s,o);case"o":return Ts(s,o);case"qw":return ks(s,o);case"cdgknstxyz":return Es(s,o);default:return n[0]}}const Is=32,Or=48;function _n(e){let t=0;for(const r of e)t=Math.max(t,r.vtime+r.vduration);return t}function Sn(e){let t=null,r=-1;for(const n of e){if(n.viseme==="sil")continue;const s=n.vtime+n.vduration;s>r&&(t=n,r=s)}return t}function Ps(e){return vn(e).find(r=>r.viseme!=="sil")??null}function He(e){const t=_n(e);return Sn(e)?t+fr:t}function xs(e){const t=[],r=Ps(e);r&&r.vtime>0&&t.push({from:"sil",to:r.viseme,fromVtime:0,toVtime:r.vtime});const n=Sn(e),s=_n(e);return n&&t.push({from:n.viseme,to:"sil",fromVtime:s,toVtime:s+fr}),t}function vn(e){const t=new Map;for(const r of e)t.has(r.vtime)||t.set(r.vtime,r);return[...t.values()].sort((r,n)=>r.vtime-n.vtime)}function Rs(e){const t=vn(e),r=[];for(let n=0;n<t.length-1;n++){const s=t[n].viseme,i=t[n+1].viseme,o=t[n].vtime,a=t[n+1].vtime;s===i||a<=o||r.push({from:s,to:i,fromVtime:o,toVtime:a})}return r}function Ms(e){return e.length===0?[]:[...xs(e),...Rs(e)]}function Ds(e,t){let r=null;for(const n of e){const{transStart:s,transEnd:i}=bn(n.fromVtime,n.toVtime,n.from,n.to);t>=s&&t<i&&(r=n)}return r}function bn(e,t,r,n){const s=Math.max(0,t-e);if((r==="PP"||n==="PP")&&s>Or){const l=Or,f=t,c=f-l;return{gapMs:s,effectiveGapMs:l,holdEnd:c,transStart:c,transEnd:f}}const o=s>0?Math.max(s,Is):0,a=e,u=e+o;return{gapMs:s,effectiveGapMs:o,holdEnd:e,transStart:a,transEnd:u}}const hr=16e3;function Tn(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:hr}function Cs(e){let t=e.trim();const r=t.indexOf(",");t.startsWith("data:")&&r>=0&&(t=t.slice(r+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const n=t.length%4;return n!==0&&(t+="=".repeat(4-n)),t}function Bs(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function Nr(e,t,r=hr){const n=t.byteOffset%2===0?t:t.slice(),s=new Int16Array(n.buffer,n.byteOffset,n.byteLength/2),i=new Float32Array(s.length);for(let a=0;a<s.length;a++)i[a]=s[a]>=32768?-(65536-s[a])/32768:s[a]/32767;const o=e.createBuffer(1,i.length,r);return o.copyToChannel(i,0),o}function Fs(e){const t=Cs(e),r=window.atob(t),n=new Uint8Array(r.length);for(let s=0;s<r.length;s++)n[s]=r.charCodeAt(s);return n}async function Ur(e,t,r=hr){const n=Fs(t),s=Tn(r);if(Bs(n)){const i=n.buffer.slice(n.byteOffset,n.byteOffset+n.byteLength);return e.decodeAudioData(i)}if(n.length>=2&&n.length%2===0)return Nr(e,n,s);try{const i=n.buffer.slice(n.byteOffset,n.byteOffset+n.byteLength);return await e.decodeAudioData(i)}catch{return Nr(e,n,s)}}function Ls(e,t){for(const r of e)if(t>=r.wtime&&t<r.wtime+r.wduration)return r;return null}function kn(e,t){const r=[],n=[];let s=null,i=!1,o=!1,a=!1,u=!1,l=0,f=0,c=0,d=null,g=1,b=t,m=!1,R=null,_=0,D=!1,w=!1,T=0,O=null,B=null,G=null;const F=()=>({visemes:[],words:[],segmentOffsetMs:0}),k=()=>{w=!1,T=0},P=()=>{d!=null&&(clearTimeout(d),d=null)},C=()=>{for(const h of n){try{h.onended=null,h.stop(0)}catch{}try{h.disconnect()}catch{}}n.length=0},V=()=>{for(const h of n)h.playbackRate.value=g},N=()=>{if(!(m||!u||i||w||n.length>0||r.length>0)){if(a&&s&&f>0){const h=(s.currentTime-f)*1e3;if(h<_){P();const A=g>0?(_-h)/g:_-h;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,A));return}}o=!1,P(),e.onSpeakingDone(),e.onNotify()}},he=h=>{const A=He(h);if(A<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,l=performance.now(),e.onSpeakingStart(),e.onNotify(),P();const M=g>0?A/g:A;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},M)},X=(h,A,M,L)=>{A.length!==0&&(c=M,A.forEach(I=>{const Y=String(I.word??"").trim();if(!Y)return;const Q=Math.round((I.start??0)*1e3),Z=I.duration!=null?Math.round(I.duration*1e3):I.end!=null?Math.max(1,Math.round((I.end-(I.start??0))*1e3)):0;h.push({word:Y,wtime:c+Q,wduration:Z,queueIndex:h.length,gesture:I.gesture??null,emphasis:I.emphasis===!0})}))},z=(h,A,M,L,I,$)=>{M.length!==0&&(c=L,M.forEach(Y=>{const Q=String(Y.symbol??Y.viseme??"").trim();if(!Q)return;const Z=Math.round((Y.start??0)*1e3),Ie=Math.max(1,Math.round((Y.duration??0)*1e3)),be=c+Z,Be=Ls(A,be),at=Be?vs(Be,be):0;let Pe;ss($)==="oculus_direct"?(Pe=yn(Q),Pe==="U"&&Be&&mr(Be.word,at)&&(Pe="FF")):Pe=As(Q,Be?.word??"",at),h.push({viseme:Pe,weight:1,vtime:be,vduration:Ie})}),e.onNotify())},J=(h,A,M,L,I,$,Y)=>{X(A,L,I),z(h,A,M,I,$,Y)},se=(h,A,M)=>{G===h&&h.visemes===A||(A.length=0,M.length=0,h.visemes=A,h.words=M,h.segmentOffsetMs=0,G=h,c=0,_=0)},ee=async()=>{const h=window.AudioContext||window.webkitAudioContext;return s||(s=new h),s.state==="suspended"&&await s.resume(),R||(R=s.createMediaStreamDestination()),s},ge=(h,A)=>{if(!s)return 0;const M=s,L=M.currentTime,I=L+.02;if(A.isFirstAudio){f<=0&&(f=I);const Y=(I-L)*1e3;l=performance.now()+Y}A.onBeforeStart?.(I);const $=M.createBufferSource();return $.buffer=h,$.playbackRate.value=g,$.connect(M.destination),R&&$.connect(R),n.push($),m?(n.pop(),I):(A.isFirstAudio&&(o=!0,e.onSpeakingStart()),$.onended=()=>{const Y=n.indexOf($);Y>=0&&n.splice(Y,1),e.onNotify(),!m&&(O&&B?ce(O,B):N())},$.start(I),I)},ce=async(h,A)=>{if(m||i||n.length>0)return;if(r.length===0){N();return}i=!0;let M=!1;try{const L=await ee();if(m)return;const I=r[0];if(!I)return;const $=!o;if(D&&(e.shouldHoldForWarmup?.()??!1)&&(w||$&&I.newStream)||w){se(I.utterance,h,A);for(const Z of r){if(m)return;Z.audioBuffer||(Z.audioBuffer=await Ur(L,Z.audio,Z.sampleRate)),a=!0,w||(w=!0,T=0,Z.newStream&&Z.onNewStream?.()),Z.visemesQueued||(J(h,A,Z.visemes,Z.words,T,!0,b),Z.visemesQueued=!0,T+=Math.round(Z.audioBuffer.duration*1e3),_=Math.max(_,He(h)))}e.onNotify();return}const Q=r.shift();if(!Q||(Q.audioBuffer||(Q.audioBuffer=await Ur(L,Q.audio,Q.sampleRate)),m))return;a=!0,P(),se(Q.utterance,h,A),Q.visemesQueued=!1,Q.newStream&&Q.onNewStream?.(),ge(Q.audioBuffer,{isFirstAudio:!o,isFirstOfStream:!1,onBeforeStart:Z=>{f=Z,c=0,l=performance.now()+(Z-L.currentTime)*1e3,J(h,A,Q.visemes,Q.words,0,!0,b),Q.visemesQueued=!0,Q.utterance.segmentOffsetMs=0}}),M=n.length>0,_=He(h),e.onNotify()}catch(L){console.error("[avatarChunkPlayback] audio chunk failed:",L)}finally{if(i=!1,m)return;if(w){r.some(I=>!I.audioBuffer||!I.visemesQueued)&&ce(h,A);return}!M&&n.length===0&&r.length>0?ce(h,A):n.length===0&&r.length===0&&N()}};return{setStreamTtsEngineId:h=>{b=h},setPlaybackSpeed:h=>{g=Math.max(.1,Math.min(1,h)),V()},getPlaybackSpeed:()=>g,setLipsyncWarmupHoldEnabled:h=>{D=h,h||k()},isWarmingUp:()=>w,releaseWarmupHold:()=>{if(w){w=!1,T=0;for(const h of r)h.newStream=!1;O&&B&&ce(O,B),e.onNotify()}},resetForNewStream:h=>{m=!1,u=!1,b=h,_=0},resumeBotStream:()=>{m=!1,u=!1},clearAborted:()=>{m=!1},primeAudioContext:async()=>{const h=window.AudioContext||window.webkitAudioContext;s||(s=new h),s.state==="suspended"&&await s.resume(),s&&!R&&(R=s.createMediaStreamDestination())},stop:()=>{m=!0,P(),C(),r.length=0,i=!1,o=!1,a=!1,u=!1,c=0,f=0,l=0,_=0,O=null,B=null,G=null,k()},markStreamComplete:h=>{if(_=Math.max(_,He(h)),u=!0,!a&&h.length>0){he(h);return}N()},playAudioChunk:async(h,A,M,L,I,$,Y)=>{if(m)return;O=h,B=A;const Q=Tn(Y?.sampleRate),Z=Y?.newStream===!0,Ie=F();if(!M?.trim()){if(L.length===0&&I.length===0)return;se(Ie,h,A),Z&&Y?.onNewStream?.(),J(h,A,L,I,0,!0,b),_=He(h),!o&&_>0&&he(h),e.onNotify();return}r.push({audio:M,audioBuffer:null,utterance:Ie,visemes:L,words:I,isNewSegment:$,sampleRate:Q,newStream:Z,onNewStream:Y?.onNewStream,visemesQueued:!1}),await ce(h,A)},queueTimingsOnly:(h,A,M,L,I)=>{if(m)return;const $=F();se($,h,A),J(h,A,M,L,0,!0,b),$.segmentOffsetMs=0,_=He(h)},getPlaybackElapsedMs:()=>a&&s&&f>0?Math.max(0,(s.currentTime-f)*1e3):o?Math.max(0,(performance.now()-l)*g):0,isSpeaking:()=>o?!0:!a||!s||f<=0?!1:(s.currentTime-f)*1e3<_,getVoiceStream:()=>R?R.stream:null,getPlaybackStartPerf:()=>l,bargeIn:()=>{m=!0,u=!1,P(),C(),r.length=0,i=!1,o=!1,a=!1,c=0,f=0,l=0,_=0,O=null,B=null,G=null,k()},interrupt:()=>{m=!0,u=!1,P(),C(),r.length=0,i=!1,o=!1,a=!1,c=0,f=0,l=0,_=0,O=null,B=null,G=null,k()}}}function bt(e,t){let r=null;for(const n of e)t<n.vtime||t>=n.vtime+n.vduration||(!r||n.vtime>=r.vtime)&&(r=n);return r?r.viseme:"sil"}function En(e,t,r=120){const n=bt(e,t);if(n!=="sil")return n;let s=null;for(const i of e){if(i.vtime>t)break;const o=i.vtime+i.vduration;o<=t&&t-o<=r&&(s=i)}return s?.viseme??"sil"}function Os(e,t,r=400){const n=t-r;let s=0;for(const i of e)if(i.vtime+i.vduration<n)s+=1;else break;s>0&&e.splice(0,s)}function Ns(e,t){for(const r of e)if(t>=r.wtime&&t<r.wtime+r.wduration)return r;return null}function Us(e,t){const r=e.trim();if(!r)return null;try{const n=r.replace(/\s+/g,""),s=atob(n).length,i=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return s/2/i*1e3}catch{return null}}const Wt=.5,Ht=1.5,pr="Olivia",wr="f786b574-daa5-4673-aa0c-cbe3e8534c02",Tt=.85;function nr(e,t=St,r={}){const n=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,s=[],i=[],o=[],a=[];let u="",l="neutral",f="",c=qe;const d=new Set;let g=0,b=null;const m=()=>d.forEach(F=>F());let R=()=>!1;const _=kn({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:m,shouldHoldForWarmup:()=>R()},qe),D=()=>_.getPlaybackElapsedMs(),w=(F,k=u)=>{if(!r.onLipsyncDebug)return;const P=D(),C=s;r.onLipsyncDebug({event:F,spokenText:k,mood:l,playbackAnchorPerf:_.getPlaybackStartPerf(),playbackElapsedMs:P,queueLength:C.length,activeViseme:bt(C,P),segments:a.map(V=>({...V,visemes:[...V.visemes]}))})},T=(F,k)=>{r.onLipsyncDebug&&(b={event:F,spokenText:k},g===0&&(g=requestAnimationFrame(()=>{g=0;const P=b;b=null,P&&w(P.event,P.spokenText)})))},O=(F,k,P,C,V)=>{if(!r.onLipsyncDebug)return;const N=s.slice(k),he=P.filter(X=>String(X.symbol??X.viseme??"").trim());a.push({spokenText:F,audioOffsetMs:N[0]?.vtime??0,isNewSegment:C,chunkDurationMs:V,receivedAtMs:performance.now(),visemes:N.map((X,z)=>{const J=he[z];return{rawSymbol:String(J?.symbol??J?.viseme??X.viseme).trim(),viseme:X.viseme,startSec:Number(J?.start??0),durationSec:Number(J?.duration??0),vtimeMs:X.vtime,vdurationMs:X.vduration}})})},B=()=>{_.stop(),s.length=0,i.length=0,o.length=0,l="neutral",a.length=0,u="",e("idle"),T("reset"),m()};return{setDeveloperToken:F=>{f=F},setTtsEngineId:F=>{c=F},getTtsEngineId:()=>c,speak:async(F,k)=>{B(),e("loading"),u=F,a.length=0;const P=f.trim();if(!P)throw new Error("Developer token required for avatar_ttsWithPoses");const C=k?.ttsEngineId??c??qe;_.resetForNewStream(C),_.setStreamTtsEngineId(C);const V=k?.voiceId??pr,N=k?.speakingRate??Tt,he=Math.max(Wt,Math.min(Ht,N)),X={user_query:F,ttsEngineId:C,speakingRate:he,tenant:"aiTwin"};on.has(C)&&(X.voice_id=V);try{const z=await fetch(n,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${P}`},body:JSON.stringify(X)});if(!z.ok)throw new Error(`avatar_ttsWithPoses failed (${z.status})`);const J=z.body;if(!J)throw new Error("No response body");const se=J.getReader(),ee=new TextDecoder;let ge="",ce=!0;const fe=async S=>{const E=wn(S);if(E){if(E.event==="audio"){const h=E.data.chunk,A=E.data.visemes??[],M=E.data.words??[],L=E.data.is_new_segment??!1,I=E.data.sample_rate,$=s.length;if(h){const Y=ce;ce=!1,await _.playAudioChunk(s,i,h,A,M,L,{sampleRate:I,newStream:Y}),O(F,$,A,L,Us(h,I)),T("audio_chunk",F)}else(A.length>0||M.length>0)&&(_.queueTimingsOnly(s,i,A,M,L),O(F,$,A,L,null),T("audio_chunk",F))}else if(E.event==="metadata"){const h=E.data.mood;typeof h=="string"&&h.trim()&&(l=h.trim());const A=E.data.sentence_emotions;if(Array.isArray(A)){o.length=0;for(const M of A){if(!M||typeof M!="object")continue;const L=M;o.push({sentence_index:Number(L.sentence_index??0),text:String(L.text??""),sentiment:typeof L.sentiment=="string"?L.sentiment:void 0,emotion:String(L.emotion??l),start_word:Number(L.start_word??0),end_word:Number(L.end_word??0)})}}m(),T("metadata",F)}else if(E.event==="error")throw new Error(String(E.data.message??"avatar_ttsWithPoses stream error"))}};for(;;){const{done:S,value:E}=await se.read();E&&(ge+=ee.decode(E,{stream:!0}));const h=ge.split(`
|
|
2
2
|
|
|
3
|
-
`);ge=h.pop()??"";for(const A of h)await fe(A);if(S){ge.trim()&&await fe(ge.trim()),_.markStreamComplete(s),T("bot_stopped",F);break}}}catch(z){throw console.error("[avatarTtsLipsync]",z),B(),e("error"),z}},stop:B,getVisemeQueue:()=>s,getWordQueue:()=>i,getSentenceEmotions:()=>o,getStreamMood:()=>l,getPlaybackElapsedMs:()=>_.getPlaybackElapsedMs(),isSpeaking:()=>_.isSpeaking(),isWarmingUp:()=>_.isWarmingUp(),releaseWarmupHold:()=>_.releaseWarmupHold(),setLipsyncWarmupHold:(F,k)=>{k&&(R=k),_.setLipsyncWarmupHoldEnabled(F)},getVoiceStream:()=>_.getVoiceStream(),setPlaybackSpeed:F=>{_.setPlaybackSpeed(F)},getPlaybackSpeed:()=>_.getPlaybackSpeed(),subscribe:F=>(d.add(F),()=>d.delete(F))}}const gr=16e3,yr=16e3;class Vs extends ur.ProtobufFrameSerializer{async deserialize(t){try{return await super.deserialize(t)}catch(r){if(r instanceof Error&&r.message==="Unknown frame kind")return{type:"raw",message:null};throw r}}}class Ws extends ur.DailyMediaManager{constructor(){super(!0,!0,void 0,void 0,512,gr,yr)}bufferBotAudio(t,r){}}const Hs=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function $s(e){return Hs.test(e.trim())}function qs(e){const t=e.replace(/\/$/,"");return t.startsWith("https://")?`wss://${t.slice(8)}`:t.startsWith("http://")?`ws://${t.slice(7)}`:t}function An(e){if(e.wsUrl?.trim())return e.wsUrl.trim();const t=e.authToken?.trim();if(!t)throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");const r=e.apiBase??St,n=new URL(`${qs(r)}/ws/voice`);n.searchParams.set("authToken",t),n.searchParams.set("tenant","aiTwin");const s=e.voiceId?.trim();s&&$s(s)&&n.searchParams.set("voiceId",s);const i=e.personaId?.trim();i&&n.searchParams.set("personaId",i);const o=e.speakingRate??Tt;return n.searchParams.set("speaking_rate",String(Math.max(Wt,Math.min(Ht,o)))),e.ttsStream===!1?n.searchParams.set("tts_stream","false"):n.searchParams.set("tts_stream","true"),n.toString()}function Jt(e,t){if(!Array.isArray(e))return[];const r=[];for(const n of e)!n||typeof n!="object"||r.push({sentence_index:Number(n.sentence_index??0),text:String(n.text??""),sentiment:typeof n.sentiment=="string"?n.sentiment:void 0,emotion:String(n.emotion??t),start_word:Number(n.start_word??0),end_word:Number(n.end_word??0)});return r}function Ks(e){if(!e||typeof e!="object")return null;const t=e,n=(t.data&&typeof t.data=="object"&&!Array.isArray(t.data)?t.data:null)||t;return{...n,type:String(n.type??t.type??"server-message")}}function Qt(e){return e.is_new_segment===!0||Number(e.audio_offset_ms??-1)===0?!0:Number(e.chunk_index??-1)===0}function Gs(e){return e?.transport?._mediaManager?._wavStreamPlayer??null}function In(e,t={}){const r=[],n=[],s=[],i=[];let o="",a="neutral",u=Ge,l=!1,f=null,c=null,d={...t},g=0,
|
|
3
|
+
`);ge=h.pop()??"";for(const A of h)await fe(A);if(S){ge.trim()&&await fe(ge.trim()),_.markStreamComplete(s),T("bot_stopped",F);break}}}catch(z){throw console.error("[avatarTtsLipsync]",z),B(),e("error"),z}},stop:B,getVisemeQueue:()=>s,getWordQueue:()=>i,getSentenceEmotions:()=>o,getStreamMood:()=>l,getPlaybackElapsedMs:()=>_.getPlaybackElapsedMs(),isSpeaking:()=>_.isSpeaking(),isWarmingUp:()=>_.isWarmingUp(),releaseWarmupHold:()=>_.releaseWarmupHold(),setLipsyncWarmupHold:(F,k)=>{k&&(R=k),_.setLipsyncWarmupHoldEnabled(F)},getVoiceStream:()=>_.getVoiceStream(),setPlaybackSpeed:F=>{_.setPlaybackSpeed(F)},getPlaybackSpeed:()=>_.getPlaybackSpeed(),subscribe:F=>(d.add(F),()=>d.delete(F))}}const gr=16e3,yr=16e3;class Vs extends ur.ProtobufFrameSerializer{async deserialize(t){try{return await super.deserialize(t)}catch(r){if(r instanceof Error&&r.message==="Unknown frame kind")return{type:"raw",message:null};throw r}}}class Ws extends ur.DailyMediaManager{constructor(){super(!0,!0,void 0,void 0,512,gr,yr)}bufferBotAudio(t,r){}}const Hs=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function $s(e){return Hs.test(e.trim())}function qs(e){const t=e.replace(/\/$/,"");return t.startsWith("https://")?`wss://${t.slice(8)}`:t.startsWith("http://")?`ws://${t.slice(7)}`:t}function An(e){if(e.wsUrl?.trim())return e.wsUrl.trim();const t=e.authToken?.trim();if(!t)throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");const r=e.apiBase??St,n=new URL(`${qs(r)}/ws/voice`);n.searchParams.set("authToken",t),n.searchParams.set("tenant","aiTwin");const s=e.voiceId?.trim();s&&$s(s)&&n.searchParams.set("voiceId",s);const i=e.personaId?.trim();i&&n.searchParams.set("personaId",i);const o=e.speakingRate??Tt;return n.searchParams.set("speaking_rate",String(Math.max(Wt,Math.min(Ht,o)))),e.ttsStream===!1?n.searchParams.set("tts_stream","false"):n.searchParams.set("tts_stream","true"),n.toString()}function Jt(e,t){if(!Array.isArray(e))return[];const r=[];for(const n of e)!n||typeof n!="object"||r.push({sentence_index:Number(n.sentence_index??0),text:String(n.text??""),sentiment:typeof n.sentiment=="string"?n.sentiment:void 0,emotion:String(n.emotion??t),start_word:Number(n.start_word??0),end_word:Number(n.end_word??0)});return r}function Ks(e){if(!e||typeof e!="object")return null;const t=e,n=(t.data&&typeof t.data=="object"&&!Array.isArray(t.data)?t.data:null)||t;return{...n,type:String(n.type??t.type??"server-message")}}function Qt(e){return e.is_new_segment===!0||Number(e.audio_offset_ms??-1)===0?!0:Number(e.chunk_index??-1)===0}function Gs(e){return e?.transport?._mediaManager?._wavStreamPlayer??null}function In(e,t={}){const r=[],n=[],s=[],i=[];let o="",a="neutral",u=Ge,l=!1,f=null,c=null,d={...t},g=0,b=null;const m=new Set,R=[];let _=!1,D=null,w=!0,T="default";const O=new Set;let B=null,G=0,F=!1,k=null;const P=()=>{k!=null&&(clearTimeout(k),k=null)},C=()=>m.forEach(p=>p());let V=()=>!1;const N=kn({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:C,shouldHoldForWarmup:()=>V()},u),he=()=>{if(D)return D;const p=new MessageChannel;return p.port1.onmessage=()=>{const x=R.shift();x!==void 0&&Q(x),R.length>0?p.port2.postMessage(null):_=!1},D=p,p},X=p=>{R.push(p),!_&&(_=!0,he().port2.postMessage(null))},z=()=>N.getPlaybackElapsedMs(),J=(p,x=o)=>{if(!d.onLipsyncDebug)return;const j=z(),ae=r;d.onLipsyncDebug({event:p,spokenText:x,mood:a,playbackAnchorPerf:N.getPlaybackStartPerf(),playbackElapsedMs:j,queueLength:ae.length,activeViseme:bt(ae,j),segments:i.map(pe=>({...pe,visemes:[...pe.visemes]}))})},se=(p,x)=>{d.onLipsyncDebug&&(b={event:p,spokenText:x},g===0&&(g=requestAnimationFrame(()=>{g=0;const j=b;b=null,j&&J(j.event,j.spokenText)})))},ee=()=>{w=!0,O.clear(),N.clearAborted()},ge=()=>{const p=Gs(f);if(p){T=`bot-${Date.now()}`,p.interruptedTrackIds={[T]:!0};try{p.interrupt?.()}catch{}}const x=c;try{x?.interrupt?.()}catch{}},ce=p=>{const x=p.context_id?.trim();return x&&O.has(x)?!0:w?!1:!(p.type==="avatar_metadata"||Qt(p))},fe=p=>{w=!1;const x=p.context_id?.trim();x&&O.add(x),P(),F=!1,R.length=0,_=!1,B?.abort(),B=null,G+=1,ge(),N.bargeIn(),S(),e("idle"),se("interrupt"),C()},S=()=>{r.length=0,n.length=0,i.length=0,o="",a="neutral",s.length=0},E=()=>{N.stop(),S(),se("reset"),C()},h=()=>{E(),e("idle"),C()},A=(p,x)=>{typeof p.mood=="string"&&p.mood.trim()&&(a=p.mood.trim()),Array.isArray(p.sentence_emotions)&&p.sentence_emotions.length>0&&(x&&(s.length=0),s.push(...Jt(p.sentence_emotions,a)))},M=p=>{if(ce({...p,type:"avatar_metadata"}))return;ee(),F=!0;const x=String(p.spoken_text??p.text??"").trim();x&&(o=x),N.resetForNewStream(u),typeof p.mood=="string"&&p.mood.trim()&&(a=p.mood.trim()),s.length=0,s.push(...Jt(p.sentence_emotions,a)),Array.isArray(p.gestures)&&p.gestures,se("metadata",x),C()},L=(p,x,j,ae)=>{if(!d.onLipsyncDebug)return;const pe=r.slice(j),_e=ae.filter(ue=>String(ue.symbol??ue.viseme??"").trim());i.push({spokenText:x,audioOffsetMs:pe[0]?.vtime??0,isNewSegment:!!p.is_new_segment,chunkDurationMs:p.chunk_duration_ms!=null?Number(p.chunk_duration_ms):null,receivedAtMs:performance.now(),visemes:pe.map((ue,xe)=>{const de=_e[xe];return{rawSymbol:String(de?.symbol??de?.viseme??ue.viseme).trim(),viseme:ue.viseme,startSec:Number(de?.start??0),durationSec:Number(de?.duration??0),vtimeMs:ue.vtime,vdurationMs:ue.vduration}})})},I=p=>{if(ce(p))return;P(),Qt(p)&&ee();const x=String(p.spoken_text??p.text??"").trim(),j=Qt(p),ae=p.visemes??[],pe=p.words??[],_e=String(p.chunk??"").trim(),ue=F;ue&&(F=!1),x&&(o=x),A(p,j),ue?N.resetForNewStream(u):j&&N.resumeBotStream();const xe=r.length;N.playAudioChunk(r,n,_e,ae,pe,j,{sampleRate:p.sample_rate,newStream:ue,onNewStream:()=>{i.length=0}}).then(()=>{L(p,x,xe,ae),se("audio_chunk",x)})},$=p=>{if(ce(p))return;(p.is_new_segment||Number(p.audio_offset_ms??-1)===0)&&ee(),console.warn("[realtimeVoice] avatar_lipsync is deprecated — expect avatar_audio_chunk");const x=String(p.spoken_text??p.text??"").trim();x&&(o=x),A(p,!!p.is_new_segment);const j=r.length;N.queueTimingsOnly(r,n,p.visemes??[],p.words??[],!!p.is_new_segment),L(p,x,j,p.visemes??[]),se("lipsync",x),C()},Y=async p=>{const x=String(p.text??"").trim();if(!x)return;const j=d.authToken?.trim();if(!j){console.error("[realtimeVoice] avatar_speak requires authToken");return}B?.abort();const ae=new AbortController;B=ae;const pe=++G;ee(),N.resetForNewStream(u),o=x;const ue=`${(d.apiBase??St).replace(/\/$/,"")}/avatar_ttsWithPoses`,xe=d.voiceId?.trim()||wr,de=d.speakingRate??Tt,Ue=Math.max(Wt,Math.min(Ht,de)),Ye={user_query:x,ttsEngineId:u,speakingRate:Ue,tenant:"aiTwin"};on.has(u)&&(Ye.voice_id=xe);try{const Se=await fetch(ue,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${j}`},body:JSON.stringify(Ye),signal:ae.signal});if(!Se.ok)throw new Error(`avatar_ttsWithPoses failed (${Se.status})`);const ot=Se.body;if(!ot)throw new Error("No response body");const Xe=ot.getReader(),ct=new TextDecoder;let Oe="",le=!0;const ut=async Je=>{if(pe!==G||ae.signal.aborted||!w)return;const me=wn(Je);if(me){if(me.event==="audio"){const we=me.data.chunk,Te=me.data.visemes??[],Re=me.data.words??[],lt=me.data.is_new_segment??!1,Me=me.data.sample_rate;if(we){const Ve=le;le=!1,await N.playAudioChunk(r,n,we,Te,Re,lt,{sampleRate:Me,newStream:Ve})}else(Te.length>0||Re.length>0)&&N.queueTimingsOnly(r,n,Te,Re,lt)}else if(me.event==="metadata"){const we=me.data.mood;typeof we=="string"&&we.trim()&&(a=we.trim());const Te=me.data.sentence_emotions;Array.isArray(Te)&&(s.length=0,s.push(...Jt(Te,a))),C()}else if(me.event==="error")throw new Error(String(me.data.message??"avatar_ttsWithPoses stream error"))}};for(;!(pe!==G||ae.signal.aborted||!w);){const{done:Je,value:me}=await Xe.read();me&&(Oe+=ct.decode(me,{stream:!0}));const we=Oe.split(`
|
|
4
4
|
|
|
5
|
-
`);Oe=we.pop()??"";for(const Te of we)await ut(Te);if(Je){Oe.trim()&&await ut(Oe.trim()),pe===G&&!ae.signal.aborted&&w&&N.markStreamComplete(r);break}}}catch(Se){if(ae.signal.aborted||!w)return;console.error("[realtimeVoice] avatar_speak failed:",Se),e("error"),C()}finally{B===ae&&(B=null)}},Q=p=>{const x=Ks(p);if(!x){console.warn("[realtimeVoice] server message with no payload",p);return}const j=String(x.type??"unknown");switch(j!=="avatar_metadata"&&j!=="avatar_audio_chunk"&&j!=="avatar_lipsync"&&j!=="avatar_interrupt"&&j!=="error"&&console.debug("[realtimeVoice] server message",j,x),j){case"avatar_metadata":M(x);break;case"avatar_audio_chunk":I(x);break;case"avatar_lipsync":$(x);break;case"avatar_interrupt":fe(x);break;case"error":{const ae=String(x.message??"Voice session error");console.error("[realtimeVoice] server error:",ae),e("error"),C();break}case"avatar_speak":Y(x);break}},Z=()=>f||(c=new Ws,f=new zn.PipecatClient({transport:new ur.WebSocketTransport({mediaManager:c,serializer:new Vs,recorderSampleRate:gr,playerSampleRate:yr}),enableCam:!1,enableMic:!0,callbacks:{onBotReady:()=>{l=!0,e("idle"),C()},onDisconnected:()=>{l=!1,h()},onError:p=>{const x=typeof p=="object"&&p!==null&&"data"in p?String(p.data??p):String(p);console.error("[realtimeVoice]",x,p),e("error"),C()},onTransportStateChanged:p=>{p==="error"&&(e("error"),C())},onServerMessage:p=>{X(p)},onUserStartedSpeaking:()=>{fe({})},onBotStartedSpeaking:()=>{ee(),se("bot_started")},onBotStoppedSpeaking:()=>{w&&(P(),k=setTimeout(()=>{k=null,w&&(N.markStreamComplete(r),se("bot_stopped"))},450))},onUserTranscript:p=>{d.onUserTranscript?.(p.text,!!p.final)},onBotOutput:p=>{d.onBotOutput?.(p.text)}}}),f);return{setDeveloperToken:p=>{},setTtsEngineId:p=>{u=p,N.setStreamTtsEngineId(p)},getTtsEngineId:()=>u,speak:async(p,x)=>{const j=p.trim();if(j){if(!f||!l)throw new Error("Realtime voice is not connected. Call connect() first.");E(),await f.sendText(j)}},stop:()=>{B?.abort(),B=null,G+=1,ee(),E(),e("idle"),C()},connect:async p=>{if(p&&(d={...d,...p}),!d.authToken?.trim())throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");e("loading");const x=Z();await x.initDevices();const j=An(d);console.info("[realtimeVoice] connecting (avatar_audio_chunk RTVI):",j);try{await x.connect({wsUrl:j})}catch(ae){l=!1,e("error"),C();const pe=ae instanceof Error?ae.message:"Realtime voice connect failed (WebSocket 101 only means upgrade OK — check authToken/origin/RTVI handshake)";throw new Error(`${pe} — ${j}`)}l=!0,e("idle"),C()},disconnect:async()=>{if(P(),F=!1,R.length=0,_=!1,B?.abort(),B=null,G+=1,ee(),f)try{await f.disconnect()}catch(p){console.error("[realtimeVoice] disconnect failed:",p)}l=!1,h()},isConnected:()=>l,unlockAudio:async()=>{await N.primeAudioContext()},getVisemeQueue:()=>r,getWordQueue:()=>n,getSentenceEmotions:()=>s,getStreamMood:()=>a,getPlaybackElapsedMs:z,isSpeaking:()=>N.isSpeaking(),isWarmingUp:()=>N.isWarmingUp(),releaseWarmupHold:()=>N.releaseWarmupHold(),setLipsyncWarmupHold:(p,x)=>{x&&(V=x),N.setLipsyncWarmupHoldEnabled(p)},getVoiceStream:()=>N.getVoiceStream(),setPlaybackSpeed:p=>{N.setPlaybackSpeed(p)},getPlaybackSpeed:()=>N.getPlaybackSpeed(),subscribe:p=>(m.add(p),()=>m.delete(p))}}function Pn(e,t,r){return{width:e.crop?.source_width??t,height:e.crop?.source_height??r}}function xn(e,t){const r=e.crop;return{x:t?.source_x??r?.x??0,y:t?.source_y??r?.y??0,width:t?.source_w??r?.width??e.frame_width??0,height:t?.source_h??r?.height??e.frame_height??0}}function Rn(e){return e?(e.w??0)>0&&(e.h??0)>0:!1}async function it(e,t){const r=await fetch(e);if(!r.ok)throw new Error(`Failed to load encrypted asset: ${e}`);const n=await r.arrayBuffer();return ps(n,t)}async function Mn(e,t){if(!t)throw new Error("Missing asset key for encrypted JSON");const r=await it(e,t),n=new TextDecoder().decode(r);return JSON.parse(n)}async function zs(e){const t=new Blob([e],{type:"image/webp"});return createImageBitmap(t)}async function js(e){const t=new Blob([e],{type:"image/webp"}),r=URL.createObjectURL(t);return new Promise((n,s)=>{const i=new Image;i.decoding="async",i.onload=()=>{URL.revokeObjectURL(r),n(i)},i.onerror=()=>{URL.revokeObjectURL(r),s(new Error("Failed to decode decrypted image"))},i.src=r})}async function Ys(e){const t=new Blob([e],{type:"image/png"});return createImageBitmap(t)}async function Xs(e){const t=new Blob([e],{type:"image/png"}),r=URL.createObjectURL(t);return new Promise((n,s)=>{const i=new Image;i.decoding="async",i.onload=()=>{URL.revokeObjectURL(r),n(i)},i.onerror=()=>{URL.revokeObjectURL(r),s(new Error("Failed to decode decrypted PNG"))},i.src=r})}function Js(e){const t=new Blob([e],{type:"video/mp4"});return URL.createObjectURL(t)}async function Qs(e){const t=gs();if(!vt())return t;if(!e)throw new Error("Missing asset key for encrypted idle video");const r=await it(t,e);return Js(r)}function _r(e,t){return`${e}/${t}_minus_sil.png`}const Dn=5;function $t(e){const t=e?.composite_threshold;if(typeof t=="number"&&t>0)return t;const r=e?.crop?.content_threshold;return typeof r=="number"&&r>0?r:Dn}const Zs=["aa","E","I","O","U","PP","FF","DD","SS","TH","CH","RR","kk","nn"],sr=["t01_0.25","t02_0.50","t03_0.75"],ei=sr.length,Vr=new Set(["aa__kk","aa__nn","aa__sil","CH__aa","CH__DD","CH__E","CH__FF","CH__I","CH__kk","CH__nn","CH__O","CH__PP","CH__RR","CH__sil","CH__SS","CH__TH","CH__U","DD__aa","DD__E","DD__FF","DD__I","DD__kk","DD__nn","DD__O","DD__PP","DD__RR","DD__sil","DD__SS","DD__TH","DD__U","E__aa","E__FF","E__I","E__kk","E__nn","E__O","E__PP","E__RR","E__sil","E__SS","E__TH","E__U","FF__aa","FF__I","FF__kk","FF__nn","FF__O","FF__PP","FF__RR","FF__sil","FF__SS","FF__TH","FF__U","I__aa","I__kk","I__nn","I__O","I__PP","I__RR","I__sil","I__SS","I__TH","I__U","kk__nn","kk__sil","nn__sil","O__aa","O__kk","O__nn","O__PP","O__RR","O__sil","O__SS","O__TH","O__U","PP__aa","PP__kk","PP__nn","PP__RR","PP__sil","PP__SS","PP__TH","PP__U","RR__aa","RR__kk","RR__nn","RR__sil","RR__SS","RR__TH","RR__U","SS__aa","SS__kk","SS__nn","SS__sil","SS__TH","SS__U","TH__aa","TH__kk","TH__nn","TH__sil","TH__U","U__aa","U__kk","U__nn","U__sil"]);function Wr(e,t){return`${e}__${t}`}function Hr(e,t){return`pairs/${e}/${t}_minus_sil.png`}function _t(e){return`${e}_minus_sil.png`}function Sr(e,t){if(t.has(e))return e;const r=/^pairs\/([^/]+)\//.exec(e);if(!r)return null;const n=r[1].split("__")[1];if(!n||n==="sil")return null;const s=_t(n);return t.has(s)?s:null}function Cn(e,t){return e.filter(r=>t.has(r))}function ti(e,t,r){const n=vr(e,t);return Cn(n,r).length>0}function vr(e,t){if(e===t)return[];const r=[],n=Wr(e,t);if(Vr.has(n)){for(const i of sr)r.push(Hr(n,i));return r}const s=Wr(t,e);if(Vr.has(s)){for(let i=ei-1;i>=0;i--)r.push(Hr(s,sr[i]));return r}return[]}function ri(e){return e<16?[1]:e<24?[1]:e<35?[0,1,2]:[0,1,2]}function ni(e,t){return ri(t).map(n=>e[n]).filter(n=>n!=null)}function si(e,t,r,n){if(n<=0)return 0;const i=Math.max(1,r-t)/n,o=e-t,a=Math.floor(o/i);return Math.min(n-1,Math.max(0,a))}function Bn(e,t,r){if(!r)return`${e}/${t}`;const n=t.split("/").pop()??t,s=n.includes(".")?n.slice(0,n.lastIndexOf(".")):n;return`${e}/${s}.bin`}const ii=[.32,.24,.16,.08,0],ai=ii.map(e=>`eyes_${e.toFixed(2)}`),oi=["eyes_0.32","eyes_0.16","eyes_0.00"],ci=["eyes_0.32","eyes_0.16","eyes_0.08","eyes_0.00"],$r=2,ui=-30,ir=20,li={fear:-30,anger:-20,sad:-10,neutral:0,love:10,happy:0,disgust:20};function br(e){return`brow_${e}`}function Tr(e){return li[e]}function Ke(e){return Math.max(ui,Math.min(ir,e))}function fi(e,t){if(e===t)return[];const r=e<t?$r:-$r,n=[];let s=e;for(;s!==t;)s+=r,n.push(s);return n}const di={fear:"fear",afraid:"fear",anxiety:"fear",anger:"anger",angry:"anger",mad:"anger",sad:"sad",sadness:"sad",sorrow:"sad",neutral:"neutral",calm:"neutral",love:"love",loving:"love",affection:"love",happy:"happy",happiness:"happy",joy:"happy",joyful:"happy",excited:"happy",disgust:"disgust",disgusted:"disgust"};function tt(e,t="neutral"){const r=String(e??"").trim().toLowerCase();return r?di[r]??t:t}function Fn(e){return e===0?null:_r(br(e),"eyes_standard")}function Ln(e){return Fn(Tr(e))}function qr(e,t){const r=br(Tr(e));return!t||t==="eyes_standard"?Ln(e):e==="neutral"&&t==="eyes_standard"?null:_r(r,t)}function mi(e,t){const r=[];for(const n of t)r.push(qr(e,n));for(let n=t.length-2;n>=0;n--)r.push(qr(e,t[n]));return r.push(Ln(e)),r}function hi(e){return _r(br(e),"eyes_standard")}function pi(e,t){return fi(e,t).map(hi)}const wi=6;function gi(e,t){for(const r of e)if(t>=r.wtime&&t<r.wtime+r.wduration)return r;return null}function yi(e){const t=Ke(e);return t>=ir?ir:Ke(t+wi)}function _i(e,t,r){const n=Ke(r);return gi(t,e)?.emphasis?yi(n):n}function Kr(e){if(!e)return 0;const t=e.match(/^brow_(-?\d+)\//);if(!t?.[1])return null;const r=Number.parseInt(t[1],10);return Number.isFinite(r)?Ke(r):null}function gt(e,t){return e+Math.random()*(t-e)}function Mt(){return gt(14,32)}function Si(e={}){const{initialDelayMs:t=800+Math.random()*1200,intervalMinMs:r=2200,intervalMaxMs:n=5800,doubleBlinkChance:s=.18,doubleBlinkGapMinMs:i=120,doubleBlinkGapMaxMs:o=220,fullBlinkChance:a=.75,postEmotionChangeDelayMs:u=380,minDwellMs:l=550}=e;let f="neutral",c="neutral",d=0,g=0,v=0,m={kind:"idle",nextBlinkAt:performance.now()+t},R=null,_="__eyes_open__|neutral|brow:0";function D(k=f){_=`${R??"__eyes_open__"}|${k}|brow:${g}`}function w(k,P=f){R=k;const C=Kr(k);C!=null?g=C:k==null&&(g=0),D(P)}function T(k){return Fn(k)}function O(k,P,C){const V=Ke(P),N=Ke(C);if(d=N,V===N){w(T(N));return}const he=[...pi(V,N),T(N)];m={kind:"browBlend",paths:he,index:0,holdUntil:k+Mt(),targetBrow:N},w(he[0]??null)}function B(k,P){m={kind:"idle",nextBlinkAt:k+gt(r,n)},c=P,w(T(d),P)}function G(k,P,C){const V=C?oi:Math.random()<a?ai:ci;m={kind:"playing",paths:mi(P,V),index:0,holdUntil:k+Mt(),after:"idle",blinkEmotion:P},c=P,w(m.paths[0]??null,P)}function F(k){if(m.kind==="browBlend"){for(;k>=m.holdUntil&&m.index<m.paths.length-1;)m.index+=1,m.holdUntil=k+Mt();const P=m.paths[m.index]??null;w(P),k>=m.holdUntil&&m.index>=m.paths.length-1&&(g=m.targetBrow,d=m.targetBrow,w(T(m.targetBrow)),m={kind:"idle",nextBlinkAt:k+u});return}if(m.kind==="idle"){w(T(d),c),k>=m.nextBlinkAt&&G(k,c,!1);return}if(m.kind==="doublePause"){w(T(d),m.blinkEmotion),k>=m.resumeAt&&G(k,m.blinkEmotion,!0);return}if(m.kind==="playing"){for(;k>=m.holdUntil&&m.index<m.paths.length-1;)m.index+=1,m.holdUntil=k+Mt();if(w(m.paths[m.index]??null,m.blinkEmotion),k>=m.holdUntil&&m.index>=m.paths.length-1){if(m.after==="resumeIdle"){B(k+u,f);return}if(m.after==="doublePause"){m={kind:"doublePause",resumeAt:k+gt(i,o),blinkEmotion:m.blinkEmotion},w(T(d),m.blinkEmotion);return}if(m.after==="idle"&&Math.random()<s){m={kind:"doublePause",resumeAt:k+gt(i,o),blinkEmotion:m.blinkEmotion},w(T(d),m.blinkEmotion);return}B(k,m.blinkEmotion)}}}return{advance:F,setTargetEmotion(k){const P=tt(k,f);if(P===f)return;const C=performance.now();l>0&&P!==f&&!(f==="neutral"&&P!=="neutral")&&C-v<l||(f=P,c=P,v=C)},setTargetEyebrow(k){const P=Ke(k);if(P===d)return;const C=performance.now(),V=m.kind==="browBlend"?Kr(R)??g:g;d=P,!(P===V&&m.kind!=="browBlend")&&(m.kind==="playing"||m.kind==="doublePause"||O(C,V,P))},getTargetEmotion:()=>f,getTargetEyebrow:()=>d,getExpressionPath:()=>R,getDrawKey:()=>_,reset(){f="neutral",c="neutral",d=0,g=0,v=0,B(performance.now()+gt(r,n),"neutral")}}}function vi(){const e=document.createElement("video");e.muted=!0,e.playsInline=!0,e.preload="auto",e.setAttribute("playsinline",""),e.loop=!0;let t=!1,r=null,n="";const s=()=>{r&&(URL.revokeObjectURL(r),r=null)};return{async load(i){t&&n===i||(s(),t=!1,n=i,i.startsWith("blob:")&&(r=i),await new Promise((o,a)=>{const u=()=>{const l=e.duration;if(!Number.isFinite(l)||l<=0){a(new Error(`Idle video has invalid duration: ${i}`));return}t=!0,e.currentTime=0,o()};e.addEventListener("loadeddata",u,{once:!0}),e.addEventListener("error",()=>a(new Error(`Failed to load idle video: ${i}`)),{once:!0}),e.src=i,e.load()}))},isReady(){return t},getVideo(){return e},setActive(i){t&&(i?e.paused&&e.play().catch(()=>{}):e.paused||e.pause())},restart(){t&&(e.currentTime=0)},dispose(){e.pause(),e.removeAttribute("src"),e.load(),s(),t=!1,n=""}}}const bi=50,On=400;function Ti(e,t,r){return e.filter(n=>{const s=n.queueIndex??-1;return s>=t&&s<=r})}function ki(e,t){const r=[];for(const n of e){const s=Ti(t,n.start_word,n.end_word);if(s.length===0)continue;const i=s[0],o=s[s.length-1];r.push({sentence:n,startMs:Math.max(0,i.wtime-bi),endMs:o.wtime+o.wduration+On})}return r.sort((n,s)=>n.startMs-s.startMs)}function Ei(e,t,r,n="neutral"){const s=tt(n);if(t.length===0)return s;if(r.length===0||e<r[0].wtime)return tt(t[0]?.emotion,s);const i=ki(t,r);let o=null;for(const l of i)e>=l.startMs&&e<=l.endMs&&(o=l);if(o)return tt(o.sentence.emotion,s);const a=r[r.length-1],u=a.wtime+a.wduration;return e>u+On?tt(t[t.length-1]?.emotion,s):s}function kr(e,t,r,n,s=Dn){const i=new OffscreenCanvas(r,n),o=i.getContext("2d",{willReadFrequently:!0});o.drawImage(e,0,0,r,n);const a=o.getImageData(0,0,r,n),u=o.createImageData(r,n);u.data.set(a.data);const f=new OffscreenCanvas(r,n).getContext("2d",{willReadFrequently:!0});for(const c of t){f.clearRect(0,0,r,n),f.drawImage(c,0,0,r,n);const d=f.getImageData(0,0,r,n);for(let g=0;g<u.data.length;g+=4){const v=d.data[g],m=d.data[g+1],R=d.data[g+2];Math.max(v,m,R)>s&&(u.data[g]=v,u.data[g+1]=m,u.data[g+2]=R,u.data[g+3]=255)}}return o.putImageData(u,0,0),i}let $e=null,nt=null;const Ot=new Map;function Ai(){$e=null,nt=null,Ot.clear()}dn(Ai);function Ii(e){return e??nt??void 0}function Er(e){return new Promise((t,r)=>{const n=new Image;n.decoding="async",n.onload=()=>t(n),n.onerror=()=>r(new Error(`Failed to load image: ${e}`)),n.src=e})}async function Pi(e){return vt()?Xs(await it(Lt(),e??"")):Er(Lt())}async function Gr(e,t,r,n){const s=r?await Mn(e,n):await fetch(e).then(l=>{if(!l.ok)throw new Error(`Failed to load ${e}`);return l.json()}),i=s.sheets?.[0];if(!i?.path)throw new Error(`${e} has no sheets[0].path`);const o=r?await js(await it(Bn(t,i.path,r),n??"")):await Er(`${t}/${i.path}`),a=new Map;for(const l of s.cells??[])l.path&&a.set(l.path,l);const u=new Map;for(const l of s.sheets??[])u.set(l.index,l);return{atlas:o,atlasMeta:s,cellByPath:a,sheetByIndex:u,diffBase:t}}async function ar(e){const t=Ii(e),r=vt();if(r&&!t)throw new Error("Encrypted assets enabled but no key provided");if($e&&(!r||nt===(t??null))){const i=await $e;return{...st(i),visemePaths:[...i.viseme.cellByPath.keys()]}}nt=t??null,Ot.clear();const n=mn();$e=(async()=>{const i=await Pi(t),o=await Gr(hn(),n,r,t);let a=null;try{a=await Gr(pn(),n,r,t)}catch{a=null}return{sil:i,viseme:o,expression:a}})();const s=await $e;return{...st(s),visemePaths:[...s.viseme.cellByPath.keys()]}}async function Ar(e){if(await ar(e),!$e)throw new Error("Assets not loaded");return $e}function st(e){return Pn(e.viseme.atlasMeta,e.sil.naturalWidth,e.sil.naturalHeight)}function xi(e,t){const r=document.createElement("canvas");return r.width=t.w,r.height=t.h,r.getContext("2d",{willReadFrequently:!0}).drawImage(e.atlas,t.x,t.y,t.w,t.h,0,0,t.w,t.h),r}async function Nt(e,t,r){const n=`${t.diffBase}::${r}`,s=Ot.get(n);if(s)return s;const i=(async()=>{const o=t.cellByPath.get(r);if(!o)throw new Error(`No atlas cell for path: ${r}`);const a=xn(t.atlasMeta,o),u=st(e);if(Rn(o)){const c=xi(t,o),d=document.createElement("canvas");d.width=u.width,d.height=u.height;const g=d.getContext("2d");return g.fillStyle="#000",g.fillRect(0,0,u.width,u.height),g.drawImage(c,a.x,a.y),d}const l=`${t.diffBase}/${r}`,f=await Er(l);if(f.naturalWidth===u.width&&f.naturalHeight===u.height)return f;throw new Error(`Diff PNG wrong size for ${r}`)})();return Ot.set(n,i),i}async function zr(e,t,r,n,s,i){if(s){const{width:g,height:v}=s;if(i&&!i()||t.readyState<HTMLMediaElement.HAVE_CURRENT_DATA)return;const m=document.createElement("canvas");m.width=g,m.height=v;const R=m.getContext("2d");if(R.clearRect(0,0,g,v),R.drawImage(t,0,0,g,v),i&&!i())return;(e.width!==g||e.height!==v)&&(e.width=g,e.height=v);const _=e.getContext("2d");_.clearRect(0,0,g,v),_.drawImage(m,0,0);return}const o=await Ar(),a=$t(o.viseme.atlasMeta),{width:u,height:l}=st(o);(e.width!==u||e.height!==l)&&(e.width=u,e.height=l);const f=[],c=e.getContext("2d");if(c.clearRect(0,0,u,l),t.readyState<HTMLMediaElement.HAVE_CURRENT_DATA){const g=o.sil;c.drawImage(g,0,0,u,l);return}if(f.length===0){c.drawImage(t,0,0,u,l);return}const d=kr(t,f,u,l,a);c.drawImage(d,0,0)}async function or(e,t,r){const n=await Ar(nt??void 0),s=$t(n.viseme.atlasMeta),{width:i,height:o}=st(n);(e.width!==i||e.height!==o)&&(e.width=i,e.height=o);const a=[];if(t.mouthPath){const f=new Set(n.viseme.cellByPath.keys()),c=Sr(t.mouthPath,f)??(f.has(t.mouthPath)?t.mouthPath:null);c&&a.push(await Nt(n,n.viseme,c))}t.expressionPath&&n.expression&&a.push(await Nt(n,n.expression,t.expressionPath));const u=e.getContext("2d");if(u.clearRect(0,0,i,o),a.length===0){u.drawImage(n.sil,0,0,i,o);return}const l=kr(n.sil,a,i,o,s);u.drawImage(l,0,0)}async function Ri(e,t){const r=await Ar(nt??void 0),n=$t(r.viseme.atlasMeta),{width:s,height:i}=st(r),o=[];if(e.mouthPath){const f=new Set(r.viseme.cellByPath.keys()),c=Sr(e.mouthPath,f)??(f.has(e.mouthPath)?e.mouthPath:null);c&&o.push(await Nt(r,r.viseme,c))}e.expressionPath&&r.expression&&o.push(await Nt(r,r.expression,e.expressionPath));const a=new OffscreenCanvas(s,i),u=a.getContext("2d");if(o.length===0)return u.drawImage(r.sil,0,0,s,i),a.transferToImageBitmap();const l=kr(r.sil,o,s,i,n);return u.drawImage(l,0,0),a.transferToImageBitmap()}async function Mi(e){await or(e,{mouthPath:null,expressionPath:null})}let Nn=null;function jr(e){Nn=e}function Ne(){return Nn}function Di(e){return`${e.from}→${e.to}@${e.fromVtime}`}function Yr(){return{rafTicksSpeaking:0,rafTicksDeduped:0,drawsRequested:0,drawsDisplayed:0,queueSuperseded:0,workerStale:0,rifeFramesOmittedByGap:0,transitionFramesSampled:0,transitionFramesDisplayed:0,transitionFramesMissed:0,bufferTicks:0,bufferHits:0,bufferUnderruns:0,bufferPrefetched:0}}function Ci(){let e=Yr();const t=new Set,r=new Set,n=new Set;let s="",i=-1,o="";const a=()=>{e=Yr(),t.clear(),r.clear(),n.clear(),s="",i=-1,o=""},u=f=>{t.add(f)},l=f=>{f&&n.add(f)};return{reset:a,onRafTickSpeaking(){e.rafTicksSpeaking+=1},onDrawDeduped(){e.rafTicksDeduped+=1},onDrawRequested(){e.drawsRequested+=1},onQueueSuperseded(){e.queueSuperseded+=1},onWorkerStale(){e.workerStale+=1},onTransitionSample(f,c){const d=Di(f);e.transitionFramesSampled+=1,d!==s&&(s=d,i=-1,o!==d&&(o=d,e.rifeFramesOmittedByGap+=f.omittedByGap)),c&&r.add(c),i>=0&&f.frameIdx>i+1&&(e.transitionFramesMissed+=f.frameIdx-i-1),i=f.frameIdx},onBufferTick(){e.bufferTicks+=1},onBufferHit(){e.bufferHits+=1},onBufferUnderrun(){e.bufferUnderruns+=1},onBufferPrefetched(){e.bufferPrefetched+=1},onBufferReset(){},onDrawDisplayed(f,c){if(e.drawsDisplayed+=1,f.transitionMeta){e.transitionFramesDisplayed+=1;const g=f.transitionMeta;u(g.fromVtime),u(g.toVtime),l(f.mouthPath);return}if(f.label.includes("→")){e.transitionFramesDisplayed+=1;return}const d=f.label.trim();if(!(!d||d==="sil")){for(const g of c)if(g.viseme===d&&f.elapsedMs>=g.vtime&&f.elapsedMs<g.vtime+g.vduration){u(g.vtime);break}}},snapshot(f,c){let d=0,g=0;for(const D of f)if(D.viseme!=="sil"){if(D.vtime>=c)break;d+=1,t.has(D.vtime)&&(g+=1)}const v=d>0?Math.round(g/d*100):100,m=e.queueSuperseded+e.workerStale+Math.max(0,e.drawsRequested-e.drawsDisplayed),R=d-g,_=e.bufferTicks>0?Math.round(e.bufferHits/e.bufferTicks*100):100;return{...e,bufferHitRate:_,renderLoss:m,visemeSlotsTotal:d,visemeSlotsCovered:g,visemeCoveragePct:v,transitionFramesUniqueSampled:r.size,transitionFramesUniquePainted:n.size,drawsDeduped:e.rafTicksDeduped,renderSkips:e.rafTicksDeduped+m,visemesShown:g,visemesSkipped:R,rifeTransitionFramesSkipped:e.transitionFramesMissed}}}}const Un=fr,Bi=200,Fi=100,Li=600,Vn="__sil__";function Xr(e){return e.lastDrawnKey?e.lastDrawnKey.split("|")[0]===Vn:!1}function Ir(){return{lastDrawnKey:""}}function Ae(e){e.lastDrawnKey="",e.transitionCache=void 0}function Oi(e,t){if(t?.transitionCache&&t.transitionCache.queueLength===e.length)return t.transitionCache.transitions;const r=Ms(e);return t&&(t.transitionCache={queueLength:e.length,transitions:r}),r}function Ni(e,t){let r="sil";for(const n of e)n.viseme!=="sil"&&(t!=null&&n.vtime>t||(r=n.viseme));return r}function Ui(e,t,r,n=120){if(r!=="sil")return r;const s=En(e,t,n);return s!=="sil"?s:Ni(e,t)}function Vi(e,t){return e==="sil"?!1:t?ti(e,"sil",t):vr(e,"sil").length>0}function Wi(){return Un+Li}function Ut(e){return e??Vn}function Pr(e,t,r,n,s=Fi){if(e.length===0)return{diffPath:null,label:"sil"};if(t>=He(e))return{diffPath:null,label:"sil"};const i=Oi(e,n),o=Ds(i,t);if(o){const u=bn(o.fromVtime,o.toVtime,o.from,o.to);if(t>=u.transStart&&t<u.transEnd){let l=vr(o.from,o.to);r&&(l=Cn(l,r));const f=ni(l,u.effectiveGapMs);if(f.length>0){const c=si(t,u.transStart,u.transEnd,f.length);return{diffPath:f[c]??f[f.length-1],label:`${o.from}→${o.to}`,transitionMeta:{from:o.from,to:o.to,fromVtime:o.fromVtime,toVtime:o.toVtime,frameIdx:c,frameCount:f.length,fullFrameCount:l.length,omittedByGap:Math.max(0,l.length-f.length)}}}return{diffPath:_t(o.to),label:o.to}}}const a=En(e,t,s);return a==="sil"?{diffPath:null,label:"sil"}:{diffPath:_t(a),label:a}}function xr(e,t){const r=t??Ut(e.expressionPath);return`${Ut(e.mouthPath)}|${r}`}function Jr(e,t,r,n,s,i,o,a,u,l){const f=Pr(t,r,u,i),c={mouthPath:f.diffPath,expressionPath:n},d=xr(c,s),g=Ne();if(f.transitionMeta&&g?.onTransitionSample(f.transitionMeta,f.diffPath),d===i.lastDrawnKey){g?.onDrawDeduped();return}g?.onDrawRequested(),a(f),i.lastDrawnKey=d,o(e,c,d,v=>{v===d&&(i.lastDrawnKey=d)})}function Qr(e,t,r,n,s,i,o=null){const a={mouthPath:null,expressionPath:t},u=o!=null?`|idle@${Math.round(o*1e3)}`:"",l=xr(a,r)+u;!(o!=null)&&l===n.lastDrawnKey||(i(o!=null?"idle":"sil"),s(e,a,l,c=>{c===l&&(n.lastDrawnKey=l)}))}function Wn(e,t,r,n,s=yt,i){if(r<=t||e.length===0)return[];const o=i??Ir(),a=[];let u="";for(let l=t;l<r;l+=s){const f=Pr(e,l,n,o),c=Ut(f.diffPath);c!==u&&(u=c,a.push({displayAtMs:l,mouthPath:f.diffPath,mouthKey:c,label:f.label}))}return a}const Hi=3;function Rr(e){let t=0;const r=[];let n=!1,s=!1;const i=async()=>{if(!n){n=!0;try{for(;r.length>0;){const a=r.shift(),u=++t;await e(a.diffs,u,a)&&u===t&&a.onDisplayed?.(a.drawKey)}}finally{n=!1,r.length>0&&i()}}},o=()=>{s||(s=!0,queueMicrotask(()=>{s=!1,i()}))};return{enqueue(a,u,l){const f=r.findIndex(c=>c.drawKey===u);f>=0?r[f]={diffs:a,drawKey:u,onDisplayed:l}:(r.length>=Hi&&(r.shift(),Ne()?.onQueueSuperseded()),r.push({diffs:a,drawKey:u,onDisplayed:l})),o()},reset(){t+=1,r.length=0},isLatestSeq(a){return a===t},getLatestSeq(){return t}}}function $i(e,t=2){const r=[],n=new Set;let s=0,i=!1,o=0;const a=(l,f)=>`${l}|${f}`,u=async()=>{if(!i){i=!0;try{for(;s<t&&r.length>0;){const l=r.shift(),f=a(l.displayAtMs,l.drawKey);if(n.has(f))continue;n.add(f),s+=1;const c=o;try{await e(l)}finally{if(n.delete(f),s-=1,c!==o)return}}}finally{i=!1,r.length>0&&s<t&&u()}}};return{enqueue(l){const f=a(l.displayAtMs,l.drawKey);n.has(f)||r.some(c=>a(c.displayAtMs,c.drawKey)===f)||(r.push(l),u())},isQueuedOrInflight(l,f){const c=a(l,f);return n.has(c)?!0:r.some(d=>a(d.displayAtMs,d.drawKey)===c)},reset(){o+=1,r.length=0,n.clear(),s=0},pendingCount(){return r.length+s}}}function qi(){const e=new Map,t=r=>{r.bitmap.close()};return{has(r,n){return e.has(n)},markRendered(r,n,s,i,o){const a=e.get(n);a&&t(a),e.set(n,{displayAtMs:r,mouthKey:n,label:s,mouthPath:i,bitmap:o})},findForMouthKey(r){return e.get(r)??null},removeThrough(r){},pruneBefore(r){},reset(){for(const r of e.values())t(r);e.clear()},size(){return e.size}}}function Ki(e,t,r,n,s,i){const o=Wn(t,r,n,s,yt,i);if(o.length===0)return{ready:0,total:0};let a=0;for(const u of o)e.has(u.displayAtMs,u.mouthKey)&&(a+=1);return{ready:a,total:o.length}}function Gi(e){const t=qi(),r=Ir(),n=new Set;let s="",i=null;const o=$i(async c=>{try{if(!e.supportsBufferedPlayback)return;const d=await e.compositePrefetchFrame(c.diffs,c.mouthKey);if(!d)return;t.markRendered(c.displayAtMs,c.mouthKey,c.label,c.mouthPath,d),Ne()?.onBufferPrefetched()}finally{n.delete(c.mouthKey)}},ns),a=c=>{for(const d of c)t.has(d.displayAtMs,d.mouthKey)||n.has(d.mouthKey)||o.isQueuedOrInflight(0,d.mouthKey)||(n.add(d.mouthKey),o.enqueue({...d,displayAtMs:0,drawKey:d.mouthKey,diffs:{mouthPath:d.mouthPath,expressionPath:null}}))},u=(c,d,g,v=Xt)=>{if(!e.supportsBufferedPlayback)return;const m=Math.max(0,d-yt*4),R=Wn(c,m,d+v,g,yt,r);a(R)},l=(c,d)=>{s=c,i=d},f=(c,d)=>!(s!==c||i!==d);return{reset(){t.reset(),o.reset(),n.clear(),Ae(r),s="",i=null,Ne()?.onBufferReset()},markExternallyDisplayed(c,d){l(c.split("|")[0]??c,d)},warmupTick(c){e.supportsBufferedPlayback&&u(c.queue,0,c.availablePaths,Math.max(Fr,Xt))},isWarmupReady(c,d=Fr){if(!e.supportsBufferedPlayback)return!0;const{ready:g,total:v}=Ki(t,c.queue,0,d,c.availablePaths,r);return v===0?!1:g/v>=rs},getBufferedFrameCount:()=>t.size(),tick(c){const d=Ne();d?.onBufferTick(),u(c.queue,c.elapsedMs,c.availablePaths),t.pruneBefore(c.elapsedMs-Xt);const g=Pr(c.queue,c.elapsedMs,c.availablePaths,r),v=Ut(g.diffPath),m=xr({mouthPath:g.diffPath,expressionPath:c.expressionPath},c.expressionDrawKey),R=()=>(l(v,c.expressionPath),d?.onBufferHit(),{usedBuffer:!0,underrun:!1,held:!0});if(f(v,c.expressionPath)||c.playbackState.lastDrawnKey===m)return R();const _=t.findForMouthKey(v,c.elapsedMs,yt);return _&&c.expressionPath!=null?(d?.onBufferHit(),{usedBuffer:!1,underrun:!1,held:!1}):_?(e.blitBitmap(c.canvas,_.bitmap),l(_.mouthKey,c.expressionPath),c.playbackState.lastDrawnKey=m,c.onStatus({diffPath:_.mouthPath,label:_.label}),c.onDisplayed(m,_.mouthPath,_.label),d?.onBufferHit(),{usedBuffer:!0,underrun:!1,held:!1}):(d?.onBufferUnderrun(),{usedBuffer:!1,underrun:!0,held:!1})}}}const zi=`
|
|
5
|
+
`);Oe=we.pop()??"";for(const Te of we)await ut(Te);if(Je){Oe.trim()&&await ut(Oe.trim()),pe===G&&!ae.signal.aborted&&w&&N.markStreamComplete(r);break}}}catch(Se){if(ae.signal.aborted||!w)return;console.error("[realtimeVoice] avatar_speak failed:",Se),e("error"),C()}finally{B===ae&&(B=null)}},Q=p=>{const x=Ks(p);if(!x){console.warn("[realtimeVoice] server message with no payload",p);return}const j=String(x.type??"unknown");switch(j!=="avatar_metadata"&&j!=="avatar_audio_chunk"&&j!=="avatar_lipsync"&&j!=="avatar_interrupt"&&j!=="error"&&console.debug("[realtimeVoice] server message",j,x),j){case"avatar_metadata":M(x);break;case"avatar_audio_chunk":I(x);break;case"avatar_lipsync":$(x);break;case"avatar_interrupt":fe(x);break;case"error":{const ae=String(x.message??"Voice session error");console.error("[realtimeVoice] server error:",ae),e("error"),C();break}case"avatar_speak":Y(x);break}},Z=()=>f||(c=new Ws,f=new zn.PipecatClient({transport:new ur.WebSocketTransport({mediaManager:c,serializer:new Vs,recorderSampleRate:gr,playerSampleRate:yr}),enableCam:!1,enableMic:!0,callbacks:{onBotReady:()=>{l=!0,e("idle"),C()},onDisconnected:()=>{l=!1,h()},onError:p=>{const x=typeof p=="object"&&p!==null&&"data"in p?String(p.data??p):String(p);console.error("[realtimeVoice]",x,p),e("error"),C()},onTransportStateChanged:p=>{p==="error"&&(e("error"),C())},onServerMessage:p=>{X(p)},onUserStartedSpeaking:()=>{fe({})},onBotStartedSpeaking:()=>{ee(),se("bot_started")},onBotStoppedSpeaking:()=>{w&&(P(),k=setTimeout(()=>{k=null,w&&(N.markStreamComplete(r),se("bot_stopped"))},450))},onUserTranscript:p=>{d.onUserTranscript?.(p.text,!!p.final)},onBotOutput:p=>{d.onBotOutput?.(p.text)}}}),f);return{setDeveloperToken:p=>{},setTtsEngineId:p=>{u=p,N.setStreamTtsEngineId(p)},getTtsEngineId:()=>u,speak:async(p,x)=>{const j=p.trim();if(j){if(!f||!l)throw new Error("Realtime voice is not connected. Call connect() first.");E(),await f.sendText(j)}},stop:()=>{B?.abort(),B=null,G+=1,ee(),E(),e("idle"),C()},connect:async p=>{if(p&&(d={...d,...p}),!d.authToken?.trim())throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");e("loading");const x=Z();await x.initDevices();const j=An(d);console.info("[realtimeVoice] connecting (avatar_audio_chunk RTVI):",j);try{await x.connect({wsUrl:j})}catch(ae){l=!1,e("error"),C();const pe=ae instanceof Error?ae.message:"Realtime voice connect failed (WebSocket 101 only means upgrade OK — check authToken/origin/RTVI handshake)";throw new Error(`${pe} — ${j}`)}l=!0,e("idle"),C()},disconnect:async()=>{if(P(),F=!1,R.length=0,_=!1,B?.abort(),B=null,G+=1,ee(),f)try{await f.disconnect()}catch(p){console.error("[realtimeVoice] disconnect failed:",p)}l=!1,h()},isConnected:()=>l,unlockAudio:async()=>{await N.primeAudioContext()},getVisemeQueue:()=>r,getWordQueue:()=>n,getSentenceEmotions:()=>s,getStreamMood:()=>a,getPlaybackElapsedMs:z,isSpeaking:()=>N.isSpeaking(),isWarmingUp:()=>N.isWarmingUp(),releaseWarmupHold:()=>N.releaseWarmupHold(),setLipsyncWarmupHold:(p,x)=>{x&&(V=x),N.setLipsyncWarmupHoldEnabled(p)},getVoiceStream:()=>N.getVoiceStream(),setPlaybackSpeed:p=>{N.setPlaybackSpeed(p)},getPlaybackSpeed:()=>N.getPlaybackSpeed(),subscribe:p=>(m.add(p),()=>m.delete(p))}}function Pn(e,t,r){return{width:e.crop?.source_width??t,height:e.crop?.source_height??r}}function xn(e,t){const r=e.crop;return{x:t?.source_x??r?.x??0,y:t?.source_y??r?.y??0,width:t?.source_w??r?.width??e.frame_width??0,height:t?.source_h??r?.height??e.frame_height??0}}function Rn(e){return e?(e.w??0)>0&&(e.h??0)>0:!1}async function it(e,t){const r=await fetch(e);if(!r.ok)throw new Error(`Failed to load encrypted asset: ${e}`);const n=await r.arrayBuffer();return ps(n,t)}async function Mn(e,t){if(!t)throw new Error("Missing asset key for encrypted JSON");const r=await it(e,t),n=new TextDecoder().decode(r);return JSON.parse(n)}async function zs(e){const t=new Blob([e],{type:"image/webp"});return createImageBitmap(t)}async function js(e){const t=new Blob([e],{type:"image/webp"}),r=URL.createObjectURL(t);return new Promise((n,s)=>{const i=new Image;i.decoding="async",i.onload=()=>{URL.revokeObjectURL(r),n(i)},i.onerror=()=>{URL.revokeObjectURL(r),s(new Error("Failed to decode decrypted image"))},i.src=r})}async function Ys(e){const t=new Blob([e],{type:"image/png"});return createImageBitmap(t)}async function Xs(e){const t=new Blob([e],{type:"image/png"}),r=URL.createObjectURL(t);return new Promise((n,s)=>{const i=new Image;i.decoding="async",i.onload=()=>{URL.revokeObjectURL(r),n(i)},i.onerror=()=>{URL.revokeObjectURL(r),s(new Error("Failed to decode decrypted PNG"))},i.src=r})}function Js(e){const t=new Blob([e],{type:"video/mp4"});return URL.createObjectURL(t)}async function Qs(e){const t=gs();if(!vt())return t;if(!e)throw new Error("Missing asset key for encrypted idle video");const r=await it(t,e);return Js(r)}function _r(e,t){return`${e}/${t}_minus_sil.png`}const Dn=5;function $t(e){const t=e?.composite_threshold;if(typeof t=="number"&&t>0)return t;const r=e?.crop?.content_threshold;return typeof r=="number"&&r>0?r:Dn}const Zs=["aa","E","I","O","U","PP","FF","DD","SS","TH","CH","RR","kk","nn"],sr=["t01_0.25","t02_0.50","t03_0.75"],ei=sr.length,Vr=new Set(["aa__kk","aa__nn","aa__sil","CH__aa","CH__DD","CH__E","CH__FF","CH__I","CH__kk","CH__nn","CH__O","CH__PP","CH__RR","CH__sil","CH__SS","CH__TH","CH__U","DD__aa","DD__E","DD__FF","DD__I","DD__kk","DD__nn","DD__O","DD__PP","DD__RR","DD__sil","DD__SS","DD__TH","DD__U","E__aa","E__FF","E__I","E__kk","E__nn","E__O","E__PP","E__RR","E__sil","E__SS","E__TH","E__U","FF__aa","FF__I","FF__kk","FF__nn","FF__O","FF__PP","FF__RR","FF__sil","FF__SS","FF__TH","FF__U","I__aa","I__kk","I__nn","I__O","I__PP","I__RR","I__sil","I__SS","I__TH","I__U","kk__nn","kk__sil","nn__sil","O__aa","O__kk","O__nn","O__PP","O__RR","O__sil","O__SS","O__TH","O__U","PP__aa","PP__kk","PP__nn","PP__RR","PP__sil","PP__SS","PP__TH","PP__U","RR__aa","RR__kk","RR__nn","RR__sil","RR__SS","RR__TH","RR__U","SS__aa","SS__kk","SS__nn","SS__sil","SS__TH","SS__U","TH__aa","TH__kk","TH__nn","TH__sil","TH__U","U__aa","U__kk","U__nn","U__sil"]);function Wr(e,t){return`${e}__${t}`}function Hr(e,t){return`pairs/${e}/${t}_minus_sil.png`}function _t(e){return`${e}_minus_sil.png`}function Sr(e,t){if(t.has(e))return e;const r=/^pairs\/([^/]+)\//.exec(e);if(!r)return null;const n=r[1].split("__")[1];if(!n||n==="sil")return null;const s=_t(n);return t.has(s)?s:null}function Cn(e,t){return e.filter(r=>t.has(r))}function ti(e,t,r){const n=vr(e,t);return Cn(n,r).length>0}function vr(e,t){if(e===t)return[];const r=[],n=Wr(e,t);if(Vr.has(n)){for(const i of sr)r.push(Hr(n,i));return r}const s=Wr(t,e);if(Vr.has(s)){for(let i=ei-1;i>=0;i--)r.push(Hr(s,sr[i]));return r}return[]}function ri(e){return e<16?[1]:e<24?[1]:e<35?[0,1,2]:[0,1,2]}function ni(e,t){return ri(t).map(n=>e[n]).filter(n=>n!=null)}function si(e,t,r,n){if(n<=0)return 0;const i=Math.max(1,r-t)/n,o=e-t,a=Math.floor(o/i);return Math.min(n-1,Math.max(0,a))}function Bn(e,t,r){if(!r)return`${e}/${t}`;const n=t.split("/").pop()??t,s=n.includes(".")?n.slice(0,n.lastIndexOf(".")):n;return`${e}/${s}.bin`}const ii=[.32,.24,.16,.08,0],ai=ii.map(e=>`eyes_${e.toFixed(2)}`),oi=["eyes_0.32","eyes_0.16","eyes_0.00"],ci=["eyes_0.32","eyes_0.16","eyes_0.08","eyes_0.00"],$r=2,ui=-30,ir=20,li={fear:-30,anger:-20,sad:-10,neutral:0,love:10,happy:0,disgust:20};function br(e){return`brow_${e}`}function Tr(e){return li[e]}function Ke(e){return Math.max(ui,Math.min(ir,e))}function fi(e,t){if(e===t)return[];const r=e<t?$r:-$r,n=[];let s=e;for(;s!==t;)s+=r,n.push(s);return n}const di={fear:"fear",afraid:"fear",anxiety:"fear",anger:"anger",angry:"anger",mad:"anger",sad:"sad",sadness:"sad",sorrow:"sad",neutral:"neutral",calm:"neutral",love:"love",loving:"love",affection:"love",happy:"happy",happiness:"happy",joy:"happy",joyful:"happy",excited:"happy",disgust:"disgust",disgusted:"disgust"};function tt(e,t="neutral"){const r=String(e??"").trim().toLowerCase();return r?di[r]??t:t}function Fn(e){return e===0?null:_r(br(e),"eyes_standard")}function Ln(e){return Fn(Tr(e))}function qr(e,t){const r=br(Tr(e));return!t||t==="eyes_standard"?Ln(e):e==="neutral"&&t==="eyes_standard"?null:_r(r,t)}function mi(e,t){const r=[];for(const n of t)r.push(qr(e,n));for(let n=t.length-2;n>=0;n--)r.push(qr(e,t[n]));return r.push(Ln(e)),r}function hi(e){return _r(br(e),"eyes_standard")}function pi(e,t){return fi(e,t).map(hi)}const wi=6;function gi(e,t){for(const r of e)if(t>=r.wtime&&t<r.wtime+r.wduration)return r;return null}function yi(e){const t=Ke(e);return t>=ir?ir:Ke(t+wi)}function _i(e,t,r){const n=Ke(r);return gi(t,e)?.emphasis?yi(n):n}function Kr(e){if(!e)return 0;const t=e.match(/^brow_(-?\d+)\//);if(!t?.[1])return null;const r=Number.parseInt(t[1],10);return Number.isFinite(r)?Ke(r):null}function gt(e,t){return e+Math.random()*(t-e)}function Mt(){return gt(14,32)}function Si(e={}){const{initialDelayMs:t=800+Math.random()*1200,intervalMinMs:r=2200,intervalMaxMs:n=5800,doubleBlinkChance:s=.18,doubleBlinkGapMinMs:i=120,doubleBlinkGapMaxMs:o=220,fullBlinkChance:a=.75,postEmotionChangeDelayMs:u=380,minDwellMs:l=550}=e;let f="neutral",c="neutral",d=0,g=0,b=0,m={kind:"idle",nextBlinkAt:performance.now()+t},R=null,_="__eyes_open__|neutral|brow:0";function D(k=f){_=`${R??"__eyes_open__"}|${k}|brow:${g}`}function w(k,P=f){R=k;const C=Kr(k);C!=null?g=C:k==null&&(g=0),D(P)}function T(k){return Fn(k)}function O(k,P,C){const V=Ke(P),N=Ke(C);if(d=N,V===N){w(T(N));return}const he=[...pi(V,N),T(N)];m={kind:"browBlend",paths:he,index:0,holdUntil:k+Mt(),targetBrow:N},w(he[0]??null)}function B(k,P){m={kind:"idle",nextBlinkAt:k+gt(r,n)},c=P,w(T(d),P)}function G(k,P,C){const V=C?oi:Math.random()<a?ai:ci;m={kind:"playing",paths:mi(P,V),index:0,holdUntil:k+Mt(),after:"idle",blinkEmotion:P},c=P,w(m.paths[0]??null,P)}function F(k){if(m.kind==="browBlend"){for(;k>=m.holdUntil&&m.index<m.paths.length-1;)m.index+=1,m.holdUntil=k+Mt();const P=m.paths[m.index]??null;w(P),k>=m.holdUntil&&m.index>=m.paths.length-1&&(g=m.targetBrow,d=m.targetBrow,w(T(m.targetBrow)),m={kind:"idle",nextBlinkAt:k+u});return}if(m.kind==="idle"){w(T(d),c),k>=m.nextBlinkAt&&G(k,c,!1);return}if(m.kind==="doublePause"){w(T(d),m.blinkEmotion),k>=m.resumeAt&&G(k,m.blinkEmotion,!0);return}if(m.kind==="playing"){for(;k>=m.holdUntil&&m.index<m.paths.length-1;)m.index+=1,m.holdUntil=k+Mt();if(w(m.paths[m.index]??null,m.blinkEmotion),k>=m.holdUntil&&m.index>=m.paths.length-1){if(m.after==="resumeIdle"){B(k+u,f);return}if(m.after==="doublePause"){m={kind:"doublePause",resumeAt:k+gt(i,o),blinkEmotion:m.blinkEmotion},w(T(d),m.blinkEmotion);return}if(m.after==="idle"&&Math.random()<s){m={kind:"doublePause",resumeAt:k+gt(i,o),blinkEmotion:m.blinkEmotion},w(T(d),m.blinkEmotion);return}B(k,m.blinkEmotion)}}}return{advance:F,setTargetEmotion(k){const P=tt(k,f);if(P===f)return;const C=performance.now();l>0&&P!==f&&!(f==="neutral"&&P!=="neutral")&&C-b<l||(f=P,c=P,b=C)},setTargetEyebrow(k){const P=Ke(k);if(P===d)return;const C=performance.now(),V=m.kind==="browBlend"?Kr(R)??g:g;d=P,!(P===V&&m.kind!=="browBlend")&&(m.kind==="playing"||m.kind==="doublePause"||O(C,V,P))},getTargetEmotion:()=>f,getTargetEyebrow:()=>d,getExpressionPath:()=>R,getDrawKey:()=>_,reset(){f="neutral",c="neutral",d=0,g=0,b=0,B(performance.now()+gt(r,n),"neutral")}}}function vi(){const e=document.createElement("video");e.muted=!0,e.playsInline=!0,e.preload="auto",e.setAttribute("playsinline",""),e.loop=!0;let t=!1,r=null,n="";const s=()=>{r&&(URL.revokeObjectURL(r),r=null)};return{async load(i){t&&n===i||(s(),t=!1,n=i,i.startsWith("blob:")&&(r=i),await new Promise((o,a)=>{const u=()=>{const l=e.duration;if(!Number.isFinite(l)||l<=0){a(new Error(`Idle video has invalid duration: ${i}`));return}t=!0,e.currentTime=0,o()};e.addEventListener("loadeddata",u,{once:!0}),e.addEventListener("error",()=>a(new Error(`Failed to load idle video: ${i}`)),{once:!0}),e.src=i,e.load()}))},isReady(){return t},getVideo(){return e},setActive(i){t&&(i?e.paused&&e.play().catch(()=>{}):e.paused||e.pause())},restart(){t&&(e.currentTime=0)},dispose(){e.pause(),e.removeAttribute("src"),e.load(),s(),t=!1,n=""}}}const bi=50,On=400;function Ti(e,t,r){return e.filter(n=>{const s=n.queueIndex??-1;return s>=t&&s<=r})}function ki(e,t){const r=[];for(const n of e){const s=Ti(t,n.start_word,n.end_word);if(s.length===0)continue;const i=s[0],o=s[s.length-1];r.push({sentence:n,startMs:Math.max(0,i.wtime-bi),endMs:o.wtime+o.wduration+On})}return r.sort((n,s)=>n.startMs-s.startMs)}function Ei(e,t,r,n="neutral"){const s=tt(n);if(t.length===0)return s;if(r.length===0||e<r[0].wtime)return tt(t[0]?.emotion,s);const i=ki(t,r);let o=null;for(const l of i)e>=l.startMs&&e<=l.endMs&&(o=l);if(o)return tt(o.sentence.emotion,s);const a=r[r.length-1],u=a.wtime+a.wduration;return e>u+On?tt(t[t.length-1]?.emotion,s):s}function kr(e,t,r,n,s=Dn){const i=new OffscreenCanvas(r,n),o=i.getContext("2d",{willReadFrequently:!0});o.drawImage(e,0,0,r,n);const a=o.getImageData(0,0,r,n),u=o.createImageData(r,n);u.data.set(a.data);const f=new OffscreenCanvas(r,n).getContext("2d",{willReadFrequently:!0});for(const c of t){f.clearRect(0,0,r,n),f.drawImage(c,0,0,r,n);const d=f.getImageData(0,0,r,n);for(let g=0;g<u.data.length;g+=4){const b=d.data[g],m=d.data[g+1],R=d.data[g+2];Math.max(b,m,R)>s&&(u.data[g]=b,u.data[g+1]=m,u.data[g+2]=R,u.data[g+3]=255)}}return o.putImageData(u,0,0),i}let $e=null,nt=null;const Ot=new Map;function Ai(){$e=null,nt=null,Ot.clear()}dn(Ai);function Ii(e){return e??nt??void 0}function Er(e){return new Promise((t,r)=>{const n=new Image;n.decoding="async",n.onload=()=>t(n),n.onerror=()=>r(new Error(`Failed to load image: ${e}`)),n.src=e})}async function Pi(e){return vt()?Xs(await it(Lt(),e??"")):Er(Lt())}async function Gr(e,t,r,n){const s=r?await Mn(e,n):await fetch(e).then(l=>{if(!l.ok)throw new Error(`Failed to load ${e}`);return l.json()}),i=s.sheets?.[0];if(!i?.path)throw new Error(`${e} has no sheets[0].path`);const o=r?await js(await it(Bn(t,i.path,r),n??"")):await Er(`${t}/${i.path}`),a=new Map;for(const l of s.cells??[])l.path&&a.set(l.path,l);const u=new Map;for(const l of s.sheets??[])u.set(l.index,l);return{atlas:o,atlasMeta:s,cellByPath:a,sheetByIndex:u,diffBase:t}}async function ar(e){const t=Ii(e),r=vt();if(r&&!t)throw new Error("Encrypted assets enabled but no key provided");if($e&&(!r||nt===(t??null))){const i=await $e;return{...st(i),visemePaths:[...i.viseme.cellByPath.keys()]}}nt=t??null,Ot.clear();const n=mn();$e=(async()=>{const i=await Pi(t),o=await Gr(hn(),n,r,t);let a=null;try{a=await Gr(pn(),n,r,t)}catch{a=null}return{sil:i,viseme:o,expression:a}})();const s=await $e;return{...st(s),visemePaths:[...s.viseme.cellByPath.keys()]}}async function Ar(e){if(await ar(e),!$e)throw new Error("Assets not loaded");return $e}function st(e){return Pn(e.viseme.atlasMeta,e.sil.naturalWidth,e.sil.naturalHeight)}function xi(e,t){const r=document.createElement("canvas");return r.width=t.w,r.height=t.h,r.getContext("2d",{willReadFrequently:!0}).drawImage(e.atlas,t.x,t.y,t.w,t.h,0,0,t.w,t.h),r}async function Nt(e,t,r){const n=`${t.diffBase}::${r}`,s=Ot.get(n);if(s)return s;const i=(async()=>{const o=t.cellByPath.get(r);if(!o)throw new Error(`No atlas cell for path: ${r}`);const a=xn(t.atlasMeta,o),u=st(e);if(Rn(o)){const c=xi(t,o),d=document.createElement("canvas");d.width=u.width,d.height=u.height;const g=d.getContext("2d");return g.fillStyle="#000",g.fillRect(0,0,u.width,u.height),g.drawImage(c,a.x,a.y),d}const l=`${t.diffBase}/${r}`,f=await Er(l);if(f.naturalWidth===u.width&&f.naturalHeight===u.height)return f;throw new Error(`Diff PNG wrong size for ${r}`)})();return Ot.set(n,i),i}async function zr(e,t,r,n,s,i){if(s){const{width:g,height:b}=s;if(i&&!i()||t.readyState<HTMLMediaElement.HAVE_CURRENT_DATA)return;const m=document.createElement("canvas");m.width=g,m.height=b;const R=m.getContext("2d");if(R.clearRect(0,0,g,b),R.drawImage(t,0,0,g,b),i&&!i())return;(e.width!==g||e.height!==b)&&(e.width=g,e.height=b);const _=e.getContext("2d");_.clearRect(0,0,g,b),_.drawImage(m,0,0);return}const o=await Ar(),a=$t(o.viseme.atlasMeta),{width:u,height:l}=st(o);(e.width!==u||e.height!==l)&&(e.width=u,e.height=l);const f=[],c=e.getContext("2d");if(c.clearRect(0,0,u,l),t.readyState<HTMLMediaElement.HAVE_CURRENT_DATA){const g=o.sil;c.drawImage(g,0,0,u,l);return}if(f.length===0){c.drawImage(t,0,0,u,l);return}const d=kr(t,f,u,l,a);c.drawImage(d,0,0)}async function or(e,t,r){const n=await Ar(nt??void 0),s=$t(n.viseme.atlasMeta),{width:i,height:o}=st(n);(e.width!==i||e.height!==o)&&(e.width=i,e.height=o);const a=[];if(t.mouthPath){const f=new Set(n.viseme.cellByPath.keys()),c=Sr(t.mouthPath,f)??(f.has(t.mouthPath)?t.mouthPath:null);c&&a.push(await Nt(n,n.viseme,c))}t.expressionPath&&n.expression&&a.push(await Nt(n,n.expression,t.expressionPath));const u=e.getContext("2d");if(u.clearRect(0,0,i,o),a.length===0){u.drawImage(n.sil,0,0,i,o);return}const l=kr(n.sil,a,i,o,s);u.drawImage(l,0,0)}async function Ri(e,t){const r=await Ar(nt??void 0),n=$t(r.viseme.atlasMeta),{width:s,height:i}=st(r),o=[];if(e.mouthPath){const f=new Set(r.viseme.cellByPath.keys()),c=Sr(e.mouthPath,f)??(f.has(e.mouthPath)?e.mouthPath:null);c&&o.push(await Nt(r,r.viseme,c))}e.expressionPath&&r.expression&&o.push(await Nt(r,r.expression,e.expressionPath));const a=new OffscreenCanvas(s,i),u=a.getContext("2d");if(o.length===0)return u.drawImage(r.sil,0,0,s,i),a.transferToImageBitmap();const l=kr(r.sil,o,s,i,n);return u.drawImage(l,0,0),a.transferToImageBitmap()}async function Mi(e){await or(e,{mouthPath:null,expressionPath:null})}let Nn=null;function jr(e){Nn=e}function Ne(){return Nn}function Di(e){return`${e.from}→${e.to}@${e.fromVtime}`}function Yr(){return{rafTicksSpeaking:0,rafTicksDeduped:0,drawsRequested:0,drawsDisplayed:0,queueSuperseded:0,workerStale:0,rifeFramesOmittedByGap:0,transitionFramesSampled:0,transitionFramesDisplayed:0,transitionFramesMissed:0,bufferTicks:0,bufferHits:0,bufferUnderruns:0,bufferPrefetched:0}}function Ci(){let e=Yr();const t=new Set,r=new Set,n=new Set;let s="",i=-1,o="";const a=()=>{e=Yr(),t.clear(),r.clear(),n.clear(),s="",i=-1,o=""},u=f=>{t.add(f)},l=f=>{f&&n.add(f)};return{reset:a,onRafTickSpeaking(){e.rafTicksSpeaking+=1},onDrawDeduped(){e.rafTicksDeduped+=1},onDrawRequested(){e.drawsRequested+=1},onQueueSuperseded(){e.queueSuperseded+=1},onWorkerStale(){e.workerStale+=1},onTransitionSample(f,c){const d=Di(f);e.transitionFramesSampled+=1,d!==s&&(s=d,i=-1,o!==d&&(o=d,e.rifeFramesOmittedByGap+=f.omittedByGap)),c&&r.add(c),i>=0&&f.frameIdx>i+1&&(e.transitionFramesMissed+=f.frameIdx-i-1),i=f.frameIdx},onBufferTick(){e.bufferTicks+=1},onBufferHit(){e.bufferHits+=1},onBufferUnderrun(){e.bufferUnderruns+=1},onBufferPrefetched(){e.bufferPrefetched+=1},onBufferReset(){},onDrawDisplayed(f,c){if(e.drawsDisplayed+=1,f.transitionMeta){e.transitionFramesDisplayed+=1;const g=f.transitionMeta;u(g.fromVtime),u(g.toVtime),l(f.mouthPath);return}if(f.label.includes("→")){e.transitionFramesDisplayed+=1;return}const d=f.label.trim();if(!(!d||d==="sil")){for(const g of c)if(g.viseme===d&&f.elapsedMs>=g.vtime&&f.elapsedMs<g.vtime+g.vduration){u(g.vtime);break}}},snapshot(f,c){let d=0,g=0;for(const D of f)if(D.viseme!=="sil"){if(D.vtime>=c)break;d+=1,t.has(D.vtime)&&(g+=1)}const b=d>0?Math.round(g/d*100):100,m=e.queueSuperseded+e.workerStale+Math.max(0,e.drawsRequested-e.drawsDisplayed),R=d-g,_=e.bufferTicks>0?Math.round(e.bufferHits/e.bufferTicks*100):100;return{...e,bufferHitRate:_,renderLoss:m,visemeSlotsTotal:d,visemeSlotsCovered:g,visemeCoveragePct:b,transitionFramesUniqueSampled:r.size,transitionFramesUniquePainted:n.size,drawsDeduped:e.rafTicksDeduped,renderSkips:e.rafTicksDeduped+m,visemesShown:g,visemesSkipped:R,rifeTransitionFramesSkipped:e.transitionFramesMissed}}}}const Un=fr,Bi=200,Fi=100,Li=600,Vn="__sil__";function Xr(e){return e.lastDrawnKey?e.lastDrawnKey.split("|")[0]===Vn:!1}function Ir(){return{lastDrawnKey:""}}function Ae(e){e.lastDrawnKey="",e.transitionCache=void 0}function Oi(e,t){if(t?.transitionCache&&t.transitionCache.queueLength===e.length)return t.transitionCache.transitions;const r=Ms(e);return t&&(t.transitionCache={queueLength:e.length,transitions:r}),r}function Ni(e,t){let r="sil";for(const n of e)n.viseme!=="sil"&&(t!=null&&n.vtime>t||(r=n.viseme));return r}function Ui(e,t,r,n=120){if(r!=="sil")return r;const s=En(e,t,n);return s!=="sil"?s:Ni(e,t)}function Vi(e,t){return e==="sil"?!1:t?ti(e,"sil",t):vr(e,"sil").length>0}function Wi(){return Un+Li}function Ut(e){return e??Vn}function Pr(e,t,r,n,s=Fi){if(e.length===0)return{diffPath:null,label:"sil"};if(t>=He(e))return{diffPath:null,label:"sil"};const i=Oi(e,n),o=Ds(i,t);if(o){const u=bn(o.fromVtime,o.toVtime,o.from,o.to);if(t>=u.transStart&&t<u.transEnd){let l=vr(o.from,o.to);r&&(l=Cn(l,r));const f=ni(l,u.effectiveGapMs);if(f.length>0){const c=si(t,u.transStart,u.transEnd,f.length);return{diffPath:f[c]??f[f.length-1],label:`${o.from}→${o.to}`,transitionMeta:{from:o.from,to:o.to,fromVtime:o.fromVtime,toVtime:o.toVtime,frameIdx:c,frameCount:f.length,fullFrameCount:l.length,omittedByGap:Math.max(0,l.length-f.length)}}}return{diffPath:_t(o.to),label:o.to}}}const a=En(e,t,s);return a==="sil"?{diffPath:null,label:"sil"}:{diffPath:_t(a),label:a}}function xr(e,t){const r=t??Ut(e.expressionPath);return`${Ut(e.mouthPath)}|${r}`}function Jr(e,t,r,n,s,i,o,a,u,l){const f=Pr(t,r,u,i),c={mouthPath:f.diffPath,expressionPath:n},d=xr(c,s),g=Ne();if(f.transitionMeta&&g?.onTransitionSample(f.transitionMeta,f.diffPath),d===i.lastDrawnKey){g?.onDrawDeduped();return}g?.onDrawRequested(),a(f),i.lastDrawnKey=d,o(e,c,d,b=>{b===d&&(i.lastDrawnKey=d)})}function Qr(e,t,r,n,s,i,o=null){const a={mouthPath:null,expressionPath:t},u=o!=null?`|idle@${Math.round(o*1e3)}`:"",l=xr(a,r)+u;!(o!=null)&&l===n.lastDrawnKey||(i(o!=null?"idle":"sil"),s(e,a,l,c=>{c===l&&(n.lastDrawnKey=l)}))}function Wn(e,t,r,n,s=yt,i){if(r<=t||e.length===0)return[];const o=i??Ir(),a=[];let u="";for(let l=t;l<r;l+=s){const f=Pr(e,l,n,o),c=Ut(f.diffPath);c!==u&&(u=c,a.push({displayAtMs:l,mouthPath:f.diffPath,mouthKey:c,label:f.label}))}return a}const Hi=3;function Rr(e){let t=0;const r=[];let n=!1,s=!1;const i=async()=>{if(!n){n=!0;try{for(;r.length>0;){const a=r.shift(),u=++t;await e(a.diffs,u,a)&&u===t&&a.onDisplayed?.(a.drawKey)}}finally{n=!1,r.length>0&&i()}}},o=()=>{s||(s=!0,queueMicrotask(()=>{s=!1,i()}))};return{enqueue(a,u,l){const f=r.findIndex(c=>c.drawKey===u);f>=0?r[f]={diffs:a,drawKey:u,onDisplayed:l}:(r.length>=Hi&&(r.shift(),Ne()?.onQueueSuperseded()),r.push({diffs:a,drawKey:u,onDisplayed:l})),o()},reset(){t+=1,r.length=0},isLatestSeq(a){return a===t},getLatestSeq(){return t}}}function $i(e,t=2){const r=[],n=new Set;let s=0,i=!1,o=0;const a=(l,f)=>`${l}|${f}`,u=async()=>{if(!i){i=!0;try{for(;s<t&&r.length>0;){const l=r.shift(),f=a(l.displayAtMs,l.drawKey);if(n.has(f))continue;n.add(f),s+=1;const c=o;try{await e(l)}finally{if(n.delete(f),s-=1,c!==o)return}}}finally{i=!1,r.length>0&&s<t&&u()}}};return{enqueue(l){const f=a(l.displayAtMs,l.drawKey);n.has(f)||r.some(c=>a(c.displayAtMs,c.drawKey)===f)||(r.push(l),u())},isQueuedOrInflight(l,f){const c=a(l,f);return n.has(c)?!0:r.some(d=>a(d.displayAtMs,d.drawKey)===c)},reset(){o+=1,r.length=0,n.clear(),s=0},pendingCount(){return r.length+s}}}function qi(){const e=new Map,t=r=>{r.bitmap.close()};return{has(r,n){return e.has(n)},markRendered(r,n,s,i,o){const a=e.get(n);a&&t(a),e.set(n,{displayAtMs:r,mouthKey:n,label:s,mouthPath:i,bitmap:o})},findForMouthKey(r){return e.get(r)??null},removeThrough(r){},pruneBefore(r){},reset(){for(const r of e.values())t(r);e.clear()},size(){return e.size}}}function Ki(e,t,r,n,s,i){const o=Wn(t,r,n,s,yt,i);if(o.length===0)return{ready:0,total:0};let a=0;for(const u of o)e.has(u.displayAtMs,u.mouthKey)&&(a+=1);return{ready:a,total:o.length}}function Gi(e){const t=qi(),r=Ir(),n=new Set;let s="",i=null;const o=$i(async c=>{try{if(!e.supportsBufferedPlayback)return;const d=await e.compositePrefetchFrame(c.diffs,c.mouthKey);if(!d)return;t.markRendered(c.displayAtMs,c.mouthKey,c.label,c.mouthPath,d),Ne()?.onBufferPrefetched()}finally{n.delete(c.mouthKey)}},ns),a=c=>{for(const d of c)t.has(d.displayAtMs,d.mouthKey)||n.has(d.mouthKey)||o.isQueuedOrInflight(0,d.mouthKey)||(n.add(d.mouthKey),o.enqueue({...d,displayAtMs:0,drawKey:d.mouthKey,diffs:{mouthPath:d.mouthPath,expressionPath:null}}))},u=(c,d,g,b=Xt)=>{if(!e.supportsBufferedPlayback)return;const m=Math.max(0,d-yt*4),R=Wn(c,m,d+b,g,yt,r);a(R)},l=(c,d)=>{s=c,i=d},f=(c,d)=>!(s!==c||i!==d);return{reset(){t.reset(),o.reset(),n.clear(),Ae(r),s="",i=null,Ne()?.onBufferReset()},markExternallyDisplayed(c,d){l(c.split("|")[0]??c,d)},warmupTick(c){e.supportsBufferedPlayback&&u(c.queue,0,c.availablePaths,Math.max(Fr,Xt))},isWarmupReady(c,d=Fr){if(!e.supportsBufferedPlayback)return!0;const{ready:g,total:b}=Ki(t,c.queue,0,d,c.availablePaths,r);return b===0?!1:g/b>=rs},getBufferedFrameCount:()=>t.size(),tick(c){const d=Ne();d?.onBufferTick(),u(c.queue,c.elapsedMs,c.availablePaths),t.pruneBefore(c.elapsedMs-Xt);const g=Pr(c.queue,c.elapsedMs,c.availablePaths,r),b=Ut(g.diffPath),m=xr({mouthPath:g.diffPath,expressionPath:c.expressionPath},c.expressionDrawKey),R=()=>(l(b,c.expressionPath),d?.onBufferHit(),{usedBuffer:!0,underrun:!1,held:!0});if(f(b,c.expressionPath)||c.playbackState.lastDrawnKey===m)return R();const _=t.findForMouthKey(b,c.elapsedMs,yt);return _&&c.expressionPath!=null?(d?.onBufferHit(),{usedBuffer:!1,underrun:!1,held:!1}):_?(e.blitBitmap(c.canvas,_.bitmap),l(_.mouthKey,c.expressionPath),c.playbackState.lastDrawnKey=m,c.onStatus({diffPath:_.mouthPath,label:_.label}),c.onDisplayed(m,_.mouthPath,_.label),d?.onBufferHit(),{usedBuffer:!0,underrun:!1,held:!1}):(d?.onBufferUnderrun(),{usedBuffer:!1,underrun:!0,held:!1})}}}const zi=`
|
|
6
6
|
in vec2 aPosition;
|
|
7
7
|
out vec2 vTextureCoord;
|
|
8
8
|
|
|
@@ -44,4 +44,4 @@ void main(void)
|
|
|
44
44
|
float keep = step(uThreshold + 1e-6, peak);
|
|
45
45
|
finalColor = vec4(color.rgb, keep);
|
|
46
46
|
}
|
|
47
|
-
`;function Zr(e=5){const t=new ne.Filter({glProgram:ne.GlProgram.from({vertex:zi,fragment:ji,name:"threshold-key-filter"}),resources:{thresholdUniforms:{uThreshold:{value:e/255,type:"f32"}}}});return t.setThreshold=r=>{t.resources.thresholdUniforms.uniforms.uThreshold=r/255},t.getThreshold=()=>Math.round(t.resources.thresholdUniforms.uniforms.uThreshold*255),t}function Yi(e,t,r){return e>r||t>r}function Xi(e){return e?e.getParameter(e.MAX_TEXTURE_SIZE):4096}function Ji(e,t){return t!=null&&t>0?Math.floor(t):Xi(e)}function Qi(e,t){const r=document.createElement("canvas");r.width=t.w,r.height=t.h;const n=r.getContext("2d");if(!n)throw new Error("Failed to acquire 2d context for atlas cell crop");return n.drawImage(e,t.x,t.y,t.w,t.h,0,0,t.w,t.h),r}function Hn(e){return fetch(e).then(t=>{if(!t.ok)throw new Error(`Failed to load image: ${e}`);return t.blob()}).then(t=>createImageBitmap(t))}async function en(e,t,r,n,s){const i=r?await Mn(e,s):await fetch(e).then(c=>{if(!c.ok)throw new Error(`Failed to load ${e}`);return c.json()}),o=i.sheets?.[0];if(!o?.path)throw new Error(`${e} has no sheets[0].path`);const a=new Map,u=async(c,d)=>{const g=Bn(t,d,r),v=r?await zs(await it(g,s??"")):await Hn(g);if(Yi(v.width,v.height,n))return console.info(`[PixiVisemeCompositor] Sheet ${d} (${v.width}x${v.height}) exceeds MAX_TEXTURE_SIZE=${n}; using per-cell crops.`),{mode:"cells",bitmap:v};const m=ne.Texture.from(v);return m.source.scaleMode="nearest",v.close(),{mode:"full",texture:m}};a.set(o.index,await u(o.index,o.path));for(const c of i.sheets??[])c.index!==o.index&&c.path&&a.set(c.index,await u(c.index,c.path));const l=new Map;for(const c of i.cells??[])c.path&&l.set(c.path,c);const f=new Map;for(const c of i.sheets??[])f.set(c.index,c);return{atlasMeta:i,cellByPath:l,sheetByIndex:f,diffBase:t,sheets:a}}async function Zi(e,t={}){const r=new ne.Application;try{await r.init({canvas:e,width:842,height:1264,preference:"webgl",autoStart:!1,antialias:!1,resolution:1,autoDensity:!1,backgroundAlpha:1,backgroundColor:0,powerPreference:"high-performance"})}catch(S){throw r.destroy({removeView:!1}),S}if(r.renderer.type!==ne.RendererType.WEBGL)throw r.destroy({removeView:!1}),new Error("Pixi did not initialize a WebGL renderer");const n=Ji(r.renderer.gl,t.maxTextureSize);let s=!1,i=!1,o=!1;const a=r.canvas,u=()=>{s=!0,!i&&!o&&(i=!0,t.onWebGlContextLost?.())};a.addEventListener("webglcontextlost",u);const l=new ne.Sprite(ne.Texture.EMPTY),f=new ne.Sprite(ne.Texture.EMPTY),c=new ne.Sprite(ne.Texture.EMPTY),d=new ne.Sprite(ne.Texture.EMPTY);d.visible=!1;const g=Zr(5),v=Zr(5);f.filters=[g],c.filters=[v],r.stage.addChild(l),r.stage.addChild(f),r.stage.addChild(c),r.stage.addChild(d);let m=null,R=null,_=null,D=null,w=null,T=null,O=842,B=1264,G=null,F=5;const k=new Map;let P=null,C=null,V=null;const N=dn(()=>{k.clear(),w=null,T=null,m=null,G=null}),he=(S,E)=>{O=S,B=E,r.renderer.resize(S,E),(e.width!==S||e.height!==E)&&(e.width=S,e.height=E)},X=()=>{f.visible=!1,c.visible=!1,f.texture=ne.Texture.EMPTY,c.texture=ne.Texture.EMPTY,P=null,C=null},z=()=>{d.visible=!1,m&&(l.texture=m,l.visible=!0,l.width=O,l.height=B,l.x=0,l.y=0)},J=()=>{if(_){try{_.resource=null}catch{}_.destroy(),_=null}R&&(R.destroy(!1),R=null),D=null},se=(S,E)=>{const h=S.cellByPath.get(E);if(!h||!Rn(h))return null;const A=`${S.diffBase}::${E}`;let M=k.get(A);if(!M){const L=h.sheet??0,I=S.sheets.get(L);if(!I)return null;let $,Y=!1;if(I.mode==="full")$=new ne.Texture({source:I.texture.source,frame:new ne.Rectangle(h.x,h.y,h.w,h.h)});else{const Q=Qi(I.bitmap,h);$=ne.Texture.from(Q),$.source.scaleMode="nearest",Y=!0}M={texture:$,ownsSource:Y},k.set(A,M)}return{texture:M.texture,cell:h}},ee=(S,E,h,A)=>{if(!A||!h)return S.visible=!1,S.texture=ne.Texture.EMPTY,!1;const M=se(h,A);if(!M)return S.visible=!1,S.texture=ne.Texture.EMPTY,!1;const L=xn(h.atlasMeta,M.cell);return E.setThreshold(F),S.texture=M.texture,S.x=L.x,S.y=L.y,S.width=M.cell.w,S.height=M.cell.h,S.visible=!0,!0},ge=S=>{if(!S||!w)return null;const E=G??new Set(w.cellByPath.keys());return Sr(S,E)??(E.has(S)?S:null)},ce=S=>{d.visible=!1,z();const E=ge(S.mouthPath),h=S.expressionPath&&T&&T.cellByPath.has(S.expressionPath)?S.expressionPath:null;E!==P?(ee(f,g,w,E),P=E):E?f.visible=!0:f.visible=!1,h!==C?(ee(c,v,T,h),C=h):h?c.visible=!0:c.visible=!1},fe=()=>{if(!(o||s||document.hidden))try{r.render()}catch{}};return{async loadAssets(S){if(o)return{width:O,height:B,visemePaths:[]};const E=vt();if(E&&!S)throw new Error("Encrypted assets enabled but no key provided");const h=E?await Ys(await it(Lt(),S??"")):await Hn(Lt());m&&m.destroy(!0),m=ne.Texture.from(h),m.source.scaleMode="linear";const A=mn();w=await en(hn(),A,E,n,S);try{T=await en(pn(),A,E,n,S)}catch{T=null}const M=Pn(w.atlasMeta,h.width,h.height);return he(M.width,M.height),F=$t(w.atlasMeta),g.setThreshold(F),v.setThreshold(F),G=new Set(w.cellByPath.keys()),k.clear(),z(),X(),fe(),{width:M.width,height:M.height,visemePaths:[...w.cellByPath.keys()]}},getVisemeAtlasPaths(){return G},drawLiveFrame(S){o||s||!m||(ce(S),fe())},drawIdleVideo(S,E){o||s||document.hidden||E&&!E()||S.readyState<HTMLMediaElement.HAVE_CURRENT_DATA||(X(),d.visible=!1,!R||D!==S?(J(),_=new ne.VideoSource({resource:S,autoPlay:!1,updateFPS:0}),_.autoUpdate=!1,R=new ne.Texture({source:_}),D=S):R.source.update(),l.texture=R,l.visible=!0,l.x=0,l.y=0,l.width=O,l.height=B,fe())},renderSilOnly(){o||s||!m||(X(),z(),fe())},blitBitmap(S){o||s||(X(),V&&(V.destroy(!0),V=null),V=ne.Texture.from(S),d.texture=V,d.visible=!0,d.x=0,d.y=0,d.width=O,d.height=B,l.visible=!1,fe())},async compositePrefetchFrame(S){return null},renderFrame:fe,isContextLost(){return s},dispose(){if(o)return;o=!0,a.removeEventListener("webglcontextlost",u),N(),J(),V&&(V.destroy(!0),V=null);for(const{texture:E,ownsSource:h}of k.values())E.destroy(h);k.clear(),m&&(m.destroy(!0),m=null);const S=E=>{E.mode==="full"?E.texture.destroy(!0):E.bitmap.close()};if(w)for(const E of w.sheets.values())S(E);if(T)for(const E of T.sheets.values())S(E);g.destroy(!0),v.destroy(!0),r.destroy({removeView:!1},{children:!0,texture:!0})}}}const tn=-1,Dt=1e5,rn=new Map;async function ea(e){let t=rn.get(e);if(!t){const r=await fetch(e);if(!r.ok)throw new Error(`Failed to fetch worker script (${r.status}): ${e}`);const n=await r.blob();t=URL.createObjectURL(n),rn.set(e,t)}return new Worker(t,{name:"visemeDiffPreview"})}function ta(){return typeof Worker<"u"&&typeof OffscreenCanvas<"u"}function cr(e,t){(e.width!==t.width||e.height!==t.height)&&(e.width=t.width,e.height=t.height);const r=e.getContext("2d");r&&r.drawImage(t,0,0)}function nn(e,t){let r=0,n=null;const s=Rr(async(i,o)=>(await ar(),await or(e,i),s.isLatestSeq(o)));return{usesWorker:!1,usesPixi:!1,async loadAssets(i){const o=await ar(i);return n=o.visemePaths?new Set(o.visemePaths):null,o},getVisemeAtlasPaths(){return n},async renderSilOnly(){r+=1,s.reset(),await Mi(e)},drawLiveFrame(i,o,a,u){s.enqueue(o,a,u)},async compositePrefetchFrame(i,o){try{return await Ri(i)}catch{return null}},blitBitmap(i,o){cr(i,o)},drawIdleVideo(){},supportsBufferedPlayback:!0,async renderViseme(i,o){r+=1;const a=r;t.onStatus?.(o),await or(i,{mouthPath:_t(o),expressionPath:null})},dispose(){r+=1,s.reset()}}}function ra(e,t,r){let n=null,s=1,i=0,o=!1,a=!1,u=null;const l=new Map,f=new Map;let c=Dt;const d=Rr(async(D,w,T)=>(l.set(w,{drawKey:T.drawKey,onDisplayed:T.onDisplayed}),await R({type:"drawFrame",diffPath:D.mouthPath,expressionDiffPath:D.expressionPath,liveSeq:w},void 0,!0),!1)),g=D=>{if(D===tn){let O=-1,B;for(const[G,F]of l)G>=O&&(O=G,B=F);O>=0&&l.delete(O),B?.onDisplayed?.(B.drawKey);return}const w=-D,T=l.get(w);l.delete(w),T?.onDisplayed?.(T.drawKey)},v=new Map,m=(D,w)=>{a||!n||n.postMessage(D,[])},R=(D,w,T=!1)=>{if(a)return Promise.resolve();const O=s++;return new Promise((B,G)=>{v.set(O,{resolve:F=>B(F),reject:G,generation:typeof D.generation=="number"?D.generation:void 0,expectRenderDone:T}),m({...D,requestId:O})})},_=(async()=>{n=await ea(r),n.onmessage=D=>{if(a)return;const w=D.data;if(w.type==="status"){t.onStatus?.(w.label);return}if(w.type==="frame"){if(w.generation<=-Dt){const B=-w.generation,G=f.get(B);f.delete(B),G?.resolve(w.bitmap);return}const O=w.generation<0;if(!O&&w.generation!==i){w.bitmap.close();return}if(O&&!(w.generation===tn)){const G=-w.generation;if(!d.isLatestSeq(G)){Ne()?.onWorkerStale(),w.bitmap.close();return}}if(O&&t.shouldAcceptLiveFrame&&!t.shouldAcceptLiveFrame()){Ne()?.onWorkerStale(),w.bitmap.close();return}cr(e,w.bitmap),w.bitmap.close(),O&&g(w.generation);return}const T=v.get(w.requestId);if(T)switch(w.type){case"ready":v.delete(w.requestId),T.resolve();break;case"loadAssetsDone":v.delete(w.requestId),u=w.visemePaths?.length?new Set(w.visemePaths):null,T.resolve({width:w.width,height:w.height,visemePaths:w.visemePaths});break;case"renderDone":if(T.expectRenderDone&&T.generation!=null&&w.generation!==T.generation)return;v.delete(w.requestId),T.resolve();break;case"renderAborted":v.delete(w.requestId),T.resolve();break;case"error":v.delete(w.requestId),t.onError?.(w.message),T.reject(new Error(w.message));break}},n.onerror=D=>{if(a)return;const w=D.message||"Worker error";t.onError?.(w);for(const[,T]of v)T.reject(new Error(w));v.clear()},await R({type:"init"}),o=!0})();return{usesWorker:!0,usesPixi:!1,async loadAssets(D){if(a)return{width:842,height:1264};if(await _,a)return{width:842,height:1264};if(!o)throw new Error("Worker init failed");return await R({type:"loadAssets",keyHex:D,urls:ws(je())})??{width:842,height:1264}},getVisemeAtlasPaths(){return u},async renderSilOnly(){if(a||(await _,a))return;i+=1,d.reset(),l.clear(),f.forEach(({resolve:w})=>w(null)),f.clear(),c=Dt;const D=i;await R({type:"cancelLiveDraws"}),await R({type:"renderSil",generation:D},void 0,!0)},drawLiveFrame(D,w,T,O){a||_.then(()=>{a||d.enqueue(w,T,O)})},compositePrefetchFrame(D,w){return a?Promise.resolve(null):_.then(()=>{if(a)return null;const T=++c;return new Promise(O=>{f.set(T,{resolve:O}),R({type:"drawFrame",diffPath:D.mouthPath,expressionDiffPath:D.expressionPath,liveSeq:T},void 0,!0).catch(()=>{f.delete(T),O(null)})})})},blitBitmap(D,w){cr(D,w)},drawIdleVideo(){},supportsBufferedPlayback:!0,async renderViseme(D,w,T={}){if(a||(await _,a))return;i+=1,d.reset(),l.clear();const O=i;try{await R({type:"cancelLiveDraws"}),await R({type:"renderViseme",generation:O,to:w,from:T.from,transitionDurationMs:T.transitionDurationMs,gapMs:T.gapMs,threshold:T.threshold},void 0,!0)}catch(B){if(B?.name==="AbortError")return;throw B}},dispose(){if(!a){a=!0,i+=1,d.reset(),l.clear(),f.forEach(({resolve:D})=>D(null)),f.clear(),c=Dt;for(const[,D]of v)D.resolve();v.clear(),n?.terminate(),n=null}}}}async function na(e,t){const r=await Zi(e,{onWebGlContextLost:t.onWebGlContextLost,maxTextureSize:t.pixiMaxTextureSize});let n=0,s=null,i=!1;const o=Rr(async(a,u)=>i?!1:t.shouldAcceptLiveFrame&&!t.shouldAcceptLiveFrame()?(Ne()?.onWorkerStale(),!1):(r.drawLiveFrame(a),o.isLatestSeq(u)));return{usesWorker:!1,usesPixi:!0,async loadAssets(a){if(i)return{width:842,height:1264};const u=await r.loadAssets(a);return s=u.visemePaths?.length?new Set(u.visemePaths):null,u},getVisemeAtlasPaths(){return s},async renderSilOnly(){i||(n+=1,o.reset(),r.renderSilOnly())},drawLiveFrame(a,u,l,f){i||o.enqueue(u,l,f)},async compositePrefetchFrame(a,u){if(i)return null;try{return await r.compositePrefetchFrame(a)}catch{return null}},blitBitmap(a,u){i||t.shouldAcceptLiveFrame&&!t.shouldAcceptLiveFrame()||r.blitBitmap(u)},drawIdleVideo(a,u){i||r.drawIdleVideo(a,u)},supportsBufferedPlayback:!1,async renderViseme(a,u){if(i)return;n+=1;const l=n;o.reset(),t.onStatus?.(u),r.drawLiveFrame({mouthPath:_t(u),expressionPath:null})},dispose(){i||(i=!0,n+=1,o.reset(),r.dispose())}}}function sn(e,t){if(!ta())return nn(e,t);const r=Zn(t.workerScriptUrl);try{return ra(e,t,r)}catch{return nn(e,t)}}function sa(e,t={}){if(t.preferLegacyRenderer)return sn(e,t);let r=null,n=!1,s=null,i=!1;const o=()=>r?Promise.resolve(r):(s||(s=(async()=>{try{const a=await na(e,t);if(i)throw a.dispose(),new Error("Renderer disposed during Pixi init");return n=!0,r=a,a}catch(a){if(i)throw a instanceof Error?a:new Error(String(a));return console.warn("[AiTwin] Pixi WebGL renderer unavailable, using Canvas2D fallback:",a),r=sn(e,t),n=!1,r}})()),s);return{get usesWorker(){return r?.usesWorker??!1},get usesPixi(){return n},async loadAssets(a){return(await o()).loadAssets(a)},getVisemeAtlasPaths(){return r?.getVisemeAtlasPaths()??null},async renderSilOnly(){return(await o()).renderSilOnly()},drawLiveFrame(a,u,l,f){o().then(c=>{i||c.drawLiveFrame(a,u,l,f)})},async compositePrefetchFrame(a,u){return(await o()).compositePrefetchFrame(a,u)},blitBitmap(a,u){r&&r.blitBitmap(a,u)},drawIdleVideo(a,u){r?.usesPixi&&r.drawIdleVideo(a,u)},get supportsBufferedPlayback(){return r?.supportsBufferedPlayback??!1},async renderViseme(a,u,l){return(await o()).renderViseme(a,u,l)},dispose(){i=!0,s=null,r?.dispose(),r=null}}}class ia extends Error{constructor(t,r){super(`${t} timed out after ${Math.round(r/1e3)}s`),this.name="LoadTimeoutError"}}function Zt(e,t,r){return new Promise((n,s)=>{const i=globalThis.setTimeout(()=>{s(new ia(r,t))},t);e.then(o=>{globalThis.clearTimeout(i),n(o)},o=>{globalThis.clearTimeout(i),s(o)})})}const aa={position:"relative",display:"inline-block",lineHeight:0},oa={display:"block",maxWidth:"100%",height:"auto"},ca={position:"absolute",inset:0,display:"flex",alignItems:"center",justifyContent:"center",padding:"1rem",background:"rgba(0,0,0,0.55)",color:"#fff",fontSize:"0.875rem",textAlign:"center"};function wt(e){return is(e)?wr:pr}function Le(e,t){return t?.isSpeaking()?t:e.isSpeaking()?e:t?.isConnected()?t:e}const ua=b.forwardRef(function({id:t,avatarId:r,faceId:n,assets:s,authToken:i,apiBase:o,ttsEngineId:a=qe,voiceId:u,speakingRate:l=Tt,className:f,style:c,canvasStyle:d,onStatusChange:g,onDisplayStatus:v,onError:m,onReady:R,showErrorOverlay:_=!0,onUserTranscript:D,onBotOutput:w,onRealtimeLipsyncDebug:T,onFrameRenderStats:O,lipsyncBufferedPlayback:B=!0,lipsyncPlaybackWarmupMs:G=1500,pixiMaxTextureSize:F},k){const P=b.useRef(null),C=b.useRef(null),V=b.useRef(null),N=b.useRef(B);N.current=B;const he=b.useRef(G);he.current=G;const X=b.useRef(0),z=b.useRef(Ir()),J=b.useRef(Si()),se=b.useRef(0),ee=b.useRef(0),ge=b.useRef(0),ce=b.useRef(null),fe=b.useRef(null),S=b.useRef(!1),E=b.useRef(!1),h=b.useRef(!1),A=b.useRef("sil"),M=b.useRef(0),L=b.useRef(!1),I=b.useRef(null),$=b.useRef(!1),[Y,Q]=b.useState("idle"),[Z,Ie]=b.useState(null),[be,Be]=b.useState(!1),[at,Pe]=b.useState(!1),[p,x]=b.useState({width:842,height:1264}),[j,ae]=b.useState(0),pe=b.useRef(!1),_e=b.useRef(i??null),ue=o?.trim()||St,xe=b.useRef(null),de=b.useRef(null),Ue=b.useRef(!1),Ye=b.useRef({width:842,height:1264}),Se=b.useRef(null),ot=b.useRef("sil"),Xe=b.useRef(void 0),ct=b.useRef(new Map),Oe=b.useRef(R),le=b.useRef(v),ut=b.useRef(m),Je=b.useRef(D),me=b.useRef(w),we=b.useRef(T),Te=b.useRef(O),Re=b.useRef(Ci());Oe.current=R,le.current=v,ut.current=m,Je.current=D,me.current=w,we.current=T,Te.current=O,b.useEffect(()=>{const y=Re.current;return jr(y),()=>jr(null)},[]);const lt=b.useCallback(y=>{const U=y?.ttsEngineId??a??de.current?.ttsEngineId??qe,H=y?.voiceId??u??de.current?.voiceId??wt(U);return{ttsEngineId:U,voiceId:H}},[a,u]),Me=b.useCallback(y=>{Ie(y),ut.current?.(y)},[]),Ve=b.useCallback(y=>{Q(y),g?.(y)},[g]),te=b.useRef(nr(Ve,ue,{onLipsyncDebug:y=>we.current?.(y)})),ie=b.useRef(null);b.useEffect(()=>(te.current=nr(Ve,ue,{onLipsyncDebug:y=>we.current?.(y)}),i&&te.current.setDeveloperToken(i),te.current.setTtsEngineId(a),()=>{te.current.stop();const y=ie.current;y&&(y.disconnect(),ie.current=null)}),[Ve,i,a,ue]);const Mr=b.useCallback(()=>(ie.current||(ie.current=In(Ve,{speakingRate:l,onUserTranscript:(y,U)=>Je.current?.(y,U),onBotOutput:y=>me.current?.(y),onLipsyncDebug:y=>we.current?.(y)}),ie.current.setTtsEngineId(a)),ie.current),[Ve,l,a]),ke=b.useCallback(()=>{ce.current!=null&&(clearTimeout(ce.current),ce.current=null)},[]),Fe=b.useCallback(()=>{fe.current!=null&&(clearTimeout(fe.current),fe.current=null),h.current=!1},[]),Dr=b.useCallback(()=>{!$.current||!I.current||(Fe(),h.current=!0,fe.current=setTimeout(()=>{if(fe.current=null,Le(te.current,ie.current).isSpeaking()){h.current=!1;return}if(S.current){h.current=!1;return}const y=I.current;if(!y){h.current=!1;return}y.restart(),y.setActive(!0);const U=y.getVideo();let H=!1;const K=()=>{H||(H=!0,U.removeEventListener("playing",q),h.current=!1)},q=()=>{K()};if(U.addEventListener("playing",q,{once:!0}),U.readyState>=HTMLMediaElement.HAVE_CURRENT_DATA&&!U.paused){K();return}setTimeout(K,300)},Bi))},[Fe]),Qe=b.useCallback(()=>{if(ke(),le.current?.("sil"),S.current=!1,!$.current||!I.current){h.current=!1;return}Dr()},[ke,Dr]),qt=b.useCallback(async y=>{const U=ee.current,H=C.current,K=P.current,q=()=>{y?.fallbackOnly||(S.current=!1)};if(!H||!K||!be){q();return}if(Le(te.current,ie.current).isSpeaking()){q();return}if(y?.fallbackOnly){if(!S.current||(Ae(z.current),await H.renderSilOnly(),U!==ee.current)||Le(te.current,ie.current).isSpeaking())return;Qe();return}if(Xr(z.current)){Ae(z.current),Qe();return}if(N.current&&H.supportsBufferedPlayback&&!y?.fallbackOnly){if(Ae(z.current),V.current?.reset(),await H.renderSilOnly(),U!==ee.current||Le(te.current,ie.current).isSpeaking())return;Qe();return}const Ee=Ui(Le(te.current,ie.current).getVisemeQueue(),M.current,A.current),dt=Vi(Ee,Se.current);if(Ae(z.current),!dt){if(await H.renderSilOnly(),U!==ee.current||Le(te.current,ie.current).isSpeaking())return;Qe();return}try{await H.renderViseme(K,"sil",{from:Ee,transitionDurationMs:Un})}catch(Ze){if(Ze?.name!=="AbortError")throw Ze}if(U!==ee.current){q();return}if(Le(te.current,ie.current).isSpeaking()){q();return}Qe()},[be,Qe]),ft=b.useCallback(()=>{if(E.current||S.current)return;E.current=!0,S.current=!0,ke();const y=ee.current;qt().finally(()=>{E.current=!1}),ce.current=setTimeout(()=>{ce.current=null,y===ee.current&&qt({fallbackOnly:!0})},Wi())},[ke,qt]),Cr=b.useCallback(async()=>{const y=Math.floor(Date.now()/1e3);if(xe.current&&y<xe.current.expiresAt-60)return xe.current.key;const U=_e.current??await Ct();_e.current=U,te.current.setDeveloperToken(U);const H=await ds(U,ue);return xe.current={key:H.value,expiresAt:H.expiresAt},H.value},[ue]);b.useEffect(()=>{i&&(_e.current=i,te.current.setDeveloperToken(i))},[i]),b.useEffect(()=>{te.current.setTtsEngineId(a),ie.current?.setTtsEngineId(a),de.current&&(de.current.ttsEngineId=a)},[a]),b.useEffect(()=>{const y=u??wt(a);de.current&&(de.current.voiceId=y)},[u,a]);const We=b.useMemo(()=>ln(r,n),[r,n]),Kn=b.useMemo(()=>s?JSON.stringify(s):We?`face:${We}`:`id:${t??""}`,[s,We,t]);return b.useEffect(()=>{if(!t&&!s&&!We){Me("AiTwin requires `assets`, `avatarId`/`faceId`, or `id`");return}let y=!1;Ue.current=!1,Be(!1),Pe(!1),Ie(null),de.current=null,I.current=null,$.current=!1;const U=P.current;if(!U)return;let H=null;return(async()=>{try{let K,q,ve,Ee;if(s)K=s,q=a,ve=u??wt(q);else if(We)le.current?.("loadFace"),K=rr(We),q=a,ve=u??wt(q);else{le.current?.("getAiTwin");const W=await Zt(un(t),3e4,"getAiTwin");if(y)return;if(!W.faceId?.trim())throw new Error("getAiTwin returned no faceId");q=W.ttsEngineId,ve=u??W.voiceId??wt(q),Ee=W.knowledgeContextId?.trim()||void 0,K=rr(W.faceId)}de.current={ttsEngineId:q,voiceId:ve,personaId:Ee},fn(K),I.current=vi();const dt=!!(s&&!ze(s));H=sa(U,{preferLegacyRenderer:dt,pixiMaxTextureSize:F,onStatus:W=>{y||le.current?.(W)},onError:W=>{y||Me(W)},shouldAcceptLiveFrame:()=>{const W=Le(te.current,ie.current);return(W.isSpeaking()||W.isWarmingUp())&&!S.current},onWebGlContextLost:()=>{y||pe.current||(pe.current=!0,queueMicrotask(()=>{y||ae(W=>W+1)}))}}),C.current=H,V.current=Gi(H),pe.current=!1,(W=>{const oe=()=>{if(!W||he.current<=0)return!1;const ye=C.current;return ye?N.current&&ye.supportsBufferedPlayback?!0:ye.usesPixi:!1};te.current.setLipsyncWarmupHold(W,oe),ie.current?.setLipsyncWarmupHold(W,oe)})(!0),le.current?.("auth");const mt=_e.current??i??await Zt(Ct(),3e4,"getAuthToken");if(y)return;_e.current=mt,te.current.setDeveloperToken(mt),te.current.setTtsEngineId(q);const kt=vt()?await Cr():void 0;if(y)return;le.current?.("loadAssets");const re=await Zt(H.loadAssets(kt),18e4,"load face assets");if(!y&&re.width>0&&re.height>0&&(Ye.current=re,x(re)),y)return;Se.current=H.getVisemeAtlasPaths(),Ue.current=!0,Be(!0),Oe.current?.(),H.renderSilOnly().then(()=>{y||le.current?.("sil")}),(async()=>{try{const W=await Qs(kt);if(await I.current.load(W),y)return;$.current=!0,Pe(!0),Ae(z.current),I.current.setActive(!0);const oe=I.current.getVideo();H.usesPixi?H.drawIdleVideo(oe):await zr(U,oe,null,void 0,Ye.current),y||le.current?.("idle")}catch(W){if(y)return;$.current=!1,Pe(!1),console.warn("[AiTwin] Idle video unavailable:",W)}})()}catch(K){if(y)return;if(K instanceof Bt){Me(K.message);return}Me(K instanceof Error?K.message:"Failed to load AI twin")}})(),()=>{y=!0,I.current?.dispose(),H?.dispose(),C.current=null}},[Kn,s,We,t,a,u,i,F,Cr,Me,j]),b.useEffect(()=>{const y={current:!1},U=()=>{const H=P.current,K=C.current,q=Le(te.current,ie.current);if(!H||!K||!be){se.current=requestAnimationFrame(U);return}if(document.hidden){I.current?.setActive(!1),se.current=requestAnimationFrame(U);return}const ve=q.isSpeaking(),Ee=q.isWarmingUp(),dt=ve&&!L.current,Ze=L.current&&!ve;dt&&(ke(),Fe(),S.current=!1,E.current=!1,Ae(z.current),V.current?.reset(),Re.current.reset(),Xe.current=void 0,ge.current=0,A.current="sil",M.current=0,I.current?.setActive(!1)),Ze&&(J.current.reset(),I.current?.setActive(!1),V.current?.reset(),ft());const mt=Ze||S.current||h.current,kt=B&&K.supportsBufferedPlayback;if(Ee){$.current&&I.current?.setActive(!1);const re=q.getVisemeQueue();re.length>0&&X.current===0&&(X.current=performance.now()),J.current.advance(performance.now());const W=J.current.getExpressionPath(),oe=J.current.getDrawKey(),ye={queue:re,elapsedMs:0,expressionPath:W,expressionDrawKey:oe,availablePaths:Se.current};let De=!1;kt&&V.current?(V.current.warmupTick(ye),De=V.current.isWarmupReady(ye,he.current)):K.usesPixi&&Jr(H,re,0,W,oe,z.current,(ht,Et,Kt,At)=>{K.drawLiveFrame(ht,Et,Kt,Ce=>{At?.(Ce),q.isWarmingUp()&&Ce!==""&&!Xr({lastDrawnKey:Ce})&&(X.current=0,q.releaseWarmupHold())})},ht=>{le.current?.(ht.label)},Se.current);const et=X.current>0&&performance.now()-X.current>ts;le.current?.(De?"buffer ready":"buffering lipsync…"),(De||et)&&(X.current=0,q.releaseWarmupHold()),L.current=q.isSpeaking(),se.current=requestAnimationFrame(U);return}if(ve){$.current&&I.current?.setActive(!1);const re=q.getVisemeQueue(),W=q.getPlaybackElapsedMs();Os(re,W),M.current=W;const oe=re.length>0?bt(re,W):"sil";oe!=="sil"&&(A.current=oe);const ye=Ei(W,q.getSentenceEmotions(),q.getWordQueue(),q.getStreamMood());J.current.setTargetEmotion(ye),J.current.setTargetEyebrow(_i(W,q.getWordQueue(),Tr(ye))),J.current.advance(performance.now());const De=J.current.getExpressionPath(),et=J.current.getDrawKey(),ht=(Ce,Gt,pt,zt)=>{const Br={label:ot.current,elapsedMs:W,mouthPath:Gt.mouthPath,transitionMeta:Xe.current};ct.current.set(pt,Br),V.current?.markExternallyDisplayed(pt,De),K.drawLiveFrame(Ce,Gt,pt,It=>{const Gn=ct.current.get(It)??Br;ct.current.delete(It),Re.current.onDrawDisplayed(Gn,re),V.current?.markExternallyDisplayed(It,De),zt?.(It)})},Et=Ce=>{ot.current=Ce.label,Xe.current=Ce.transitionMeta,le.current?.(Ce.label)},Kt=B&&K.supportsBufferedPlayback;let At=!1;Kt&&V.current&&(At=V.current.tick({canvas:H,queue:re,elapsedMs:W,expressionPath:De,expressionDrawKey:et,playbackState:z.current,availablePaths:Se.current,onStatus:Et,onDisplayed:(Gt,pt,zt)=>{Re.current.onDrawDisplayed({label:zt,elapsedMs:W,mouthPath:pt,transitionMeta:Xe.current},re)}}).usedBuffer),At||Jr(H,re,W,De,et,z.current,ht,Et,Se.current),Re.current.onRafTickSpeaking(),Te.current?.(Re.current.snapshot(re,W))}else if(!mt&&$.current&&I.current){const re=I.current,W=re.getVideo();if(W.readyState>=HTMLMediaElement.HAVE_CURRENT_DATA){re.setActive(!0);const oe=()=>!q.isSpeaking()&&!S.current&&!h.current;K.usesPixi?(K.drawIdleVideo(W,oe),oe()&&le.current?.("idle")):y.current||(y.current=!0,zr(H,W,null,void 0,Ye.current,oe).then(()=>le.current?.("idle")).finally(()=>{y.current=!1}))}else Qr(H,null,"__sil__",z.current,(oe,ye,De,et)=>K.drawLiveFrame(oe,ye,De,et),oe=>le.current?.(oe))}else mt||Qr(H,null,"__sil__",z.current,(re,W,oe,ye)=>K.drawLiveFrame(re,W,oe,ye),re=>le.current?.(re));L.current=ve,se.current=requestAnimationFrame(U)};return se.current=requestAnimationFrame(U),()=>{cancelAnimationFrame(se.current),ke(),Fe(),te.current.stop(),ie.current?.stop()}},[be,at,ft,ke,Fe]),b.useImperativeHandle(k,()=>({speakText:async(y,U)=>{const H=y.trim();if(!H)return;const K=C.current;if(!K||!Ue.current&&!be)throw new Error("AI twin is not ready");const{ttsEngineId:q,voiceId:ve}=lt(U);Ie(null),ee.current+=1,ke(),Fe(),S.current=!1,E.current=!1,A.current="sil",M.current=0,Ae(z.current),J.current.reset(),L.current=!1;try{$.current?(I.current?.setActive(!1),I.current?.restart(),Ae(z.current)):await K.renderSilOnly(),await te.current.speak(H,{voiceId:ve,speakingRate:U?.speakingRate??l,ttsEngineId:q})}catch(Ee){throw Me(Ee instanceof Error?Ee.message:"TTS failed"),Ee}},stop:()=>{ee.current+=1,ke(),Fe(),S.current=!1,E.current=!1,te.current.stop(),ie.current?.stop(),Ae(z.current),V.current?.reset(),X.current=0,L.current=!1,ft()},setTtsEngineId:y=>{te.current.setTtsEngineId(y),ie.current?.setTtsEngineId(y),de.current&&(de.current.ttsEngineId=y)},renderViseme:async(y,U)=>{const H=C.current,K=P.current;if(!H||!K||!Ue.current)throw new Error("AI twin is not ready");await H.renderViseme(K,y,{from:U?.from,transitionDurationMs:U?.transitionDurationMs,gapMs:U?.gapMs,onStatus:q=>le.current?.(q)})},isReady:()=>Ue.current,getStatus:()=>Y,connect:async(y={})=>{Ie(null);try{let U=y.authToken?.trim();U||(U=_e.current??i??await Ct(),_e.current=U);const H=y.voiceId?.trim()??u??de.current?.voiceId,K=y.personaId?.trim()??de.current?.personaId,q=Mr();await q.unlockAudio(),await q.connect({...y,apiBase:y.apiBase?.trim()||ue,authToken:U,voiceId:H,personaId:K,speakingRate:y.speakingRate??l})}catch(U){throw Me(U instanceof Error?U.message:"Voice connect failed"),U}},disconnect:async()=>{const y=ie.current;y&&(ee.current+=1,ke(),Fe(),S.current=!1,E.current=!1,Ae(z.current),V.current?.reset(),X.current=0,L.current=!1,await y.disconnect(),ft())},isConnected:()=>ie.current?.isConnected()??!1,getVoiceStream:()=>ie.current?.getVoiceStream()??te.current?.getVoiceStream()??null}),[be,l,i,u,ke,Fe,ft,Me,lt,Mr,Y]),jt.jsxs("div",{className:f,style:{...aa,...c},children:[jt.jsx("canvas",{ref:P,width:p.width,height:p.height,style:{...oa,...d},"aria-label":t?`AI twin ${t}`:"AI twin face"}),_&&Z?jt.jsx("div",{style:ca,children:Z}):null]})}),$n="thumbnail.webp";function la(e,t=lr){const r=e.trim();if(!r)throw new Error("faceId is required");return`${t.replace(/\/$/,"")}/${r}/${$n}`}const fa=new Set([".png",".jpg",".jpeg",".webp"]);function qn(e=".png"){const t=e.toLowerCase();return fa.has(t)?`source${t}`:"source.png"}function da(e,t=lr,r=".png"){const n=e.trim();if(!n)throw new Error("faceId is required");return`${t.replace(/\/$/,"")}/${n}/${qn(r)}`}exports.AI_TWIN_THUMBNAIL_FILENAME=$n;exports.AiTwin=ua;exports.AiTwinNotFoundError=Bt;exports.DEFAULT_TTS_ENGINE_ID=qe;exports.MIC_SAMPLE_RATE=gr;exports.OCULUS_VISEME_IDS=gn;exports.PLAYBACK_SAMPLE_RATE=yr;exports.SPEAKING_RATE_MAX=Ht;exports.SPEAKING_RATE_MIN=Wt;exports.TTS_ENGINE_CARTESIA=Ge;exports.TTS_ENGINE_GOOGLE=dr;exports.TTS_ENGINE_INWORLD=Vt;exports.VISEME_IDS=Zs;exports.VISEME_TEST_CARTESIA_VOICE_ID=wr;exports.VISEME_TEST_SPEAKING_RATE=Tt;exports.VISEME_TEST_VOICE_ID=pr;exports.aiTwinSourceFilename=qn;exports.buildVoiceWebSocketUrl=An;exports.createAvatarTtsLipsyncController=nr;exports.createRealtimeVoiceLipsyncController=In;exports.faceAssetUrls=rr;exports.fetchAiTwin=un;exports.fetchDevAuthToken=Ct;exports.getAiTwinSourceUrl=da;exports.getAiTwinThumbnailUrl=la;exports.normalizeOculusViseme=yn;exports.resolveDirectFaceId=ln;exports.resolveVisemeAtTime=bt;exports.resolveWordAtTime=Ns;exports.setActiveTwinAssets=fn;
|
|
47
|
+
`;function Zr(e=5){const t=new ne.Filter({glProgram:ne.GlProgram.from({vertex:zi,fragment:ji,name:"threshold-key-filter"}),resources:{thresholdUniforms:{uThreshold:{value:e/255,type:"f32"}}}});return t.setThreshold=r=>{t.resources.thresholdUniforms.uniforms.uThreshold=r/255},t.getThreshold=()=>Math.round(t.resources.thresholdUniforms.uniforms.uThreshold*255),t}function Yi(e,t,r){return e>r||t>r}function Xi(e){return e?e.getParameter(e.MAX_TEXTURE_SIZE):4096}function Ji(e,t){return t!=null&&t>0?Math.floor(t):Xi(e)}function Qi(e,t){const r=document.createElement("canvas");r.width=t.w,r.height=t.h;const n=r.getContext("2d");if(!n)throw new Error("Failed to acquire 2d context for atlas cell crop");return n.drawImage(e,t.x,t.y,t.w,t.h,0,0,t.w,t.h),r}function Hn(e){return fetch(e).then(t=>{if(!t.ok)throw new Error(`Failed to load image: ${e}`);return t.blob()}).then(t=>createImageBitmap(t))}async function en(e,t,r,n,s){const i=r?await Mn(e,s):await fetch(e).then(c=>{if(!c.ok)throw new Error(`Failed to load ${e}`);return c.json()}),o=i.sheets?.[0];if(!o?.path)throw new Error(`${e} has no sheets[0].path`);const a=new Map,u=async(c,d)=>{const g=Bn(t,d,r),b=r?await zs(await it(g,s??"")):await Hn(g);if(Yi(b.width,b.height,n))return console.info(`[PixiVisemeCompositor] Sheet ${d} (${b.width}x${b.height}) exceeds MAX_TEXTURE_SIZE=${n}; using per-cell crops.`),{mode:"cells",bitmap:b};const m=ne.Texture.from(b);return m.source.scaleMode="nearest",{mode:"full",texture:m}};a.set(o.index,await u(o.index,o.path));for(const c of i.sheets??[])c.index!==o.index&&c.path&&a.set(c.index,await u(c.index,c.path));const l=new Map;for(const c of i.cells??[])c.path&&l.set(c.path,c);const f=new Map;for(const c of i.sheets??[])f.set(c.index,c);return{atlasMeta:i,cellByPath:l,sheetByIndex:f,diffBase:t,sheets:a}}async function Zi(e,t={}){const r=new ne.Application;try{await r.init({canvas:e,width:842,height:1264,preference:"webgl",autoStart:!1,antialias:!1,resolution:1,autoDensity:!1,backgroundAlpha:1,backgroundColor:0,powerPreference:"high-performance"})}catch(S){throw r.destroy({removeView:!1}),S}if(r.renderer.type!==ne.RendererType.WEBGL)throw r.destroy({removeView:!1}),new Error("Pixi did not initialize a WebGL renderer");const n=Ji(r.renderer.gl,t.maxTextureSize);let s=!1,i=!1,o=!1;const a=r.canvas,u=()=>{s=!0,!i&&!o&&(i=!0,t.onWebGlContextLost?.())};a.addEventListener("webglcontextlost",u);const l=new ne.Sprite(ne.Texture.EMPTY),f=new ne.Sprite(ne.Texture.EMPTY),c=new ne.Sprite(ne.Texture.EMPTY),d=new ne.Sprite(ne.Texture.EMPTY);d.visible=!1;const g=Zr(5),b=Zr(5);f.filters=[g],c.filters=[b],r.stage.addChild(l),r.stage.addChild(f),r.stage.addChild(c),r.stage.addChild(d);let m=null,R=null,_=null,D=null,w=null,T=null,O=842,B=1264,G=null,F=5;const k=new Map;let P=null,C=null,V=null;const N=dn(()=>{k.clear(),w=null,T=null,m=null,G=null}),he=(S,E)=>{O=S,B=E,r.renderer.resize(S,E),(e.width!==S||e.height!==E)&&(e.width=S,e.height=E)},X=()=>{f.visible=!1,c.visible=!1,f.texture=ne.Texture.EMPTY,c.texture=ne.Texture.EMPTY,P=null,C=null},z=()=>{d.visible=!1,m&&(l.texture=m,l.visible=!0,l.width=O,l.height=B,l.x=0,l.y=0)},J=()=>{if(_){try{_.resource=null}catch{}_.destroy(),_=null}R&&(R.destroy(!1),R=null),D=null},se=(S,E)=>{const h=S.cellByPath.get(E);if(!h||!Rn(h))return null;const A=`${S.diffBase}::${E}`;let M=k.get(A);if(!M){const L=h.sheet??0,I=S.sheets.get(L);if(!I)return null;let $,Y=!1;if(I.mode==="full")$=new ne.Texture({source:I.texture.source,frame:new ne.Rectangle(h.x,h.y,h.w,h.h)});else{const Q=Qi(I.bitmap,h);$=ne.Texture.from(Q),$.source.scaleMode="nearest",Y=!0}M={texture:$,ownsSource:Y},k.set(A,M)}return{texture:M.texture,cell:h}},ee=(S,E,h,A)=>{if(!A||!h)return S.visible=!1,S.texture=ne.Texture.EMPTY,!1;const M=se(h,A);if(!M)return S.visible=!1,S.texture=ne.Texture.EMPTY,!1;const L=xn(h.atlasMeta,M.cell);return E.setThreshold(F),S.texture=M.texture,S.x=L.x,S.y=L.y,S.width=M.cell.w,S.height=M.cell.h,S.visible=!0,!0},ge=S=>{if(!S||!w)return null;const E=G??new Set(w.cellByPath.keys());return Sr(S,E)??(E.has(S)?S:null)},ce=S=>{d.visible=!1,z();const E=ge(S.mouthPath),h=S.expressionPath&&T&&T.cellByPath.has(S.expressionPath)?S.expressionPath:null;E!==P?(ee(f,g,w,E),P=E):E?f.visible=!0:f.visible=!1,h!==C?(ee(c,b,T,h),C=h):h?c.visible=!0:c.visible=!1},fe=()=>{if(!(o||s||document.hidden))try{r.render()}catch{}};return{async loadAssets(S){if(o)return{width:O,height:B,visemePaths:[]};const E=vt();if(E&&!S)throw new Error("Encrypted assets enabled but no key provided");const h=E?await Ys(await it(Lt(),S??"")):await Hn(Lt());m&&m.destroy(!0),m=ne.Texture.from(h),m.source.scaleMode="linear";const A=mn();w=await en(hn(),A,E,n,S);try{T=await en(pn(),A,E,n,S)}catch{T=null}const M=Pn(w.atlasMeta,h.width,h.height);return he(M.width,M.height),F=$t(w.atlasMeta),g.setThreshold(F),b.setThreshold(F),G=new Set(w.cellByPath.keys()),k.clear(),z(),X(),fe(),{width:M.width,height:M.height,visemePaths:[...w.cellByPath.keys()]}},getVisemeAtlasPaths(){return G},drawLiveFrame(S){o||s||!m||(ce(S),fe())},drawIdleVideo(S,E){o||s||document.hidden||E&&!E()||S.readyState<HTMLMediaElement.HAVE_CURRENT_DATA||(X(),d.visible=!1,!R||D!==S?(J(),_=new ne.VideoSource({resource:S,autoPlay:!1,updateFPS:0}),_.autoUpdate=!1,R=new ne.Texture({source:_}),D=S):R.source.update(),l.texture=R,l.visible=!0,l.x=0,l.y=0,l.width=O,l.height=B,fe())},renderSilOnly(){o||s||!m||(X(),z(),fe())},blitBitmap(S){o||s||(X(),V&&(V.destroy(!0),V=null),V=ne.Texture.from(S),d.texture=V,d.visible=!0,d.x=0,d.y=0,d.width=O,d.height=B,l.visible=!1,fe())},async compositePrefetchFrame(S){return null},renderFrame:fe,isContextLost(){return s},dispose(){if(o)return;o=!0,a.removeEventListener("webglcontextlost",u),N(),J(),V&&(V.destroy(!0),V=null);for(const{texture:E,ownsSource:h}of k.values())E.destroy(h);k.clear(),m&&(m.destroy(!0),m=null);const S=E=>{E.mode==="full"?E.texture.destroy(!0):E.bitmap.close()};if(w)for(const E of w.sheets.values())S(E);if(T)for(const E of T.sheets.values())S(E);g.destroy(!0),b.destroy(!0),r.destroy({removeView:!1},{children:!0,texture:!0})}}}const tn=-1,Dt=1e5,rn=new Map;async function ea(e){let t=rn.get(e);if(!t){const r=await fetch(e);if(!r.ok)throw new Error(`Failed to fetch worker script (${r.status}): ${e}`);const n=await r.blob();t=URL.createObjectURL(n),rn.set(e,t)}return new Worker(t,{name:"visemeDiffPreview"})}function ta(){return typeof Worker<"u"&&typeof OffscreenCanvas<"u"}function cr(e,t){(e.width!==t.width||e.height!==t.height)&&(e.width=t.width,e.height=t.height);const r=e.getContext("2d");r&&r.drawImage(t,0,0)}function nn(e,t){let r=0,n=null;const s=Rr(async(i,o)=>(await ar(),await or(e,i),s.isLatestSeq(o)));return{usesWorker:!1,usesPixi:!1,async loadAssets(i){const o=await ar(i);return n=o.visemePaths?new Set(o.visemePaths):null,o},getVisemeAtlasPaths(){return n},async renderSilOnly(){r+=1,s.reset(),await Mi(e)},drawLiveFrame(i,o,a,u){s.enqueue(o,a,u)},async compositePrefetchFrame(i,o){try{return await Ri(i)}catch{return null}},blitBitmap(i,o){cr(i,o)},drawIdleVideo(){},supportsBufferedPlayback:!0,async renderViseme(i,o){r+=1;const a=r;t.onStatus?.(o),await or(i,{mouthPath:_t(o),expressionPath:null})},dispose(){r+=1,s.reset()}}}function ra(e,t,r){let n=null,s=1,i=0,o=!1,a=!1,u=null;const l=new Map,f=new Map;let c=Dt;const d=Rr(async(D,w,T)=>(l.set(w,{drawKey:T.drawKey,onDisplayed:T.onDisplayed}),await R({type:"drawFrame",diffPath:D.mouthPath,expressionDiffPath:D.expressionPath,liveSeq:w},void 0,!0),!1)),g=D=>{if(D===tn){let O=-1,B;for(const[G,F]of l)G>=O&&(O=G,B=F);O>=0&&l.delete(O),B?.onDisplayed?.(B.drawKey);return}const w=-D,T=l.get(w);l.delete(w),T?.onDisplayed?.(T.drawKey)},b=new Map,m=(D,w)=>{a||!n||n.postMessage(D,[])},R=(D,w,T=!1)=>{if(a)return Promise.resolve();const O=s++;return new Promise((B,G)=>{b.set(O,{resolve:F=>B(F),reject:G,generation:typeof D.generation=="number"?D.generation:void 0,expectRenderDone:T}),m({...D,requestId:O})})},_=(async()=>{n=await ea(r),n.onmessage=D=>{if(a)return;const w=D.data;if(w.type==="status"){t.onStatus?.(w.label);return}if(w.type==="frame"){if(w.generation<=-Dt){const B=-w.generation,G=f.get(B);f.delete(B),G?.resolve(w.bitmap);return}const O=w.generation<0;if(!O&&w.generation!==i){w.bitmap.close();return}if(O&&!(w.generation===tn)){const G=-w.generation;if(!d.isLatestSeq(G)){Ne()?.onWorkerStale(),w.bitmap.close();return}}if(O&&t.shouldAcceptLiveFrame&&!t.shouldAcceptLiveFrame()){Ne()?.onWorkerStale(),w.bitmap.close();return}cr(e,w.bitmap),w.bitmap.close(),O&&g(w.generation);return}const T=b.get(w.requestId);if(T)switch(w.type){case"ready":b.delete(w.requestId),T.resolve();break;case"loadAssetsDone":b.delete(w.requestId),u=w.visemePaths?.length?new Set(w.visemePaths):null,T.resolve({width:w.width,height:w.height,visemePaths:w.visemePaths});break;case"renderDone":if(T.expectRenderDone&&T.generation!=null&&w.generation!==T.generation)return;b.delete(w.requestId),T.resolve();break;case"renderAborted":b.delete(w.requestId),T.resolve();break;case"error":b.delete(w.requestId),t.onError?.(w.message),T.reject(new Error(w.message));break}},n.onerror=D=>{if(a)return;const w=D.message||"Worker error";t.onError?.(w);for(const[,T]of b)T.reject(new Error(w));b.clear()},await R({type:"init"}),o=!0})();return{usesWorker:!0,usesPixi:!1,async loadAssets(D){if(a)return{width:842,height:1264};if(await _,a)return{width:842,height:1264};if(!o)throw new Error("Worker init failed");return await R({type:"loadAssets",keyHex:D,urls:ws(je())})??{width:842,height:1264}},getVisemeAtlasPaths(){return u},async renderSilOnly(){if(a||(await _,a))return;i+=1,d.reset(),l.clear(),f.forEach(({resolve:w})=>w(null)),f.clear(),c=Dt;const D=i;await R({type:"cancelLiveDraws"}),await R({type:"renderSil",generation:D},void 0,!0)},drawLiveFrame(D,w,T,O){a||_.then(()=>{a||d.enqueue(w,T,O)})},compositePrefetchFrame(D,w){return a?Promise.resolve(null):_.then(()=>{if(a)return null;const T=++c;return new Promise(O=>{f.set(T,{resolve:O}),R({type:"drawFrame",diffPath:D.mouthPath,expressionDiffPath:D.expressionPath,liveSeq:T},void 0,!0).catch(()=>{f.delete(T),O(null)})})})},blitBitmap(D,w){cr(D,w)},drawIdleVideo(){},supportsBufferedPlayback:!0,async renderViseme(D,w,T={}){if(a||(await _,a))return;i+=1,d.reset(),l.clear();const O=i;try{await R({type:"cancelLiveDraws"}),await R({type:"renderViseme",generation:O,to:w,from:T.from,transitionDurationMs:T.transitionDurationMs,gapMs:T.gapMs,threshold:T.threshold},void 0,!0)}catch(B){if(B?.name==="AbortError")return;throw B}},dispose(){if(!a){a=!0,i+=1,d.reset(),l.clear(),f.forEach(({resolve:D})=>D(null)),f.clear(),c=Dt;for(const[,D]of b)D.resolve();b.clear(),n?.terminate(),n=null}}}}async function na(e,t){const r=await Zi(e,{onWebGlContextLost:t.onWebGlContextLost,maxTextureSize:t.pixiMaxTextureSize});let n=0,s=null,i=!1;const o=Rr(async(a,u)=>i?!1:t.shouldAcceptLiveFrame&&!t.shouldAcceptLiveFrame()?(Ne()?.onWorkerStale(),!1):(r.drawLiveFrame(a),o.isLatestSeq(u)));return{usesWorker:!1,usesPixi:!0,async loadAssets(a){if(i)return{width:842,height:1264};const u=await r.loadAssets(a);return s=u.visemePaths?.length?new Set(u.visemePaths):null,u},getVisemeAtlasPaths(){return s},async renderSilOnly(){i||(n+=1,o.reset(),r.renderSilOnly())},drawLiveFrame(a,u,l,f){i||o.enqueue(u,l,f)},async compositePrefetchFrame(a,u){if(i)return null;try{return await r.compositePrefetchFrame(a)}catch{return null}},blitBitmap(a,u){i||t.shouldAcceptLiveFrame&&!t.shouldAcceptLiveFrame()||r.blitBitmap(u)},drawIdleVideo(a,u){i||r.drawIdleVideo(a,u)},supportsBufferedPlayback:!1,async renderViseme(a,u){if(i)return;n+=1;const l=n;o.reset(),t.onStatus?.(u),r.drawLiveFrame({mouthPath:_t(u),expressionPath:null})},dispose(){i||(i=!0,n+=1,o.reset(),r.dispose())}}}function sn(e,t){if(!ta())return nn(e,t);const r=Zn(t.workerScriptUrl);try{return ra(e,t,r)}catch{return nn(e,t)}}function sa(e,t={}){if(t.preferLegacyRenderer)return sn(e,t);let r=null,n=!1,s=null,i=!1;const o=()=>r?Promise.resolve(r):(s||(s=(async()=>{try{const a=await na(e,t);if(i)throw a.dispose(),new Error("Renderer disposed during Pixi init");return n=!0,r=a,a}catch(a){if(i)throw a instanceof Error?a:new Error(String(a));return console.warn("[AiTwin] Pixi WebGL renderer unavailable, using Canvas2D fallback:",a),r=sn(e,t),n=!1,r}})()),s);return{get usesWorker(){return r?.usesWorker??!1},get usesPixi(){return n},async loadAssets(a){return(await o()).loadAssets(a)},getVisemeAtlasPaths(){return r?.getVisemeAtlasPaths()??null},async renderSilOnly(){return(await o()).renderSilOnly()},drawLiveFrame(a,u,l,f){o().then(c=>{i||c.drawLiveFrame(a,u,l,f)})},async compositePrefetchFrame(a,u){return(await o()).compositePrefetchFrame(a,u)},blitBitmap(a,u){r&&r.blitBitmap(a,u)},drawIdleVideo(a,u){r?.usesPixi&&r.drawIdleVideo(a,u)},get supportsBufferedPlayback(){return r?.supportsBufferedPlayback??!1},async renderViseme(a,u,l){return(await o()).renderViseme(a,u,l)},dispose(){i=!0,s=null,r?.dispose(),r=null}}}class ia extends Error{constructor(t,r){super(`${t} timed out after ${Math.round(r/1e3)}s`),this.name="LoadTimeoutError"}}function Zt(e,t,r){return new Promise((n,s)=>{const i=globalThis.setTimeout(()=>{s(new ia(r,t))},t);e.then(o=>{globalThis.clearTimeout(i),n(o)},o=>{globalThis.clearTimeout(i),s(o)})})}const aa={position:"relative",display:"inline-block",lineHeight:0},oa={display:"block",maxWidth:"100%",height:"auto"},ca={position:"absolute",inset:0,display:"flex",alignItems:"center",justifyContent:"center",padding:"1rem",background:"rgba(0,0,0,0.55)",color:"#fff",fontSize:"0.875rem",textAlign:"center"};function wt(e){return is(e)?wr:pr}function Le(e,t){return t?.isSpeaking()?t:e.isSpeaking()?e:t?.isConnected()?t:e}const ua=v.forwardRef(function({id:t,avatarId:r,faceId:n,assets:s,authToken:i,apiBase:o,ttsEngineId:a=qe,voiceId:u,speakingRate:l=Tt,className:f,style:c,canvasStyle:d,onStatusChange:g,onDisplayStatus:b,onError:m,onReady:R,showErrorOverlay:_=!0,onUserTranscript:D,onBotOutput:w,onRealtimeLipsyncDebug:T,onFrameRenderStats:O,lipsyncBufferedPlayback:B=!0,lipsyncPlaybackWarmupMs:G=1500,pixiMaxTextureSize:F},k){const P=v.useRef(null),C=v.useRef(null),V=v.useRef(null),N=v.useRef(B);N.current=B;const he=v.useRef(G);he.current=G;const X=v.useRef(0),z=v.useRef(Ir()),J=v.useRef(Si()),se=v.useRef(0),ee=v.useRef(0),ge=v.useRef(0),ce=v.useRef(null),fe=v.useRef(null),S=v.useRef(!1),E=v.useRef(!1),h=v.useRef(!1),A=v.useRef("sil"),M=v.useRef(0),L=v.useRef(!1),I=v.useRef(null),$=v.useRef(!1),[Y,Q]=v.useState("idle"),[Z,Ie]=v.useState(null),[be,Be]=v.useState(!1),[at,Pe]=v.useState(!1),[p,x]=v.useState({width:842,height:1264}),[j,ae]=v.useState(0),pe=v.useRef(!1),_e=v.useRef(i??null),ue=o?.trim()||St,xe=v.useRef(null),de=v.useRef(null),Ue=v.useRef(!1),Ye=v.useRef({width:842,height:1264}),Se=v.useRef(null),ot=v.useRef("sil"),Xe=v.useRef(void 0),ct=v.useRef(new Map),Oe=v.useRef(R),le=v.useRef(b),ut=v.useRef(m),Je=v.useRef(D),me=v.useRef(w),we=v.useRef(T),Te=v.useRef(O),Re=v.useRef(Ci());Oe.current=R,le.current=b,ut.current=m,Je.current=D,me.current=w,we.current=T,Te.current=O,v.useEffect(()=>{const y=Re.current;return jr(y),()=>jr(null)},[]);const lt=v.useCallback(y=>{const U=y?.ttsEngineId??a??de.current?.ttsEngineId??qe,H=y?.voiceId??u??de.current?.voiceId??wt(U);return{ttsEngineId:U,voiceId:H}},[a,u]),Me=v.useCallback(y=>{Ie(y),ut.current?.(y)},[]),Ve=v.useCallback(y=>{Q(y),g?.(y)},[g]),te=v.useRef(nr(Ve,ue,{onLipsyncDebug:y=>we.current?.(y)})),ie=v.useRef(null);v.useEffect(()=>(te.current=nr(Ve,ue,{onLipsyncDebug:y=>we.current?.(y)}),i&&te.current.setDeveloperToken(i),te.current.setTtsEngineId(a),()=>{te.current.stop();const y=ie.current;y&&(y.disconnect(),ie.current=null)}),[Ve,i,a,ue]);const Mr=v.useCallback(()=>(ie.current||(ie.current=In(Ve,{speakingRate:l,onUserTranscript:(y,U)=>Je.current?.(y,U),onBotOutput:y=>me.current?.(y),onLipsyncDebug:y=>we.current?.(y)}),ie.current.setTtsEngineId(a)),ie.current),[Ve,l,a]),ke=v.useCallback(()=>{ce.current!=null&&(clearTimeout(ce.current),ce.current=null)},[]),Fe=v.useCallback(()=>{fe.current!=null&&(clearTimeout(fe.current),fe.current=null),h.current=!1},[]),Dr=v.useCallback(()=>{!$.current||!I.current||(Fe(),h.current=!0,fe.current=setTimeout(()=>{if(fe.current=null,Le(te.current,ie.current).isSpeaking()){h.current=!1;return}if(S.current){h.current=!1;return}const y=I.current;if(!y){h.current=!1;return}y.restart(),y.setActive(!0);const U=y.getVideo();let H=!1;const K=()=>{H||(H=!0,U.removeEventListener("playing",q),h.current=!1)},q=()=>{K()};if(U.addEventListener("playing",q,{once:!0}),U.readyState>=HTMLMediaElement.HAVE_CURRENT_DATA&&!U.paused){K();return}setTimeout(K,300)},Bi))},[Fe]),Qe=v.useCallback(()=>{if(ke(),le.current?.("sil"),S.current=!1,!$.current||!I.current){h.current=!1;return}Dr()},[ke,Dr]),qt=v.useCallback(async y=>{const U=ee.current,H=C.current,K=P.current,q=()=>{y?.fallbackOnly||(S.current=!1)};if(!H||!K||!be){q();return}if(Le(te.current,ie.current).isSpeaking()){q();return}if(y?.fallbackOnly){if(!S.current||(Ae(z.current),await H.renderSilOnly(),U!==ee.current)||Le(te.current,ie.current).isSpeaking())return;Qe();return}if(Xr(z.current)){Ae(z.current),Qe();return}if(N.current&&H.supportsBufferedPlayback&&!y?.fallbackOnly){if(Ae(z.current),V.current?.reset(),await H.renderSilOnly(),U!==ee.current||Le(te.current,ie.current).isSpeaking())return;Qe();return}const Ee=Ui(Le(te.current,ie.current).getVisemeQueue(),M.current,A.current),dt=Vi(Ee,Se.current);if(Ae(z.current),!dt){if(await H.renderSilOnly(),U!==ee.current||Le(te.current,ie.current).isSpeaking())return;Qe();return}try{await H.renderViseme(K,"sil",{from:Ee,transitionDurationMs:Un})}catch(Ze){if(Ze?.name!=="AbortError")throw Ze}if(U!==ee.current){q();return}if(Le(te.current,ie.current).isSpeaking()){q();return}Qe()},[be,Qe]),ft=v.useCallback(()=>{if(E.current||S.current)return;E.current=!0,S.current=!0,ke();const y=ee.current;qt().finally(()=>{E.current=!1}),ce.current=setTimeout(()=>{ce.current=null,y===ee.current&&qt({fallbackOnly:!0})},Wi())},[ke,qt]),Cr=v.useCallback(async()=>{const y=Math.floor(Date.now()/1e3);if(xe.current&&y<xe.current.expiresAt-60)return xe.current.key;const U=_e.current??await Ct();_e.current=U,te.current.setDeveloperToken(U);const H=await ds(U,ue);return xe.current={key:H.value,expiresAt:H.expiresAt},H.value},[ue]);v.useEffect(()=>{i&&(_e.current=i,te.current.setDeveloperToken(i))},[i]),v.useEffect(()=>{te.current.setTtsEngineId(a),ie.current?.setTtsEngineId(a),de.current&&(de.current.ttsEngineId=a)},[a]),v.useEffect(()=>{const y=u??wt(a);de.current&&(de.current.voiceId=y)},[u,a]);const We=v.useMemo(()=>ln(r,n),[r,n]),Kn=v.useMemo(()=>s?JSON.stringify(s):We?`face:${We}`:`id:${t??""}`,[s,We,t]);return v.useEffect(()=>{if(!t&&!s&&!We){Me("AiTwin requires `assets`, `avatarId`/`faceId`, or `id`");return}let y=!1;Ue.current=!1,Be(!1),Pe(!1),Ie(null),de.current=null,I.current=null,$.current=!1;const U=P.current;if(!U)return;let H=null;return(async()=>{try{let K,q,ve,Ee;if(s)K=s,q=a,ve=u??wt(q);else if(We)le.current?.("loadFace"),K=rr(We),q=a,ve=u??wt(q);else{le.current?.("getAiTwin");const W=await Zt(un(t),3e4,"getAiTwin");if(y)return;if(!W.faceId?.trim())throw new Error("getAiTwin returned no faceId");q=W.ttsEngineId,ve=u??W.voiceId??wt(q),Ee=W.knowledgeContextId?.trim()||void 0,K=rr(W.faceId)}de.current={ttsEngineId:q,voiceId:ve,personaId:Ee},fn(K),I.current=vi();const dt=!!(s&&!ze(s));H=sa(U,{preferLegacyRenderer:dt,pixiMaxTextureSize:F,onStatus:W=>{y||le.current?.(W)},onError:W=>{y||Me(W)},shouldAcceptLiveFrame:()=>{const W=Le(te.current,ie.current);return(W.isSpeaking()||W.isWarmingUp())&&!S.current},onWebGlContextLost:()=>{y||pe.current||(pe.current=!0,queueMicrotask(()=>{y||ae(W=>W+1)}))}}),C.current=H,V.current=Gi(H),pe.current=!1,(W=>{const oe=()=>{if(!W||he.current<=0)return!1;const ye=C.current;return ye?N.current&&ye.supportsBufferedPlayback?!0:ye.usesPixi:!1};te.current.setLipsyncWarmupHold(W,oe),ie.current?.setLipsyncWarmupHold(W,oe)})(!0),le.current?.("auth");const mt=_e.current??i??await Zt(Ct(),3e4,"getAuthToken");if(y)return;_e.current=mt,te.current.setDeveloperToken(mt),te.current.setTtsEngineId(q);const kt=vt()?await Cr():void 0;if(y)return;le.current?.("loadAssets");const re=await Zt(H.loadAssets(kt),18e4,"load face assets");if(!y&&re.width>0&&re.height>0&&(Ye.current=re,x(re)),y)return;Se.current=H.getVisemeAtlasPaths(),Ue.current=!0,Be(!0),Oe.current?.(),H.renderSilOnly().then(()=>{y||le.current?.("sil")}),(async()=>{try{const W=await Qs(kt);if(await I.current.load(W),y)return;$.current=!0,Pe(!0),Ae(z.current),I.current.setActive(!0);const oe=I.current.getVideo();H.usesPixi?H.drawIdleVideo(oe):await zr(U,oe,null,void 0,Ye.current),y||le.current?.("idle")}catch(W){if(y)return;$.current=!1,Pe(!1),console.warn("[AiTwin] Idle video unavailable:",W)}})()}catch(K){if(y)return;if(K instanceof Bt){Me(K.message);return}Me(K instanceof Error?K.message:"Failed to load AI twin")}})(),()=>{y=!0,I.current?.dispose(),H?.dispose(),C.current=null}},[Kn,s,We,t,a,u,i,F,Cr,Me,j]),v.useEffect(()=>{const y={current:!1},U=()=>{const H=P.current,K=C.current,q=Le(te.current,ie.current);if(!H||!K||!be){se.current=requestAnimationFrame(U);return}if(document.hidden){I.current?.setActive(!1),se.current=requestAnimationFrame(U);return}const ve=q.isSpeaking(),Ee=q.isWarmingUp(),dt=ve&&!L.current,Ze=L.current&&!ve;dt&&(ke(),Fe(),S.current=!1,E.current=!1,Ae(z.current),V.current?.reset(),Re.current.reset(),Xe.current=void 0,ge.current=0,A.current="sil",M.current=0,I.current?.setActive(!1)),Ze&&(J.current.reset(),I.current?.setActive(!1),V.current?.reset(),ft());const mt=Ze||S.current||h.current,kt=B&&K.supportsBufferedPlayback;if(Ee){$.current&&I.current?.setActive(!1);const re=q.getVisemeQueue();re.length>0&&X.current===0&&(X.current=performance.now()),J.current.advance(performance.now());const W=J.current.getExpressionPath(),oe=J.current.getDrawKey(),ye={queue:re,elapsedMs:0,expressionPath:W,expressionDrawKey:oe,availablePaths:Se.current};let De=!1;kt&&V.current?(V.current.warmupTick(ye),De=V.current.isWarmupReady(ye,he.current)):K.usesPixi&&Jr(H,re,0,W,oe,z.current,(ht,Et,Kt,At)=>{K.drawLiveFrame(ht,Et,Kt,Ce=>{At?.(Ce),q.isWarmingUp()&&Ce!==""&&!Xr({lastDrawnKey:Ce})&&(X.current=0,q.releaseWarmupHold())})},ht=>{le.current?.(ht.label)},Se.current);const et=X.current>0&&performance.now()-X.current>ts;le.current?.(De?"buffer ready":"buffering lipsync…"),(De||et)&&(X.current=0,q.releaseWarmupHold()),L.current=q.isSpeaking(),se.current=requestAnimationFrame(U);return}if(ve){$.current&&I.current?.setActive(!1);const re=q.getVisemeQueue(),W=q.getPlaybackElapsedMs();Os(re,W),M.current=W;const oe=re.length>0?bt(re,W):"sil";oe!=="sil"&&(A.current=oe);const ye=Ei(W,q.getSentenceEmotions(),q.getWordQueue(),q.getStreamMood());J.current.setTargetEmotion(ye),J.current.setTargetEyebrow(_i(W,q.getWordQueue(),Tr(ye))),J.current.advance(performance.now());const De=J.current.getExpressionPath(),et=J.current.getDrawKey(),ht=(Ce,Gt,pt,zt)=>{const Br={label:ot.current,elapsedMs:W,mouthPath:Gt.mouthPath,transitionMeta:Xe.current};ct.current.set(pt,Br),V.current?.markExternallyDisplayed(pt,De),K.drawLiveFrame(Ce,Gt,pt,It=>{const Gn=ct.current.get(It)??Br;ct.current.delete(It),Re.current.onDrawDisplayed(Gn,re),V.current?.markExternallyDisplayed(It,De),zt?.(It)})},Et=Ce=>{ot.current=Ce.label,Xe.current=Ce.transitionMeta,le.current?.(Ce.label)},Kt=B&&K.supportsBufferedPlayback;let At=!1;Kt&&V.current&&(At=V.current.tick({canvas:H,queue:re,elapsedMs:W,expressionPath:De,expressionDrawKey:et,playbackState:z.current,availablePaths:Se.current,onStatus:Et,onDisplayed:(Gt,pt,zt)=>{Re.current.onDrawDisplayed({label:zt,elapsedMs:W,mouthPath:pt,transitionMeta:Xe.current},re)}}).usedBuffer),At||Jr(H,re,W,De,et,z.current,ht,Et,Se.current),Re.current.onRafTickSpeaking(),Te.current?.(Re.current.snapshot(re,W))}else if(!mt&&$.current&&I.current){const re=I.current,W=re.getVideo();if(W.readyState>=HTMLMediaElement.HAVE_CURRENT_DATA){re.setActive(!0);const oe=()=>!q.isSpeaking()&&!S.current&&!h.current;K.usesPixi?(K.drawIdleVideo(W,oe),oe()&&le.current?.("idle")):y.current||(y.current=!0,zr(H,W,null,void 0,Ye.current,oe).then(()=>le.current?.("idle")).finally(()=>{y.current=!1}))}else Qr(H,null,"__sil__",z.current,(oe,ye,De,et)=>K.drawLiveFrame(oe,ye,De,et),oe=>le.current?.(oe))}else mt||Qr(H,null,"__sil__",z.current,(re,W,oe,ye)=>K.drawLiveFrame(re,W,oe,ye),re=>le.current?.(re));L.current=ve,se.current=requestAnimationFrame(U)};return se.current=requestAnimationFrame(U),()=>{cancelAnimationFrame(se.current),ke(),Fe(),te.current.stop(),ie.current?.stop()}},[be,at,ft,ke,Fe]),v.useImperativeHandle(k,()=>({speakText:async(y,U)=>{const H=y.trim();if(!H)return;const K=C.current;if(!K||!Ue.current&&!be)throw new Error("AI twin is not ready");const{ttsEngineId:q,voiceId:ve}=lt(U);Ie(null),ee.current+=1,ke(),Fe(),S.current=!1,E.current=!1,A.current="sil",M.current=0,Ae(z.current),J.current.reset(),L.current=!1;try{$.current?(I.current?.setActive(!1),I.current?.restart(),Ae(z.current)):await K.renderSilOnly(),await te.current.speak(H,{voiceId:ve,speakingRate:U?.speakingRate??l,ttsEngineId:q})}catch(Ee){throw Me(Ee instanceof Error?Ee.message:"TTS failed"),Ee}},stop:()=>{ee.current+=1,ke(),Fe(),S.current=!1,E.current=!1,te.current.stop(),ie.current?.stop(),Ae(z.current),V.current?.reset(),X.current=0,L.current=!1,ft()},setTtsEngineId:y=>{te.current.setTtsEngineId(y),ie.current?.setTtsEngineId(y),de.current&&(de.current.ttsEngineId=y)},renderViseme:async(y,U)=>{const H=C.current,K=P.current;if(!H||!K||!Ue.current)throw new Error("AI twin is not ready");await H.renderViseme(K,y,{from:U?.from,transitionDurationMs:U?.transitionDurationMs,gapMs:U?.gapMs,onStatus:q=>le.current?.(q)})},isReady:()=>Ue.current,getStatus:()=>Y,connect:async(y={})=>{Ie(null);try{let U=y.authToken?.trim();U||(U=_e.current??i??await Ct(),_e.current=U);const H=y.voiceId?.trim()??u??de.current?.voiceId,K=y.personaId?.trim()??de.current?.personaId,q=Mr();await q.unlockAudio(),await q.connect({...y,apiBase:y.apiBase?.trim()||ue,authToken:U,voiceId:H,personaId:K,speakingRate:y.speakingRate??l})}catch(U){throw Me(U instanceof Error?U.message:"Voice connect failed"),U}},disconnect:async()=>{const y=ie.current;y&&(ee.current+=1,ke(),Fe(),S.current=!1,E.current=!1,Ae(z.current),V.current?.reset(),X.current=0,L.current=!1,await y.disconnect(),ft())},isConnected:()=>ie.current?.isConnected()??!1,getVoiceStream:()=>ie.current?.getVoiceStream()??te.current?.getVoiceStream()??null}),[be,l,i,u,ke,Fe,ft,Me,lt,Mr,Y]),jt.jsxs("div",{className:f,style:{...aa,...c},children:[jt.jsx("canvas",{ref:P,width:p.width,height:p.height,style:{...oa,...d},"aria-label":t?`AI twin ${t}`:"AI twin face"}),_&&Z?jt.jsx("div",{style:ca,children:Z}):null]})}),$n="thumbnail.webp";function la(e,t=lr){const r=e.trim();if(!r)throw new Error("faceId is required");return`${t.replace(/\/$/,"")}/${r}/${$n}`}const fa=new Set([".png",".jpg",".jpeg",".webp"]);function qn(e=".png"){const t=e.toLowerCase();return fa.has(t)?`source${t}`:"source.png"}function da(e,t=lr,r=".png"){const n=e.trim();if(!n)throw new Error("faceId is required");return`${t.replace(/\/$/,"")}/${n}/${qn(r)}`}exports.AI_TWIN_THUMBNAIL_FILENAME=$n;exports.AiTwin=ua;exports.AiTwinNotFoundError=Bt;exports.DEFAULT_TTS_ENGINE_ID=qe;exports.MIC_SAMPLE_RATE=gr;exports.OCULUS_VISEME_IDS=gn;exports.PLAYBACK_SAMPLE_RATE=yr;exports.SPEAKING_RATE_MAX=Ht;exports.SPEAKING_RATE_MIN=Wt;exports.TTS_ENGINE_CARTESIA=Ge;exports.TTS_ENGINE_GOOGLE=dr;exports.TTS_ENGINE_INWORLD=Vt;exports.VISEME_IDS=Zs;exports.VISEME_TEST_CARTESIA_VOICE_ID=wr;exports.VISEME_TEST_SPEAKING_RATE=Tt;exports.VISEME_TEST_VOICE_ID=pr;exports.aiTwinSourceFilename=qn;exports.buildVoiceWebSocketUrl=An;exports.createAvatarTtsLipsyncController=nr;exports.createRealtimeVoiceLipsyncController=In;exports.faceAssetUrls=rr;exports.fetchAiTwin=un;exports.fetchDevAuthToken=Ct;exports.getAiTwinSourceUrl=da;exports.getAiTwinThumbnailUrl=la;exports.normalizeOculusViseme=yn;exports.resolveDirectFaceId=ln;exports.resolveVisemeAtTime=bt;exports.resolveWordAtTime=Ns;exports.setActiveTwinAssets=fn;
|
package/dist/aitwin.js
CHANGED
|
@@ -5,7 +5,7 @@ import { WebSocketTransport as Kn, DailyMediaManager as Gn, ProtobufFrameSeriali
|
|
|
5
5
|
import { Filter as jn, GlProgram as Yn, Application as Xn, RendererType as Jn, Sprite as It, Texture as we, VideoSource as Qn, Rectangle as Zn } from "pixi.js";
|
|
6
6
|
const es = "https://aitwin.bubu.social", nn = {
|
|
7
7
|
publicBase: es
|
|
8
|
-
}, ts = "0.4.
|
|
8
|
+
}, ts = "0.4.4", rs = {
|
|
9
9
|
version: ts
|
|
10
10
|
}, ns = {}, kt = "https://ai.aitwin.me", ar = `${nn.publicBase}/custom-faces`, ss = `${nn.publicBase}/aitwin-workers/v${rs.version}/visemeDiffPreview.worker.js`;
|
|
11
11
|
function is(e) {
|
|
@@ -2878,7 +2878,7 @@ async function Qr(e, t, r, n, s) {
|
|
|
2878
2878
|
`[PixiVisemeCompositor] Sheet ${d} (${S.width}x${S.height}) exceeds MAX_TEXTURE_SIZE=${n}; using per-cell crops.`
|
|
2879
2879
|
), { mode: "cells", bitmap: S };
|
|
2880
2880
|
const m = we.from(S);
|
|
2881
|
-
return m.source.scaleMode = "nearest",
|
|
2881
|
+
return m.source.scaleMode = "nearest", { mode: "full", texture: m };
|
|
2882
2882
|
};
|
|
2883
2883
|
a.set(o.index, await u(o.index, o.path));
|
|
2884
2884
|
for (const c of i.sheets ?? [])
|