@streamoji/aitwin 0.5.5 → 0.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/aitwin.cjs +1 -1
  2. package/dist/aitwin.js +1 -1
  3. package/package.json +1 -1
package/dist/aitwin.cjs CHANGED
@@ -1,4 +1,4 @@
1
- "use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const z=require("react/jsx-runtime"),w=require("react"),Li=require("@pipecat-ai/client-js"),Cn=require("@pipecat-ai/websocket-transport"),he=require("pixi.js");var wn=typeof document<"u"?document.currentScript:null;const Oi="https://aitwin.bubu.social",Br={publicBase:Oi},Ni="0.5.5",Vi={version:Ni},Ui={},ot="https://ai.aitwin.me",Rn=`${Br.publicBase}/custom-faces`,Wi=`${Br.publicBase}/aitwin-workers/v${Vi.version}/visemeDiffPreview.worker.js`;function Hi(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:wn&&wn.tagName.toUpperCase()==="SCRIPT"&&wn.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?Ui?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():Wi}const $i="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",gn=3e3,Pt=16,ar=3e3,zi=4e3,Ki=.85,ji=8,Dn=400,Bn="eng_a7f2b9c1",tn="eng_d4e8f3a2",ct="eng_c9b1e6d4",st=ct,Fr=new Set([tn,ct]),Lr={[Bn]:"oculus_direct",[ct]:"oculus_direct",[tn]:"inworld_word"},or={google:Bn,inworld:tn,cartesia:ct};function Or(e){return Lr[e]}function qi(e){return e===ct}function Gi(e){if(e.ttsEngineId&&e.ttsEngineId in Lr)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in or?or[t]:st}class Gt extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function Nr(e,t=$i){const n=new URL(t);n.searchParams.set("id",e);const r=await fetch(n.toString());if(r.status===404)throw new Gt;if(!r.ok)throw new Error(`getAiTwin failed (${r.status})`);const i=await r.json();if(!i.success)throw new Gt(i.error??"AI twin not found");const{tts:s,token:o,...a}=i.data,l=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:Gi(i.data),authToken:l}}const Yi="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",Qi="client_6TNvp3SCs4Og0a1ijm9TommXLql1",Xi="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",Ji="6TNvp3SCs4Og0a1ijm9TommXLql1",Zi="Swaraj Mali";async function Mt(){const e=await fetch(Yi,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":Qi,"Client-Secret":Xi},body:JSON.stringify({userId:Ji,userName:Zi})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),n=t.authToken??t.token??"";if(!n.trim())throw new Error("getAuthToken returned no token");return n}async function es(e,t=ot){const n=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!n.ok)throw new Error(`/api/session-value failed (${n.status})`);const r=await n.json();if(!r.value)throw new Error("Session value response missing value");return r}function ts(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const n=new Uint8Array(t.length/2);for(let r=0;r<t.length;r+=2)n[r/2]=Number.parseInt(t.slice(r,r+2),16);return n}function Ht(e){return Uint8Array.from(e)}function ns(e,t){const n=new Uint8Array(e.length+t.length);return n.set(e,0),n.set(t,e.length),n}async function rs(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const n=new Uint8Array(e),r=Ht(n.subarray(0,12)),i=Ht(n.subarray(12,28)),s=Ht(n.subarray(28)),o=Ht(ns(s,i)),a=await crypto.subtle.importKey("raw",ts(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:r,tagLength:128},a,o)}let bn=null;const Tn=new Set;function $t(e){return e.includes("/custom-faces/")}function Yt(e){const t=e.binBase.trim().replace(/\/$/,""),n=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||$t(t)||$t(n)))return{twinBase:n,binBase:t,encrypted:!1};const i=$t(t)?t:$t(n)?n:t;return{twinBase:i,binBase:i,encrypted:!0}}function ut(e){return Yt(e).encrypted}function kn(e,t=Rn){const n=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:n,binBase:n,encrypted:!0}}function Vr(e,t){return(e??t)?.trim()||void 0}function Ur(e){bn=Yt(e);for(const t of Tn)t()}function is(e,t){const n=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!n)return Yt(e);const r=i=>{try{return new URL(i,n).href.replace(/\/$/,"")}catch{return i}};return Yt({twinBase:r(e.twinBase),binBase:r(e.binBase),encrypted:e.encrypted})}function lt(){if(!bn)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return bn}function Wr(e){return Tn.add(e),()=>Tn.delete(e)}function Qt(){const e=lt();return ut(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function ss(){const e=lt();return ut(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function Hr(){const e=lt();return ut(e)?e.binBase:e.twinBase}function Dt(){return ut(lt())}function $r(){const e=lt();return ut(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function zr(){const e=lt();return ut(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function Kr(e){const t=e.split(/\r?\n/);let n="",r="";for(const s of t)s.startsWith("event:")?n=s.slice(6).trim():s.startsWith("data:")&&(r=s.slice(5).trim());if(!n)return null;let i={};if(r)try{i=JSON.parse(r)}catch{i={raw:r}}return{event:n,data:i}}const jr=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],as=new Map(jr.map(e=>[e.toLowerCase(),e]));function nn(e){const t=(e??"").trim();return t?as.get(t.toLowerCase())??"sil":"sil"}const os={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function cs(e){if(!e)return["sil"];const t=e.toLowerCase();return os[t]??["sil"]}function qr(e,t){const n=e.word.length;if(n<=0)return 0;const r=e.wduration;if(r<=0)return 0;const i=Math.max(0,Math.min(1,(t-e.wtime)/r));return Math.min(n-1,Math.floor(i*n))}function yt(e,t){const n=Math.max(0,Math.min(e.length-1,t));return e[n]??""}function rn(e,t){const n=e.toLowerCase().replace(/[^a-z']/g,"");return!n||yt(n,t)!=="w"||t>0&&yt(n,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(n)}function us(e,t){const n=e.toLowerCase(),r=yt(n,t),i=n.slice(Math.max(0,t-1),t+2);return r==="i"||r==="y"?"I":i.includes("ee")||r==="e"||n.includes("ea")?"E":"aa"}function ls(e,t){const n=e.toLowerCase(),r=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));if(/oo|ou|uw/.test(r))return"U";if(rn(n,t))return"FF";const i=yt(n,t);return i==="u"||i==="w"?"U":"O"}function fs(e,t){const n=e.toLowerCase();if(rn(n,t))return"FF";const r=n.slice(Math.max(0,t-2),t+1);if(/qu/.test(r))return"U";const i=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));return/wo|wh/.test(i)||yt(n,t)==="o"?"O":"U"}const zt={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function ds(e,t){const n=e.toLowerCase(),r=yt(n,t);if(zt[r])return zt[r];for(let i=0;i<n.length;i++){const s=n[i];if(zt[s])return zt[s]}return"DD"}function Gr(e,t,n){const r=cs(e);if(r.length===1)return r[0];const i=t.trim();if(!i)return r[0];const s=(e??"").toLowerCase(),o=Math.max(0,n);switch(s){case"aei":return us(i,o);case"o":return ls(i,o);case"qw":return fs(i,o);case"cdgknstxyz":return ds(i,o);default:return r[0]}}const Fn=16,xn=47,Yr=new Set(["aa","E","I"]),Qr=new Set(["DD","SS","kk","CH"]),ms=[Yr,Qr];function hs(e){for(const t of ms)if(t.has(e))return t;return null}function ps(e){return hs(e)!=null}function Xr(e,t=Fn){return t<=0?e:Math.round(e/t)*t}function Ln(e,t=Fn){return e.map(n=>({...n,vtime:Xr(n.vtime,t)}))}function Jr(e){const t=new Map;for(const n of e)t.has(n.vtime)||t.set(n.vtime,n);return[...t.values()].sort((n,r)=>n.vtime-r.vtime)}function On(e){if(e.length<=2)return[...e];const t=[e[0]];for(let n=1;n<e.length-1;n++){const r=e[n],i=e[n+1],s=t[t.length-1];if(!ps(r.viseme)){t.push(r);continue}const o=r.vtime-s.vtime,a=i.vtime-r.vtime;o<xn||a<xn||t.push(r)}return t.push(e[e.length-1]),t}function sn(e){return On(Jr(Ln(e)))}function Zr(e,t){let n="sil";for(const r of e){if(r.vtime>t)break;r.viseme!=="sil"&&(n=r.viseme)}return n}function an(e){const t=Jr(Ln(e)),n=On(t),r=new Set(n.map(s=>s.vtime)),i=[];for(const s of t)r.has(s.vtime)||i.push({viseme:s.viseme,vtime:s.vtime});return i}function ws(e){return an(e).map(t=>t.viseme)}const gs=32,cr=48;function ei(e){let t=0;for(const n of e)t=Math.max(t,n.vtime+n.vduration);return t}function ti(e){let t=null,n=-1;for(const r of e){if(r.viseme==="sil")continue;const i=r.vtime+r.vduration;i>n&&(t=r,n=i)}return t}function ys(e){return sn(e).find(n=>n.viseme!=="sil")??null}function rt(e){const t=ei(e);return ti(e)?t+Dn:t}function _s(e){const t=[],n=ys(e);n&&n.vtime>0&&t.push({from:"sil",to:n.viseme,fromVtime:0,toVtime:n.vtime});const r=ti(e),i=ei(e);return r&&t.push({from:r.viseme,to:"sil",fromVtime:i,toVtime:i+Dn}),t}function ni(e){const t=sn(e),n=[];for(let r=0;r<t.length-1;r++){const i=t[r].viseme,s=t[r+1].viseme,o=t[r].vtime,a=t[r+1].vtime;i===s||a<=o||n.push({from:i,to:s,fromVtime:o,toVtime:a})}return n}function ri(e){return e.length===0?[]:[..._s(e),...ni(e)]}function vs(e,t){let n=null;for(const r of e){const{transStart:i,transEnd:s}=ii(r.fromVtime,r.toVtime,r.from,r.to);t>=i&&t<s&&(n=r)}return n}function ii(e,t,n,r){const i=Math.max(0,t-e);if((n==="PP"||r==="PP")&&i>cr){const f=cr,u=t,c=u-f;return{gapMs:i,effectiveGapMs:f,holdEnd:c,transStart:c,transEnd:u}}const o=i>0?Math.max(i,gs):0,a=e,l=e+o;return{gapMs:i,effectiveGapMs:o,holdEnd:e,transStart:a,transEnd:l}}function si(e,t,n,r,i,s){r.forEach(o=>{const l=String(o.word??"").trim();if(!l)return;const f=Math.round((o.start??0)*1e3),u=o.duration!=null?Math.round(o.duration*1e3):o.end!=null?Math.max(1,Math.round((o.end-(o.start??0))*1e3)):0;t.push({word:l,wtime:i+f,wduration:u,queueIndex:t.length,gesture:o.gesture??null,emphasis:o.emphasis===!0})}),n.forEach(o=>{const a=String(o.symbol??o.viseme??"").trim();if(!a)return;const l=Math.round((o.start??0)*1e3),f=Math.max(1,Math.round((o.duration??0)*1e3)),u=i+l,c=ci(t,u),d=c?qr(c,u):0;let p;Or(s)==="oculus_direct"?(p=nn(a),p==="U"&&c&&rn(c.word,d)&&(p="FF")):p=Gr(a,c?.word??"",d),e.push({viseme:p,weight:1,vtime:u,vduration:f})})}function ai(e,t){let n=t;for(const r of e){const i=Math.round((r.start??0)*1e3),s=Math.max(1,Math.round((r.duration??0)*1e3));n=Math.max(n,t+i+s)}return n}const Nn=16e3;function oi(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:Nn}function Ss(e){let t=e.trim();const n=t.indexOf(",");t.startsWith("data:")&&n>=0&&(t=t.slice(n+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const r=t.length%4;return r!==0&&(t+="=".repeat(4-r)),t}function bs(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function ur(e,t,n=Nn){const r=t.byteOffset%2===0?t:t.slice(),i=new Int16Array(r.buffer,r.byteOffset,r.byteLength/2),s=new Float32Array(i.length);for(let a=0;a<i.length;a++)s[a]=i[a]>=32768?-(65536-i[a])/32768:i[a]/32767;const o=e.createBuffer(1,s.length,n);return o.copyToChannel(s,0),o}function Ts(e){const t=Ss(e),n=window.atob(t),r=new Uint8Array(n.length);for(let i=0;i<n.length;i++)r[i]=n.charCodeAt(i);return r}async function lr(e,t,n=Nn){const r=Ts(t),i=oi(n);if(bs(r)){const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return e.decodeAudioData(s)}if(r.length>=2&&r.length%2===0)return ur(e,r,i);try{const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return await e.decodeAudioData(s)}catch{return ur(e,r,i)}}function ci(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function ui(e,t){const n=[],r=[];let i=null,s=!1,o=!1,a=!1,l=!1,f=0,u=0,c=0,d=null,p=1,T=t,m=!1,A=null,C=0,x=!1,g=!1,E=0,N=null,I=null,te=null;const de=()=>({visemes:[],words:[],segmentOffsetMs:0}),P=()=>{g=!1,E=0},W=()=>{d!=null&&(clearTimeout(d),d=null)},K=()=>{for(const S of r){try{S.onended=null,S.stop(0)}catch{}try{S.disconnect()}catch{}}r.length=0},X=()=>{for(const S of r)S.playbackRate.value=p},Y=()=>{if(!(m||!l||s||g||r.length>0||n.length>0)){if(a&&i&&u>0){const S=(i.currentTime-u)*1e3;if(S<C){W();const $=p>0?(C-S)/p:C-S;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,$));return}}o=!1,W(),e.onSpeakingDone(),e.onNotify()}},pe=S=>{const $=rt(S);if($<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,f=performance.now(),e.onSpeakingStart(),e.onNotify(),W();const b=p>0?$/p:$;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},b)},j=(S,$,b,L)=>{$.length!==0&&(c=b,$.forEach(B=>{const _=String(B.word??"").trim();if(!_)return;const v=Math.round((B.start??0)*1e3),k=B.duration!=null?Math.round(B.duration*1e3):B.end!=null?Math.max(1,Math.round((B.end-(B.start??0))*1e3)):0;S.push({word:_,wtime:c+v,wduration:k,queueIndex:S.length,gesture:B.gesture??null,emphasis:B.emphasis===!0})}))},Z=(S,$,b,L,B,O)=>{b.length!==0&&(c=L,b.forEach(_=>{const v=String(_.symbol??_.viseme??"").trim();if(!v)return;const k=Math.round((_.start??0)*1e3),H=Math.max(1,Math.round((_.duration??0)*1e3)),J=c+k,ie=ci($,J),xe=ie?qr(ie,J):0;let be;Or(O)==="oculus_direct"?(be=nn(v),be==="U"&&ie&&rn(ie.word,xe)&&(be="FF")):be=Gr(v,ie?.word??"",xe),S.push({viseme:be,weight:1,vtime:J,vduration:H})}),e.onNotify())},M=(S,$,b,L,B,O,_)=>{j($,L,B),Z(S,$,b,B,O,_)},G=(S,$,b)=>{te===S&&S.visemes===$||($.length=0,b.length=0,S.visemes=$,S.words=b,S.segmentOffsetMs=0,te=S,c=0,C=0)},V=async()=>{const S=window.AudioContext||window.webkitAudioContext;return i||(i=new S),i.state==="suspended"&&await i.resume(),A||(A=i.createMediaStreamDestination()),i},ue=(S,$)=>{if(!i)return 0;const b=i,L=b.currentTime,B=L+.02;if($.isFirstAudio){u<=0&&(u=B);const _=(B-L)*1e3;f=performance.now()+_}$.onBeforeStart?.(B);const O=b.createBufferSource();return O.buffer=S,O.playbackRate.value=p,O.connect(b.destination),A&&O.connect(A),r.push(O),m?(r.pop(),B):($.isFirstAudio&&(o=!0,e.onSpeakingStart()),O.onended=()=>{const _=r.indexOf(O);_>=0&&r.splice(_,1),e.onNotify(),!m&&(N&&I?ne(N,I):Y())},O.start(B),B)},ne=async(S,$)=>{if(m||s||r.length>0)return;if(n.length===0){Y();return}s=!0;let b=!1;try{const L=await V();if(m)return;const B=n[0];if(!B)return;const O=!o;if(x&&(e.shouldHoldForWarmup?.()??!1)&&(g||O&&B.newStream)||g){G(B.utterance,S,$);for(const k of n){if(m)return;k.audioBuffer||(k.audioBuffer=await lr(L,k.audio,k.sampleRate)),a=!0,g||(g=!0,E=0,k.newStream&&k.onNewStream?.()),k.visemesQueued||(M(S,$,k.visemes,k.words,E,!0,T),k.visemesQueued=!0,E+=Math.round(k.audioBuffer.duration*1e3),C=Math.max(C,rt(S)))}e.onNotify();return}const v=n.shift();if(!v||(v.audioBuffer||(v.audioBuffer=await lr(L,v.audio,v.sampleRate)),m))return;a=!0,W(),G(v.utterance,S,$),v.visemesQueued=!1,v.newStream&&v.onNewStream?.(),ue(v.audioBuffer,{isFirstAudio:!o,isFirstOfStream:!1,onBeforeStart:k=>{u=k,c=0,f=performance.now()+(k-L.currentTime)*1e3,S.length=0,$.length=0,M(S,$,v.visemes,v.words,0,!0,T),v.visemesQueued=!0,v.utterance.segmentOffsetMs=0}}),b=r.length>0,C=rt(S),e.onNotify()}catch(L){console.error("[avatarChunkPlayback] audio chunk failed:",L)}finally{if(s=!1,m)return;if(g){n.some(B=>!B.audioBuffer||!B.visemesQueued)&&ne(S,$);return}!b&&r.length===0&&n.length>0?ne(S,$):r.length===0&&n.length===0&&Y()}};return{setStreamTtsEngineId:S=>{T=S},setPlaybackSpeed:S=>{p=Math.max(.1,Math.min(1,S)),X()},getPlaybackSpeed:()=>p,setLipsyncWarmupHoldEnabled:S=>{x=S,S||P()},isWarmingUp:()=>g,releaseWarmupHold:()=>{if(g){g=!1,E=0;for(const S of n)S.newStream=!1;N&&I&&ne(N,I),e.onNotify()}},resetForNewStream:S=>{m=!1,l=!1,T=S,C=0},resumeBotStream:()=>{m=!1,l=!1},clearAborted:()=>{m=!1},primeAudioContext:async()=>{const S=window.AudioContext||window.webkitAudioContext;i||(i=new S),i.state==="suspended"&&await i.resume(),i&&!A&&(A=i.createMediaStreamDestination())},stop:()=>{m=!0,W(),K(),n.length=0,s=!1,o=!1,a=!1,l=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},markStreamComplete:S=>{if(C=Math.max(C,rt(S)),l=!0,!a&&S.length>0){pe(S);return}Y()},playAudioChunk:async(S,$,b,L,B,O,_)=>{if(m)return;N=S,I=$;const v=oi(_?.sampleRate),k=_?.newStream===!0,H=de();if(!b?.trim()){if(L.length===0&&B.length===0)return;G(H,S,$),k&&_?.onNewStream?.(),M(S,$,L,B,0,!0,T),C=rt(S),!o&&C>0&&pe(S),e.onNotify();return}n.push({audio:b,audioBuffer:null,utterance:H,visemes:L,words:B,isNewSegment:O,sampleRate:v,newStream:k,onNewStream:_?.onNewStream,visemesQueued:!1}),await ne(S,$)},queueTimingsOnly:(S,$,b,L,B,O)=>{if(m)return;const _=de();G(_,S,$),M(S,$,b,L,0,!0,T),_.segmentOffsetMs=0,C=rt(S)},getPlaybackElapsedMs:()=>a&&i&&u>0?Math.max(0,(i.currentTime-u)*1e3*p):o?Math.max(0,(performance.now()-f)*p):0,isSpeaking:()=>o?!0:!a||!i||u<=0?!1:(i.currentTime-u)*1e3<C,getVoiceStream:()=>A?A.stream:null,getPlaybackStartPerf:()=>f,bargeIn:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},interrupt:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()}}}function Bt(e,t){let n=null;for(const r of e)t<r.vtime||t>=r.vtime+r.vduration||(!n||r.vtime>=n.vtime)&&(n=r);return n?n.viseme:"sil"}function li(e,t,n=120){const r=Bt(e,t);if(r!=="sil")return r;let i=null;for(const s of e){if(s.vtime>t)break;const o=s.vtime+s.vduration;o<=t&&t-o<=n&&(i=s)}return i?.viseme??"sil"}function ks(e,t,n=400){const r=t-n;let i=0;for(const s of e)if(s.vtime+s.vduration<r)i+=1;else break;i>0&&e.splice(0,i)}function xs(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function fr(e,t){const n=e.trim();if(!n)return null;try{const r=n.replace(/\s+/g,""),i=atob(r).length,s=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return i/2/s*1e3}catch{return null}}const on=.5,cn=1.5,Vn="Olivia",Un="f786b574-daa5-4673-aa0c-cbe3e8534c02",Ft=.85;function En(e,t=ot,n={}){const r=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,i=[],s=[],o=[],a=[];let l=0;const f=[],u=[];let c="",d="neutral",p="",T=st;const m=new Set;let A=0,C=null,x=0,g=null;const E=()=>m.forEach(j=>j());let N=()=>!1;const I=ui({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:E,shouldHoldForWarmup:()=>N()},st),te=()=>I.getPlaybackElapsedMs(),de=(j,Z=c)=>{if(!n.onLipsyncDebug)return;const M=te(),G=i,V=an(o).map(ue=>({viseme:ue.viseme,vtimeMs:ue.vtime}));n.onLipsyncDebug({event:j,spokenText:Z,mood:d,playbackAnchorPerf:I.getPlaybackStartPerf(),playbackElapsedMs:M,queueLength:G.length,activeViseme:Bt(G,M),segments:u.map(ue=>({...ue,visemes:[...ue.visemes]})),fullQueueLength:o.length,compactionSkipped:V})},P=(j,Z)=>{n.onLipsyncDebug&&(C={event:j,spokenText:Z},A===0&&(A=requestAnimationFrame(()=>{A=0;const M=C;C=null,M&&de(M.event,M.spokenText)})))},W=(j,Z,M,G,V)=>{if(!n.onLipsyncDebug)return;const ue=i.slice(Z),ne=M.filter(ae=>String(ae.symbol??ae.viseme??"").trim());u.push({spokenText:j,audioOffsetMs:ue[0]?.vtime??0,isNewSegment:G,chunkDurationMs:V,receivedAtMs:performance.now(),visemes:ue.map((ae,we)=>{const oe=ne[we];return{rawSymbol:String(oe?.symbol??oe?.viseme??ae.viseme).trim(),viseme:ae.viseme,startSec:Number(oe?.start??0),durationSec:Number(oe?.duration??0),vtimeMs:ae.vtime,vdurationMs:ae.vduration}})})},K=()=>{o.length=0,a.length=0,l=0},X=(j,Z,M,G,V)=>{if(j.length===0&&Z.length===0)return;si(o,a,j,Z,l,V);const ue=ai(j,l),ne=M?fr(M,G):null;l=Math.max(ue,ne!=null?l+ne:l)},Y=()=>{x+=1,g?.abort(),g=null,I.stop(),i.length=0,s.length=0,K(),f.length=0,d="neutral",u.length=0,c="",e("idle"),P("reset"),E()};return{setDeveloperToken:j=>{p=j},setTtsEngineId:j=>{T=j},getTtsEngineId:()=>T,speak:async(j,Z)=>{Y();const M=x;g=new AbortController,e("loading"),c=j,u.length=0,K();const G=p.trim();if(!G)throw new Error("Developer token required for avatar_ttsWithPoses");const V=Z?.ttsEngineId??T??st;I.resetForNewStream(V),I.setStreamTtsEngineId(V);const ue=Z?.voiceId??Vn,ne=Z?.speakingRate??Ft,ae=Math.max(on,Math.min(cn,ne)),we={user_query:j,ttsEngineId:V,speakingRate:ae,tenant:"aiTwin"};Fr.has(V)&&(we.voice_id=ue);try{const oe=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${G}`},body:JSON.stringify(we),signal:g.signal});if(!oe.ok)throw new Error(`avatar_ttsWithPoses failed (${oe.status})`);const S=oe.body;if(!S)throw new Error("No response body");const $=S.getReader(),b=new TextDecoder;let L="",B=!0;const O=async _=>{if(M!==x)return;const v=Kr(_);if(v){if(v.event==="audio"){const k=v.data.chunk,H=v.data.visemes??[],J=v.data.words??[],ie=v.data.is_new_segment??!1,xe=v.data.sample_rate,be=Number(v.data.audio_offset_ms??0),Re=i.length;if(X(H,J,k,xe,V),k){const De=B;B=!1,await I.playAudioChunk(i,s,k,H,J,ie,{sampleRate:xe,newStream:De,audioOffsetMs:be}),W(j,Re,H,ie,fr(k,xe)),P("audio_chunk",j)}else(H.length>0||J.length>0)&&(I.queueTimingsOnly(i,s,H,J,ie,{audioOffsetMs:be,newStream:B}),W(j,Re,H,ie,null),P("audio_chunk",j))}else if(v.event==="metadata"){const k=v.data.mood;typeof k=="string"&&k.trim()&&(d=k.trim());const H=v.data.sentence_emotions;if(Array.isArray(H)){f.length=0;for(const J of H){if(!J||typeof J!="object")continue;const ie=J;f.push({sentence_index:Number(ie.sentence_index??0),text:String(ie.text??""),sentiment:typeof ie.sentiment=="string"?ie.sentiment:void 0,emotion:String(ie.emotion??d),start_word:Number(ie.start_word??0),end_word:Number(ie.end_word??0)})}}E(),P("metadata",j)}else if(v.event==="error")throw new Error(String(v.data.message??"avatar_ttsWithPoses stream error"))}};for(;M===x;){const{done:_,value:v}=await $.read();v&&(L+=b.decode(v,{stream:!0}));const k=L.split(`
1
+ "use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const z=require("react/jsx-runtime"),w=require("react"),Li=require("@pipecat-ai/client-js"),Cn=require("@pipecat-ai/websocket-transport"),he=require("pixi.js");var wn=typeof document<"u"?document.currentScript:null;const Oi="https://aitwin.bubu.social",Br={publicBase:Oi},Ni="0.5.6",Vi={version:Ni},Ui={},ot="https://ai.aitwin.me",Rn=`${Br.publicBase}/custom-faces`,Wi=`${Br.publicBase}/aitwin-workers/v${Vi.version}/visemeDiffPreview.worker.js`;function Hi(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:wn&&wn.tagName.toUpperCase()==="SCRIPT"&&wn.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?Ui?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():Wi}const $i="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",gn=3e3,Pt=16,ar=3e3,zi=4e3,Ki=.85,ji=8,Dn=400,Bn="eng_a7f2b9c1",tn="eng_d4e8f3a2",ct="eng_c9b1e6d4",st=ct,Fr=new Set([tn,ct]),Lr={[Bn]:"oculus_direct",[ct]:"oculus_direct",[tn]:"inworld_word"},or={google:Bn,inworld:tn,cartesia:ct};function Or(e){return Lr[e]}function qi(e){return e===ct}function Gi(e){if(e.ttsEngineId&&e.ttsEngineId in Lr)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in or?or[t]:st}class Gt extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function Nr(e,t=$i){const n=new URL(t);n.searchParams.set("id",e);const r=await fetch(n.toString());if(r.status===404)throw new Gt;if(!r.ok)throw new Error(`getAiTwin failed (${r.status})`);const i=await r.json();if(!i.success)throw new Gt(i.error??"AI twin not found");const{tts:s,token:o,...a}=i.data,l=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:Gi(i.data),authToken:l}}const Yi="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",Qi="client_6TNvp3SCs4Og0a1ijm9TommXLql1",Xi="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",Ji="6TNvp3SCs4Og0a1ijm9TommXLql1",Zi="Swaraj Mali";async function Mt(){const e=await fetch(Yi,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":Qi,"Client-Secret":Xi},body:JSON.stringify({userId:Ji,userName:Zi})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),n=t.authToken??t.token??"";if(!n.trim())throw new Error("getAuthToken returned no token");return n}async function es(e,t=ot){const n=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!n.ok)throw new Error(`/api/session-value failed (${n.status})`);const r=await n.json();if(!r.value)throw new Error("Session value response missing value");return r}function ts(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const n=new Uint8Array(t.length/2);for(let r=0;r<t.length;r+=2)n[r/2]=Number.parseInt(t.slice(r,r+2),16);return n}function Ht(e){return Uint8Array.from(e)}function ns(e,t){const n=new Uint8Array(e.length+t.length);return n.set(e,0),n.set(t,e.length),n}async function rs(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const n=new Uint8Array(e),r=Ht(n.subarray(0,12)),i=Ht(n.subarray(12,28)),s=Ht(n.subarray(28)),o=Ht(ns(s,i)),a=await crypto.subtle.importKey("raw",ts(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:r,tagLength:128},a,o)}let bn=null;const Tn=new Set;function $t(e){return e.includes("/custom-faces/")}function Yt(e){const t=e.binBase.trim().replace(/\/$/,""),n=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||$t(t)||$t(n)))return{twinBase:n,binBase:t,encrypted:!1};const i=$t(t)?t:$t(n)?n:t;return{twinBase:i,binBase:i,encrypted:!0}}function ut(e){return Yt(e).encrypted}function kn(e,t=Rn){const n=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:n,binBase:n,encrypted:!0}}function Vr(e,t){return(e??t)?.trim()||void 0}function Ur(e){bn=Yt(e);for(const t of Tn)t()}function is(e,t){const n=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!n)return Yt(e);const r=i=>{try{return new URL(i,n).href.replace(/\/$/,"")}catch{return i}};return Yt({twinBase:r(e.twinBase),binBase:r(e.binBase),encrypted:e.encrypted})}function lt(){if(!bn)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return bn}function Wr(e){return Tn.add(e),()=>Tn.delete(e)}function Qt(){const e=lt();return ut(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function ss(){const e=lt();return ut(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function Hr(){const e=lt();return ut(e)?e.binBase:e.twinBase}function Dt(){return ut(lt())}function $r(){const e=lt();return ut(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function zr(){const e=lt();return ut(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function Kr(e){const t=e.split(/\r?\n/);let n="",r="";for(const s of t)s.startsWith("event:")?n=s.slice(6).trim():s.startsWith("data:")&&(r=s.slice(5).trim());if(!n)return null;let i={};if(r)try{i=JSON.parse(r)}catch{i={raw:r}}return{event:n,data:i}}const jr=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],as=new Map(jr.map(e=>[e.toLowerCase(),e]));function nn(e){const t=(e??"").trim();return t?as.get(t.toLowerCase())??"sil":"sil"}const os={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function cs(e){if(!e)return["sil"];const t=e.toLowerCase();return os[t]??["sil"]}function qr(e,t){const n=e.word.length;if(n<=0)return 0;const r=e.wduration;if(r<=0)return 0;const i=Math.max(0,Math.min(1,(t-e.wtime)/r));return Math.min(n-1,Math.floor(i*n))}function yt(e,t){const n=Math.max(0,Math.min(e.length-1,t));return e[n]??""}function rn(e,t){const n=e.toLowerCase().replace(/[^a-z']/g,"");return!n||yt(n,t)!=="w"||t>0&&yt(n,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(n)}function us(e,t){const n=e.toLowerCase(),r=yt(n,t),i=n.slice(Math.max(0,t-1),t+2);return r==="i"||r==="y"?"I":i.includes("ee")||r==="e"||n.includes("ea")?"E":"aa"}function ls(e,t){const n=e.toLowerCase(),r=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));if(/oo|ou|uw/.test(r))return"U";if(rn(n,t))return"FF";const i=yt(n,t);return i==="u"||i==="w"?"U":"O"}function fs(e,t){const n=e.toLowerCase();if(rn(n,t))return"FF";const r=n.slice(Math.max(0,t-2),t+1);if(/qu/.test(r))return"U";const i=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));return/wo|wh/.test(i)||yt(n,t)==="o"?"O":"U"}const zt={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function ds(e,t){const n=e.toLowerCase(),r=yt(n,t);if(zt[r])return zt[r];for(let i=0;i<n.length;i++){const s=n[i];if(zt[s])return zt[s]}return"DD"}function Gr(e,t,n){const r=cs(e);if(r.length===1)return r[0];const i=t.trim();if(!i)return r[0];const s=(e??"").toLowerCase(),o=Math.max(0,n);switch(s){case"aei":return us(i,o);case"o":return ls(i,o);case"qw":return fs(i,o);case"cdgknstxyz":return ds(i,o);default:return r[0]}}const Fn=16,xn=47,Yr=new Set(["aa","E","I"]),Qr=new Set(["DD","SS","kk","CH"]),ms=[Yr,Qr];function hs(e){for(const t of ms)if(t.has(e))return t;return null}function ps(e){return hs(e)!=null}function Xr(e,t=Fn){return t<=0?e:Math.round(e/t)*t}function Ln(e,t=Fn){return e.map(n=>({...n,vtime:Xr(n.vtime,t)}))}function Jr(e){const t=new Map;for(const n of e)t.has(n.vtime)||t.set(n.vtime,n);return[...t.values()].sort((n,r)=>n.vtime-r.vtime)}function On(e){if(e.length<=2)return[...e];const t=[e[0]];for(let n=1;n<e.length-1;n++){const r=e[n],i=e[n+1],s=t[t.length-1];if(!ps(r.viseme)){t.push(r);continue}const o=r.vtime-s.vtime,a=i.vtime-r.vtime;o<xn||a<xn||t.push(r)}return t.push(e[e.length-1]),t}function sn(e){return On(Jr(Ln(e)))}function Zr(e,t){let n="sil";for(const r of e){if(r.vtime>t)break;r.viseme!=="sil"&&(n=r.viseme)}return n}function an(e){const t=Jr(Ln(e)),n=On(t),r=new Set(n.map(s=>s.vtime)),i=[];for(const s of t)r.has(s.vtime)||i.push({viseme:s.viseme,vtime:s.vtime});return i}function ws(e){return an(e).map(t=>t.viseme)}const gs=32,cr=48;function ei(e){let t=0;for(const n of e)t=Math.max(t,n.vtime+n.vduration);return t}function ti(e){let t=null,n=-1;for(const r of e){if(r.viseme==="sil")continue;const i=r.vtime+r.vduration;i>n&&(t=r,n=i)}return t}function ys(e){return sn(e).find(n=>n.viseme!=="sil")??null}function rt(e){const t=ei(e);return ti(e)?t+Dn:t}function _s(e){const t=[],n=ys(e);n&&n.vtime>0&&t.push({from:"sil",to:n.viseme,fromVtime:0,toVtime:n.vtime});const r=ti(e),i=ei(e);return r&&t.push({from:r.viseme,to:"sil",fromVtime:i,toVtime:i+Dn}),t}function ni(e){const t=sn(e),n=[];for(let r=0;r<t.length-1;r++){const i=t[r].viseme,s=t[r+1].viseme,o=t[r].vtime,a=t[r+1].vtime;i===s||a<=o||n.push({from:i,to:s,fromVtime:o,toVtime:a})}return n}function ri(e){return e.length===0?[]:[..._s(e),...ni(e)]}function vs(e,t){let n=null;for(const r of e){const{transStart:i,transEnd:s}=ii(r.fromVtime,r.toVtime,r.from,r.to);t>=i&&t<s&&(n=r)}return n}function ii(e,t,n,r){const i=Math.max(0,t-e);if((n==="PP"||r==="PP")&&i>cr){const f=cr,u=t,c=u-f;return{gapMs:i,effectiveGapMs:f,holdEnd:c,transStart:c,transEnd:u}}const o=i>0?Math.max(i,gs):0,a=e,l=e+o;return{gapMs:i,effectiveGapMs:o,holdEnd:e,transStart:a,transEnd:l}}function si(e,t,n,r,i,s){r.forEach(o=>{const l=String(o.word??"").trim();if(!l)return;const f=Math.round((o.start??0)*1e3),u=o.duration!=null?Math.round(o.duration*1e3):o.end!=null?Math.max(1,Math.round((o.end-(o.start??0))*1e3)):0;t.push({word:l,wtime:i+f,wduration:u,queueIndex:t.length,gesture:o.gesture??null,emphasis:o.emphasis===!0})}),n.forEach(o=>{const a=String(o.symbol??o.viseme??"").trim();if(!a)return;const l=Math.round((o.start??0)*1e3),f=Math.max(1,Math.round((o.duration??0)*1e3)),u=i+l,c=ci(t,u),d=c?qr(c,u):0;let p;Or(s)==="oculus_direct"?(p=nn(a),p==="U"&&c&&rn(c.word,d)&&(p="FF")):p=Gr(a,c?.word??"",d),e.push({viseme:p,weight:1,vtime:u,vduration:f})})}function ai(e,t){let n=t;for(const r of e){const i=Math.round((r.start??0)*1e3),s=Math.max(1,Math.round((r.duration??0)*1e3));n=Math.max(n,t+i+s)}return n}const Nn=16e3;function oi(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:Nn}function Ss(e){let t=e.trim();const n=t.indexOf(",");t.startsWith("data:")&&n>=0&&(t=t.slice(n+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const r=t.length%4;return r!==0&&(t+="=".repeat(4-r)),t}function bs(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function ur(e,t,n=Nn){const r=t.byteOffset%2===0?t:t.slice(),i=new Int16Array(r.buffer,r.byteOffset,r.byteLength/2),s=new Float32Array(i.length);for(let a=0;a<i.length;a++)s[a]=i[a]>=32768?-(65536-i[a])/32768:i[a]/32767;const o=e.createBuffer(1,s.length,n);return o.copyToChannel(s,0),o}function Ts(e){const t=Ss(e),n=window.atob(t),r=new Uint8Array(n.length);for(let i=0;i<n.length;i++)r[i]=n.charCodeAt(i);return r}async function lr(e,t,n=Nn){const r=Ts(t),i=oi(n);if(bs(r)){const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return e.decodeAudioData(s)}if(r.length>=2&&r.length%2===0)return ur(e,r,i);try{const s=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return await e.decodeAudioData(s)}catch{return ur(e,r,i)}}function ci(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function ui(e,t){const n=[],r=[];let i=null,s=!1,o=!1,a=!1,l=!1,f=0,u=0,c=0,d=null,p=1,T=t,m=!1,A=null,C=0,x=!1,g=!1,E=0,N=null,I=null,te=null;const de=()=>({visemes:[],words:[],segmentOffsetMs:0}),P=()=>{g=!1,E=0},W=()=>{d!=null&&(clearTimeout(d),d=null)},K=()=>{for(const S of r){try{S.onended=null,S.stop(0)}catch{}try{S.disconnect()}catch{}}r.length=0},X=()=>{for(const S of r)S.playbackRate.value=p},Y=()=>{if(!(m||!l||s||g||r.length>0||n.length>0)){if(a&&i&&u>0){const S=(i.currentTime-u)*1e3;if(S<C){W();const $=p>0?(C-S)/p:C-S;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,$));return}}o=!1,W(),e.onSpeakingDone(),e.onNotify()}},pe=S=>{const $=rt(S);if($<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,f=performance.now(),e.onSpeakingStart(),e.onNotify(),W();const b=p>0?$/p:$;d=setTimeout(()=>{m||(o=!1,d=null,e.onSpeakingDone(),e.onNotify())},b)},j=(S,$,b,L)=>{$.length!==0&&(c=b,$.forEach(B=>{const _=String(B.word??"").trim();if(!_)return;const v=Math.round((B.start??0)*1e3),k=B.duration!=null?Math.round(B.duration*1e3):B.end!=null?Math.max(1,Math.round((B.end-(B.start??0))*1e3)):0;S.push({word:_,wtime:c+v,wduration:k,queueIndex:S.length,gesture:B.gesture??null,emphasis:B.emphasis===!0})}))},Z=(S,$,b,L,B,O)=>{b.length!==0&&(c=L,b.forEach(_=>{const v=String(_.symbol??_.viseme??"").trim();if(!v)return;const k=Math.round((_.start??0)*1e3),H=Math.max(1,Math.round((_.duration??0)*1e3)),J=c+k,ie=ci($,J),xe=ie?qr(ie,J):0;let be;Or(O)==="oculus_direct"?(be=nn(v),be==="U"&&ie&&rn(ie.word,xe)&&(be="FF")):be=Gr(v,ie?.word??"",xe),S.push({viseme:be,weight:1,vtime:J,vduration:H})}),e.onNotify())},M=(S,$,b,L,B,O,_)=>{j($,L,B),Z(S,$,b,B,O,_)},G=(S,$,b)=>{te===S&&S.visemes===$||($.length=0,b.length=0,S.visemes=$,S.words=b,S.segmentOffsetMs=0,te=S,c=0,C=0)},V=async()=>{const S=window.AudioContext||window.webkitAudioContext;return i||(i=new S),i.state==="suspended"&&await i.resume(),A||(A=i.createMediaStreamDestination()),i},ue=(S,$)=>{if(!i)return 0;const b=i,L=b.currentTime,B=L+.02;if($.isFirstAudio){u<=0&&(u=B);const _=(B-L)*1e3;f=performance.now()+_}$.onBeforeStart?.(B);const O=b.createBufferSource();return O.buffer=S,O.playbackRate.value=p,O.connect(b.destination),A&&O.connect(A),r.push(O),m?(r.pop(),B):($.isFirstAudio&&(o=!0,e.onSpeakingStart()),O.onended=()=>{const _=r.indexOf(O);_>=0&&r.splice(_,1),e.onNotify(),!m&&(N&&I?ne(N,I):Y())},O.start(B),B)},ne=async(S,$)=>{if(m||s||r.length>0)return;if(n.length===0){Y();return}s=!0;let b=!1;try{const L=await V();if(m)return;const B=n[0];if(!B)return;const O=!o;if(x&&(e.shouldHoldForWarmup?.()??!1)&&(g||O&&B.newStream)||g){G(B.utterance,S,$);for(const k of n){if(m)return;k.audioBuffer||(k.audioBuffer=await lr(L,k.audio,k.sampleRate)),a=!0,g||(g=!0,E=0,k.newStream&&k.onNewStream?.()),k.visemesQueued||(M(S,$,k.visemes,k.words,E,!0,T),k.visemesQueued=!0,E+=Math.round(k.audioBuffer.duration*1e3),C=Math.max(C,rt(S)))}e.onNotify();return}const v=n.shift();if(!v||(v.audioBuffer||(v.audioBuffer=await lr(L,v.audio,v.sampleRate)),m))return;a=!0,W(),G(v.utterance,S,$),v.visemesQueued=!1,v.newStream&&v.onNewStream?.(),ue(v.audioBuffer,{isFirstAudio:!o,isFirstOfStream:!1,onBeforeStart:k=>{u=k,c=0,f=performance.now()+(k-L.currentTime)*1e3,S.length=0,$.length=0,M(S,$,v.visemes,v.words,0,!0,T),v.visemesQueued=!0,v.utterance.segmentOffsetMs=0}}),b=r.length>0,C=rt(S),e.onNotify()}catch(L){console.error("[avatarChunkPlayback] audio chunk failed:",L)}finally{if(s=!1,m)return;if(g){n.some(B=>!B.audioBuffer||!B.visemesQueued)&&ne(S,$);return}!b&&r.length===0&&n.length>0?ne(S,$):r.length===0&&n.length===0&&Y()}};return{setStreamTtsEngineId:S=>{T=S},setPlaybackSpeed:S=>{p=Math.max(.1,Math.min(1,S)),X()},getPlaybackSpeed:()=>p,setLipsyncWarmupHoldEnabled:S=>{x=S,S||P()},isWarmingUp:()=>g,releaseWarmupHold:()=>{if(g){g=!1,E=0;for(const S of n)S.newStream=!1;N&&I&&ne(N,I),e.onNotify()}},resetForNewStream:S=>{m=!1,l=!1,T=S,C=0},resumeBotStream:()=>{m=!1,l=!1},clearAborted:()=>{m=!1},primeAudioContext:async()=>{const S=window.AudioContext||window.webkitAudioContext;i||(i=new S),i.state==="suspended"&&await i.resume(),i&&!A&&(A=i.createMediaStreamDestination())},stop:()=>{m=!0,W(),K(),n.length=0,s=!1,o=!1,a=!1,l=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},markStreamComplete:S=>{if(C=Math.max(C,rt(S)),l=!0,!a&&S.length>0){pe(S);return}Y()},playAudioChunk:async(S,$,b,L,B,O,_)=>{if(m)return;N=S,I=$;const v=oi(_?.sampleRate),k=_?.newStream===!0,H=de();if(!b?.trim()){if(L.length===0&&B.length===0)return;G(H,S,$),k&&_?.onNewStream?.(),M(S,$,L,B,0,!0,T),C=rt(S),!o&&C>0&&pe(S),e.onNotify();return}n.push({audio:b,audioBuffer:null,utterance:H,visemes:L,words:B,isNewSegment:O,sampleRate:v,newStream:k,onNewStream:_?.onNewStream,visemesQueued:!1}),await ne(S,$)},queueTimingsOnly:(S,$,b,L,B,O)=>{if(m)return;const _=de();G(_,S,$),M(S,$,b,L,0,!0,T),_.segmentOffsetMs=0,C=rt(S)},getPlaybackElapsedMs:()=>a&&i&&u>0?Math.max(0,(i.currentTime-u)*1e3*p):o?Math.max(0,(performance.now()-f)*p):0,isSpeaking:()=>o?!0:!a||!i||u<=0?!1:(i.currentTime-u)*1e3<C,getVoiceStream:()=>A?A.stream:null,getPlaybackStartPerf:()=>f,bargeIn:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()},interrupt:()=>{m=!0,l=!1,W(),K(),n.length=0,s=!1,o=!1,a=!1,c=0,u=0,f=0,C=0,N=null,I=null,te=null,P()}}}function Bt(e,t){let n=null;for(const r of e)t<r.vtime||t>=r.vtime+r.vduration||(!n||r.vtime>=n.vtime)&&(n=r);return n?n.viseme:"sil"}function li(e,t,n=120){const r=Bt(e,t);if(r!=="sil")return r;let i=null;for(const s of e){if(s.vtime>t)break;const o=s.vtime+s.vduration;o<=t&&t-o<=n&&(i=s)}return i?.viseme??"sil"}function ks(e,t,n=400){const r=t-n;let i=0;for(const s of e)if(s.vtime+s.vduration<r)i+=1;else break;i>0&&e.splice(0,i)}function xs(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function fr(e,t){const n=e.trim();if(!n)return null;try{const r=n.replace(/\s+/g,""),i=atob(r).length,s=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return i/2/s*1e3}catch{return null}}const on=.5,cn=1.5,Vn="Olivia",Un="f786b574-daa5-4673-aa0c-cbe3e8534c02",Ft=.85;function En(e,t=ot,n={}){const r=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,i=[],s=[],o=[],a=[];let l=0;const f=[],u=[];let c="",d="neutral",p="",T=st;const m=new Set;let A=0,C=null,x=0,g=null;const E=()=>m.forEach(j=>j());let N=()=>!1;const I=ui({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:E,shouldHoldForWarmup:()=>N()},st),te=()=>I.getPlaybackElapsedMs(),de=(j,Z=c)=>{if(!n.onLipsyncDebug)return;const M=te(),G=i,V=an(o).map(ue=>({viseme:ue.viseme,vtimeMs:ue.vtime}));n.onLipsyncDebug({event:j,spokenText:Z,mood:d,playbackAnchorPerf:I.getPlaybackStartPerf(),playbackElapsedMs:M,queueLength:G.length,activeViseme:Bt(G,M),segments:u.map(ue=>({...ue,visemes:[...ue.visemes]})),fullQueueLength:o.length,compactionSkipped:V})},P=(j,Z)=>{n.onLipsyncDebug&&(C={event:j,spokenText:Z},A===0&&(A=requestAnimationFrame(()=>{A=0;const M=C;C=null,M&&de(M.event,M.spokenText)})))},W=(j,Z,M,G,V)=>{if(!n.onLipsyncDebug)return;const ue=i.slice(Z),ne=M.filter(ae=>String(ae.symbol??ae.viseme??"").trim());u.push({spokenText:j,audioOffsetMs:ue[0]?.vtime??0,isNewSegment:G,chunkDurationMs:V,receivedAtMs:performance.now(),visemes:ue.map((ae,we)=>{const oe=ne[we];return{rawSymbol:String(oe?.symbol??oe?.viseme??ae.viseme).trim(),viseme:ae.viseme,startSec:Number(oe?.start??0),durationSec:Number(oe?.duration??0),vtimeMs:ae.vtime,vdurationMs:ae.vduration}})})},K=()=>{o.length=0,a.length=0,l=0},X=(j,Z,M,G,V)=>{if(j.length===0&&Z.length===0)return;si(o,a,j,Z,l,V);const ue=ai(j,l),ne=M?fr(M,G):null;l=Math.max(ue,ne!=null?l+ne:l)},Y=()=>{x+=1,g?.abort(),g=null,I.stop(),i.length=0,s.length=0,K(),f.length=0,d="neutral",u.length=0,c="",e("idle"),P("reset"),E()};return{setDeveloperToken:j=>{p=j},setTtsEngineId:j=>{T=j},getTtsEngineId:()=>T,speak:async(j,Z)=>{Y();const M=x;g=new AbortController,e("loading"),c=j,u.length=0,K();const G=p.trim();if(!G)throw new Error("Developer token required for avatar_ttsWithPoses");const V=Z?.ttsEngineId??T??st;I.resetForNewStream(V),I.setStreamTtsEngineId(V);const ue=Z?.voiceId??Vn,ne=Z?.speakingRate??Ft,ae=Math.max(on,Math.min(cn,ne)),we={user_query:j,ttsEngineId:V,speakingRate:ae,tenant:"aiTwin"};Fr.has(V)&&(we.voice_id=ue);try{const oe=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${G}`},body:JSON.stringify(we),signal:g.signal});if(!oe.ok)throw new Error(`avatar_ttsWithPoses failed (${oe.status})`);const S=oe.body;if(!S)throw new Error("No response body");const $=S.getReader(),b=new TextDecoder;let L="",B=!0;const O=async _=>{if(M!==x)return;const v=Kr(_);if(v){if(v.event==="audio"){const k=v.data.chunk,H=v.data.visemes??[],J=v.data.words??[],ie=v.data.is_new_segment??!1,xe=v.data.sample_rate,be=Number(v.data.audio_offset_ms??0),Re=i.length;if(X(H,J,k,xe,V),k){const De=B;B=!1,await I.playAudioChunk(i,s,k,H,J,ie,{sampleRate:xe,newStream:De,audioOffsetMs:be}),W(j,Re,H,ie,fr(k,xe)),P("audio_chunk",j)}else(H.length>0||J.length>0)&&(I.queueTimingsOnly(i,s,H,J,ie,{audioOffsetMs:be,newStream:B}),W(j,Re,H,ie,null),P("audio_chunk",j))}else if(v.event==="metadata"){const k=v.data.mood;typeof k=="string"&&k.trim()&&(d=k.trim());const H=v.data.sentence_emotions;if(Array.isArray(H)){f.length=0;for(const J of H){if(!J||typeof J!="object")continue;const ie=J;f.push({sentence_index:Number(ie.sentence_index??0),text:String(ie.text??""),sentiment:typeof ie.sentiment=="string"?ie.sentiment:void 0,emotion:String(ie.emotion??d),start_word:Number(ie.start_word??0),end_word:Number(ie.end_word??0)})}}E(),P("metadata",j)}else if(v.event==="error")throw new Error(String(v.data.message??"avatar_ttsWithPoses stream error"))}};for(;M===x;){const{done:_,value:v}=await $.read();v&&(L+=b.decode(v,{stream:!0}));const k=L.split(`
2
2
 
3
3
  `);L=k.pop()??"";for(const H of k)await O(H);if(_){L.trim()&&await O(L.trim()),I.markStreamComplete(i),P("bot_stopped",j);break}}}catch(oe){if(M!==x||oe instanceof DOMException&&oe.name==="AbortError")return;throw console.error("[avatarTtsLipsync]",oe),Y(),e("error"),oe}},stop:Y,getVisemeQueue:()=>i,getWordQueue:()=>s,getFullVisemeQueue:()=>o,getFullWordQueue:()=>a,getSentenceEmotions:()=>f,getStreamMood:()=>d,getPlaybackElapsedMs:()=>I.getPlaybackElapsedMs(),isSpeaking:()=>I.isSpeaking(),isWarmingUp:()=>I.isWarmingUp(),releaseWarmupHold:()=>I.releaseWarmupHold(),setLipsyncWarmupHold:(j,Z)=>{Z&&(N=Z),I.setLipsyncWarmupHoldEnabled(j)},getVoiceStream:()=>I.getVoiceStream(),setPlaybackSpeed:j=>{I.setPlaybackSpeed(j)},getPlaybackSpeed:()=>I.getPlaybackSpeed(),subscribe:j=>(m.add(j),()=>m.delete(j))}}const Wn=16e3,Hn=16e3;class Es extends Cn.ProtobufFrameSerializer{async deserialize(t){try{return await super.deserialize(t)}catch(n){if(n instanceof Error&&n.message==="Unknown frame kind")return{type:"raw",message:null};throw n}}}class Is extends Cn.DailyMediaManager{constructor(){super(!0,!0,void 0,void 0,512,Wn,Hn)}bufferBotAudio(t,n){}}const As=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function Ps(e){return As.test(e.trim())}function Ms(e){const t=e.replace(/\/$/,"");return t.startsWith("https://")?`wss://${t.slice(8)}`:t.startsWith("http://")?`ws://${t.slice(7)}`:t}function fi(e){if(e.wsUrl?.trim())return e.wsUrl.trim();const t=e.authToken?.trim();if(!t)throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");const n=e.apiBase??ot,r=new URL(`${Ms(n)}/ws/voice`);r.searchParams.set("authToken",t),r.searchParams.set("tenant","aiTwin");const i=e.voiceId?.trim();i&&Ps(i)&&r.searchParams.set("voiceId",i);const s=e.personaId?.trim();s&&r.searchParams.set("personaId",s);const o=e.speakingRate??Ft;return r.searchParams.set("speaking_rate",String(Math.max(on,Math.min(cn,o)))),e.ttsStream===!1?r.searchParams.set("tts_stream","false"):r.searchParams.set("tts_stream","true"),r.toString()}function yn(e,t){if(!Array.isArray(e))return[];const n=[];for(const r of e)!r||typeof r!="object"||n.push({sentence_index:Number(r.sentence_index??0),text:String(r.text??""),sentiment:typeof r.sentiment=="string"?r.sentiment:void 0,emotion:String(r.emotion??t),start_word:Number(r.start_word??0),end_word:Number(r.end_word??0)});return n}function Cs(e){if(!e||typeof e!="object")return null;const t=e,r=(t.data&&typeof t.data=="object"&&!Array.isArray(t.data)?t.data:null)||t;return{...r,type:String(r.type??t.type??"server-message")}}function dr(e){return e.is_new_segment===!0||Number(e.audio_offset_ms??-1)===0?!0:Number(e.chunk_index??-1)===0}function Rs(e){return e?.transport?._mediaManager?._wavStreamPlayer??null}function di(e,t={}){const n=[],r=[],i=[],s=[];let o=0;const a=[],l=[];let f="",u="neutral",c=ct,d=!1,p=null,T=null,m={...t},A=0,C=null;const x=new Set,g=[];let E=!1,N=null,I=!0,te="default";const de=new Set;let P=null,W=0,K=!1,X=!1,Y=null,pe=!1;const j=()=>{Y!=null&&(clearTimeout(Y),Y=null)},Z=()=>{j(),Y=setTimeout(()=>{Y=null,I&&(V.markStreamComplete(n),oe("bot_stopped"))},450)},M=()=>x.forEach(h=>h());let G=()=>!1;const V=ui({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>{pe=!1,X=!1,e("done")},onNotify:M,shouldHoldForWarmup:()=>G()},c),ue=()=>{if(N)return N;const h=new MessageChannel;return h.port1.onmessage=()=>{const R=g.shift();R!==void 0&&Re(R),g.length>0?h.port2.postMessage(null):E=!1},N=h,h},ne=h=>{g.push(h),!E&&(E=!0,ue().port2.postMessage(null))},ae=()=>V.getPlaybackElapsedMs(),we=(h,R=f)=>{if(!m.onLipsyncDebug)return;const ee=ae(),le=n,ve=an(i).map(ge=>({viseme:ge.viseme,vtimeMs:ge.vtime}));m.onLipsyncDebug({event:h,spokenText:R,mood:u,playbackAnchorPerf:V.getPlaybackStartPerf(),playbackElapsedMs:ee,queueLength:le.length,activeViseme:Bt(le,ee),segments:l.map(ge=>({...ge,visemes:[...ge.visemes]})),fullQueueLength:i.length,compactionSkipped:ve})},oe=(h,R)=>{m.onLipsyncDebug&&(C={event:h,spokenText:R},A===0&&(A=requestAnimationFrame(()=>{A=0;const ee=C;C=null,ee&&we(ee.event,ee.spokenText)})))},S=()=>{I=!0,de.clear(),V.clearAborted()},$=()=>{const h=Rs(p);if(h){te=`bot-${Date.now()}`,h.interruptedTrackIds={[te]:!0};try{h.interrupt?.()}catch{}}const R=T;try{R?.interrupt?.()}catch{}},b=h=>{if(pe&&V.isSpeaking())return!0;const R=h.context_id?.trim();return R&&de.has(R)?!0:I?!1:!(h.type==="avatar_metadata"||dr(h))},L=h=>{I=!1,pe=!1;const R=h.context_id?.trim();R&&de.add(R),j(),K=!1,X=!1,g.length=0,E=!1,P?.abort(),P=null,W+=1,$(),V.bargeIn(),B(),e("idle"),oe("interrupt"),M()},B=()=>{n.length=0,r.length=0,i.length=0,s.length=0,o=0,l.length=0,f="",u="neutral",a.length=0},O=()=>{j(),X=!1,K=!1,V.stop(),B(),oe("reset"),M()},_=()=>{O(),e("idle"),M()},v=(h,R)=>{typeof h.mood=="string"&&h.mood.trim()&&(u=h.mood.trim()),Array.isArray(h.sentence_emotions)&&h.sentence_emotions.length>0&&(R&&(a.length=0),a.push(...yn(h.sentence_emotions,u)))},k=h=>{if(b({...h,type:"avatar_metadata"}))return;S(),K=!0;const R=String(h.spoken_text??h.text??"").trim();R&&(f=R),V.resetForNewStream(c),typeof h.mood=="string"&&h.mood.trim()&&(u=h.mood.trim()),a.length=0,a.push(...yn(h.sentence_emotions,u)),Array.isArray(h.gestures)&&h.gestures,oe("metadata",R),M()},H=(h,R,ee,le)=>{if(!m.onLipsyncDebug)return;const ve=n.slice(ee),ge=le.filter(ye=>String(ye.symbol??ye.viseme??"").trim());l.push({spokenText:R,audioOffsetMs:ve[0]?.vtime??0,isNewSegment:!!h.is_new_segment,chunkDurationMs:h.chunk_duration_ms!=null?Number(h.chunk_duration_ms):null,receivedAtMs:performance.now(),visemes:ve.map((ye,Be)=>{const _e=ge[Be];return{rawSymbol:String(_e?.symbol??_e?.viseme??ye.viseme).trim(),viseme:ye.viseme,startSec:Number(_e?.start??0),durationSec:Number(_e?.duration??0),vtimeMs:ye.vtime,vdurationMs:ye.vduration}})})},J=(h,R,ee,le)=>{if(h.length===0&&R.length===0)return;si(i,s,h,R,o,c);const ve=ai(h,o);let ge=null;const ye=ee.trim();if(ye)try{const Be=atob(ye.replace(/\s+/g,"")).length,_e=Number.isFinite(le)&&(le??0)>0?Number(le):16e3;ge=Be/2/_e*1e3}catch{ge=null}o=Math.max(ve,ge!=null?o+ge:o)},ie=h=>{if(b(h))return;const R=dr(h);R&&(j(),S());const ee=String(h.spoken_text??h.text??"").trim(),le=h.visemes??[],ve=h.words??[],ge=String(h.chunk??"").trim(),ye=K||R||!X;K&&(K=!1),ye&&(X=!0),ee&&(f=ee),v(h,R),ye?(V.resetForNewStream(c),i.length=0,s.length=0,o=0):R&&V.resumeBotStream(),J(le,ve,ge,h.sample_rate);const Be=n.length;V.playAudioChunk(n,r,ge,le,ve,R,{sampleRate:h.sample_rate,newStream:ye,audioOffsetMs:Number(h.audio_offset_ms??0),onNewStream:()=>{l.length=0}}).then(()=>{H(h,ee,Be,le),oe("audio_chunk",ee),Z()})},xe=h=>{if(b(h))return;(h.is_new_segment||Number(h.audio_offset_ms??-1)===0)&&S(),console.warn("[realtimeVoice] avatar_lipsync is deprecated — expect avatar_audio_chunk");const R=String(h.spoken_text??h.text??"").trim();R&&(f=R),v(h,!!h.is_new_segment),J(h.visemes??[],h.words??[],"",void 0);const ee=n.length;V.queueTimingsOnly(n,r,h.visemes??[],h.words??[],!!h.is_new_segment,{audioOffsetMs:Number(h.audio_offset_ms??0)}),H(h,R,ee,h.visemes??[]),oe("lipsync",R),M()},be=async h=>{const R=String(h.text??"").trim();if(!R)return;const ee=m.authToken?.trim();if(!ee){console.error("[realtimeVoice] avatar_speak requires authToken");return}P?.abort();const le=new AbortController;P=le;const ve=++W;S(),V.resetForNewStream(c),f=R;const ye=`${(m.apiBase??ot).replace(/\/$/,"")}/avatar_ttsWithPoses`,Be=m.voiceId?.trim()||Un,_e=m.speakingRate??Ft,ze=Math.max(on,Math.min(cn,_e)),Xe={user_query:R,ttsEngineId:c,speakingRate:ze,tenant:"aiTwin"};Fr.has(c)&&(Xe.voice_id=Be);try{const Fe=await fetch(ye,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${ee}`},body:JSON.stringify(Xe),signal:le.signal});if(!Fe.ok)throw new Error(`avatar_ttsWithPoses failed (${Fe.status})`);const Tt=Fe.body;if(!Tt)throw new Error("No response body");const Je=Tt.getReader(),Ue=new TextDecoder;let Ee="",Ze=!0;const et=async tt=>{if(ve!==W||le.signal.aborted||!I)return;const Se=Kr(tt);if(Se){if(Se.event==="audio"){const re=Se.data.chunk,Ne=Se.data.visemes??[],ft=Se.data.words??[],kt=Se.data.is_new_segment??!1,dt=Se.data.sample_rate,xt=Number(Se.data.audio_offset_ms??0);if(re){const qe=Ze;Ze=!1,await V.playAudioChunk(n,r,re,Ne,ft,kt,{sampleRate:dt,newStream:qe,audioOffsetMs:xt})}else(Ne.length>0||ft.length>0)&&V.queueTimingsOnly(n,r,Ne,ft,kt,{audioOffsetMs:xt,newStream:Ze})}else if(Se.event==="metadata"){const re=Se.data.mood;typeof re=="string"&&re.trim()&&(u=re.trim());const Ne=Se.data.sentence_emotions;Array.isArray(Ne)&&(a.length=0,a.push(...yn(Ne,u))),M()}else if(Se.event==="error")throw new Error(String(Se.data.message??"avatar_ttsWithPoses stream error"))}};for(;!(ve!==W||le.signal.aborted||!I);){const{done:tt,value:Se}=await Je.read();Se&&(Ee+=Ue.decode(Se,{stream:!0}));const re=Ee.split(`
4
4
 
package/dist/aitwin.js CHANGED
@@ -5,7 +5,7 @@ import { WebSocketTransport as Ei, DailyMediaManager as Ii, ProtobufFrameSeriali
5
5
  import { Filter as Pi, GlProgram as Mi, Application as Ci, RendererType as Ri, Sprite as Rt, Texture as Me, VideoSource as Di, Rectangle as Bi } from "pixi.js";
6
6
  const Fi = "https://aitwin.bubu.social", Rr = {
7
7
  publicBase: Fi
8
- }, Li = "0.5.5", Oi = {
8
+ }, Li = "0.5.6", Oi = {
9
9
  version: Li
10
10
  }, Ni = {}, ut = "https://ai.aitwin.me", Pn = `${Rr.publicBase}/custom-faces`, Ui = `${Rr.publicBase}/aitwin-workers/v${Oi.version}/visemeDiffPreview.worker.js`;
11
11
  function Vi(e) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@streamoji/aitwin",
3
- "version": "0.5.5",
3
+ "version": "0.5.6",
4
4
  "description": "Embeddable React AI twin face with TTS lipsync",
5
5
  "type": "module",
6
6
  "main": "./dist/aitwin.cjs",