@streamoji/aitwin 0.3.4 → 0.3.5

package/README.md

@@ -168,7 +168,7 @@ The viseme diff Web Worker is hosted on R2 and loaded at runtime via `fetch` + b
 
 The worker URL is baked into each release from `package.json` version (`config/defaults.ts`) and uploaded to the matching versioned R2 path via `npm run upload:worker` (see below). This avoids CDN cache mismatches between the npm bundle and the worker script.
 
-Example URL for `0.1.4`: `https://pub-607ad1fc22e2400eb57d17240aab857c.r2.dev/aitwin-workers/v0.1.4/visemeDiffPreview.worker.js`
+Example URL for `0.1.4`: `https://aitwin.bubu.social/aitwin-workers/v0.1.4/visemeDiffPreview.worker.js`
 
 ### Upload worker after build
 

package/dist/aitwin.cjs

@@ -1,4 +1,4 @@
-"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const vt=require("react/jsx-runtime"),y=require("react"),Qn=require("@pipecat-ai/client-js"),Bt=require("@pipecat-ai/websocket-transport");var bt=typeof document<"u"?document.currentScript:null;const Zn="https://pub-607ad1fc22e2400eb57d17240aab857c.r2.dev",kn={publicBase:Zn},er="0.3.4",tr={version:er},nr={},Ze="https://ai.aitwin.me",Ot=`${kn.publicBase}/custom-faces`,rr=`${kn.publicBase}/aitwin-workers/v${tr.version}/visemeDiffPreview.worker.js`;function sr(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:bt&&bt.tagName.toUpperCase()==="SCRIPT"&&bt.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?nr?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():rr}const ir="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",Nt="eng_a7f2b9c1",ht="eng_d4e8f3a2",Pe="eng_c9b1e6d4",Ce=Pe,En=new Set([ht,Pe]),vn={[Nt]:"oculus_direct",[Pe]:"oculus_direct",[ht]:"inworld_word"},tn={google:Nt,inworld:ht,cartesia:Pe};function ar(e){return vn[e]}function or(e){return e===Pe}function cr(e){if(e.ttsEngineId&&e.ttsEngineId in vn)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in tn?tn[t]:Ce}class ut extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function bn(e,t=ir){const n=new URL(t);n.searchParams.set("id",e);const r=await fetch(n.toString());if(r.status===404)throw new ut;if(!r.ok)throw new Error(`getAiTwin failed (${r.status})`);const s=await r.json();if(!s.success)throw new ut(s.error??"AI twin not found");const{tts:i,token:o,...a}=s.data,l=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:cr(s.data),authToken:l}}const ur="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",lr="client_6TNvp3SCs4Og0a1ijm9TommXLql1",fr="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",dr="6TNvp3SCs4Og0a1ijm9TommXLql1",mr="Swaraj Mali";async function ct(){const e=await fetch(ur,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":lr,"Client-Secret":fr},body:JSON.stringify({userId:dr,userName:mr})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),n=t.authToken??t.token??"";if(!n.trim())throw new Error("getAuthToken returned no token");return n}async function hr(e,t=Ze){const n=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!n.ok)throw new Error(`/api/session-value failed (${n.status})`);const r=await n.json();if(!r.value)throw new Error("Session value response missing value");return r}function gr(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const n=new Uint8Array(t.length/2);for(let r=0;r<t.length;r+=2)n[r/2]=Number.parseInt(t.slice(r,r+2),16);return n}function st(e){return Uint8Array.from(e)}function wr(e,t){const n=new Uint8Array(e.length+t.length);return n.set(e,0),n.set(t,e.length),n}async function pr(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const n=new Uint8Array(e),r=st(n.subarray(0,12)),s=st(n.subarray(12,28)),i=st(n.subarray(28)),o=st(wr(i,s)),a=await crypto.subtle.importKey("raw",gr(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:r,tagLength:128},a,o)}let Rt=null;const Dt=new Set;function it(e){return e.includes("/custom-faces/")}function lt(e){const t=e.binBase.trim().replace(/\/$/,""),n=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||it(t)||it(n)))return{twinBase:n,binBase:t,encrypted:!1};const s=it(t)?t:it(n)?n:t;return{twinBase:s,binBase:s,encrypted:!0}}function Ue(e){return lt(e).encrypted}function Mt(e,t=Ot){const n=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:n,binBase:n,encrypted:!0}}function Tn(e,t){return(e??t)?.trim()||void 0}function An(e){Rt=lt(e);for(const t of Dt)t()}function _r(e,t){const n=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!n)return lt(e);const r=s=>{try{return new URL(s,n).href.replace(/\/$/,"")}catch{return s}};return lt({twinBase:r(e.twinBase),binBase:r(e.binBase),encrypted:e.encrypted})}function Fe(){if(!Rt)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return Rt}function yr(e){return Dt.add(e),()=>Dt.delete(e)}function nn(){const e=Fe();return Ue(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function Sr(){const e=Fe();return Ue(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function kr(){const e=Fe();return Ue(e)?e.binBase:e.twinBase}function gt(){return Ue(Fe())}function Er(){const e=Fe();return Ue(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function vr(){const e=Fe();return Ue(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function In(e){const t=e.split(/\r?\n/);let n="",r="";for(const i of t)i.startsWith("event:")?n=i.slice(6).trim():i.startsWith("data:")&&(r=i.slice(5).trim());if(!n)return null;let s={};if(r)try{s=JSON.parse(r)}catch{s={raw:r}}return{event:n,data:s}}const Rn=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],br=new Map(Rn.map(e=>[e.toLowerCase(),e]));function Dn(e){const t=(e??"").trim();return t?br.get(t.toLowerCase())??"sil":"sil"}const Tr={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function Ar(e){if(!e)return["sil"];const t=e.toLowerCase();return Tr[t]??["sil"]}function Ir(e,t){const n=e.word.length;if(n<=0)return 0;const r=e.wduration;if(r<=0)return 0;const s=Math.max(0,Math.min(1,(t-e.wtime)/r));return Math.min(n-1,Math.floor(s*n))}function Ne(e,t){const n=Math.max(0,Math.min(e.length-1,t));return e[n]??""}function Ut(e,t){const n=e.toLowerCase().replace(/[^a-z']/g,"");return!n||Ne(n,t)!=="w"||t>0&&Ne(n,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(n)}function Rr(e,t){const n=e.toLowerCase(),r=Ne(n,t),s=n.slice(Math.max(0,t-1),t+2);return r==="i"||r==="y"?"I":s.includes("ee")||r==="e"||n.includes("ea")?"E":"aa"}function Dr(e,t){const n=e.toLowerCase(),r=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));if(/oo|ou|uw/.test(r))return"U";if(Ut(n,t))return"FF";const s=Ne(n,t);return s==="u"||s==="w"?"U":"O"}function Mr(e,t){const n=e.toLowerCase();if(Ut(n,t))return"FF";const r=n.slice(Math.max(0,t-2),t+1);if(/qu/.test(r))return"U";const s=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));return/wo|wh/.test(s)||Ne(n,t)==="o"?"O":"U"}const at={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function Cr(e,t){const n=e.toLowerCase(),r=Ne(n,t);if(at[r])return at[r];for(let s=0;s<n.length;s++){const i=n[s];if(at[i])return at[i]}return"DD"}function xr(e,t,n){const r=Ar(e);if(r.length===1)return r[0];const s=t.trim();if(!s)return r[0];const i=(e??"").toLowerCase(),o=Math.max(0,n);switch(i){case"aei":return Rr(s,o);case"o":return Dr(s,o);case"qw":return Mr(s,o);case"cdgknstxyz":return Cr(s,o);default:return r[0]}}const Vt=16e3;function Mn(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:Vt}function Pr(e){let t=e.trim();const n=t.indexOf(",");t.startsWith("data:")&&n>=0&&(t=t.slice(n+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const r=t.length%4;return r!==0&&(t+="=".repeat(4-r)),t}function Fr(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function rn(e,t,n=Vt){const r=t.byteOffset%2===0?t:t.slice(),s=new Int16Array(r.buffer,r.byteOffset,r.byteLength/2),i=new Float32Array(s.length);for(let a=0;a<s.length;a++)i[a]=s[a]>=32768?-(65536-s[a])/32768:s[a]/32767;const o=e.createBuffer(1,i.length,n);return o.copyToChannel(i,0),o}function Lr(e){const t=Pr(e),n=window.atob(t),r=new Uint8Array(n.length);for(let s=0;s<n.length;s++)r[s]=n.charCodeAt(s);return r}async function Br(e,t,n=Vt){const r=Lr(t),s=Mn(n);if(Fr(r)){const i=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return e.decodeAudioData(i)}if(r.length>=2&&r.length%2===0)return rn(e,r,s);try{const i=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return await e.decodeAudioData(i)}catch{return rn(e,r,s)}}function De(e){let t=0;for(const n of e)t=Math.max(t,n.vtime+n.vduration);return t}function Or(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function Cn(e,t){const n=[],r=[];let s=null,i=!1,o=!1,a=!1,l=!1,f=0,u=0,k=0,m=0,p=null,I=1,d=t,h=!1,g=null,E=0;const b=()=>{p!=null&&(clearTimeout(p),p=null)},W=()=>{for(const S of r){try{S.onended=null,S.stop(0)}catch{}try{S.disconnect()}catch{}}r.length=0},Z=()=>{for(const S of r)S.playbackRate.value=I},Y=()=>{if(!(h||!l||i||r.length>0||n.length>0)){if(a&&s&&u>0){const S=(s.currentTime-u)*1e3;if(S<E){b();const M=I>0?(E-S)/I:E-S;p=setTimeout(()=>{h||(o=!1,p=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,M));return}}o=!1,b(),e.onSpeakingDone(),e.onNotify()}},D=S=>{const M=De(S);if(M<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,f=performance.now(),e.onSpeakingStart(),e.onNotify(),b();const V=I>0?M/I:M;p=setTimeout(()=>{h||(o=!1,p=null,e.onSpeakingDone(),e.onNotify())},V)},G=(S,M,V,L)=>{M.length!==0&&(L&&(k=V),M.forEach(P=>{const K=String(P.word??"").trim();if(!K)return;const ee=Math.round((P.start??0)*1e3),Q=P.duration!=null?Math.round(P.duration*1e3):P.end!=null?Math.max(1,Math.round((P.end-(P.start??0))*1e3)):0;S.push({word:K,wtime:k+ee,wduration:Q,queueIndex:S.length,gesture:P.gesture??null,emphasis:P.emphasis===!0})}))},_=(S,M,V,L,P,U)=>{V.length!==0&&(P&&(k=L),V.forEach(K=>{const ee=String(K.symbol??K.viseme??"").trim();if(!ee)return;const Q=Math.round((K.start??0)*1e3),T=Math.max(1,Math.round((K.duration??0)*1e3)),O=k+Q,J=Or(M,O),se=J?Ir(J,O):0;let $;ar(U)==="oculus_direct"?($=Dn(ee),$==="U"&&J&&Ut(J.word,se)&&($="FF")):$=xr(ee,J?.word??"",se),S.push({viseme:$,weight:1,vtime:O,vduration:T})}),e.onNotify())},A=(S,M,V,L,P,U,K)=>{G(M,L,P,U),_(S,M,V,P,U,K)},C=async(S,M,V,L,P,U,K)=>{if(h)return;const ee=Mn(K?.sampleRate),Q=K?.newStream===!0;if(!V?.trim()){if(L.length===0&&P.length===0)return;Q&&(k=0,E=0,S.length=0,M.length=0,K?.onNewStream?.());let T=k;U&&S.length>0&&!Q&&(T=De(S)),A(S,M,L,P,T,U,d),E=Math.max(E,De(S)),!o&&E>0&&D(S),e.onNotify();return}if(i){n.push({audio:V,visemes:L,words:P,isNewSegment:U,sampleRate:ee,newStream:Q});return}i=!0;try{const T=window.AudioContext||window.webkitAudioContext;s||(s=new T);const O=s;O.state==="suspended"&&await O.resume(),g||(g=O.createMediaStreamDestination());const J=await Br(O,V,ee);a=!0;const se=O.currentTime;let $=m;const de=!o;$<se&&($=se+.1),m=$+J.duration;const te=O.createBufferSource();if(te.buffer=J,te.playbackRate.value=I,te.connect(O.destination),g&&te.connect(g),r.push(te),h){r.pop();return}if(de){o=!0,e.onSpeakingStart(),Q?(u=$,k=0,E=0,S.length=0,M.length=0,K?.onNewStream?.()):u<=0&&(u=$);const ye=($-se)*1e3;f=performance.now()+ye}te.onended=()=>{const ye=r.indexOf(te);ye>=0&&r.splice(ye,1),Y(),e.onNotify()},te.start($);const Le=($-u)*1e3;A(S,M,L,P,Le,U,d),E=Math.max(E,De(S))}catch(T){console.error("[avatarChunkPlayback] audio chunk failed:",T)}finally{if(i=!1,h)return;const T=n.shift();T?await C(S,M,T.audio,T.visemes,T.words,T.isNewSegment,{sampleRate:T.sampleRate,newStream:T.newStream,onNewStream:K?.onNewStream}):Y()}};return{setStreamTtsEngineId:S=>{d=S},setPlaybackSpeed:S=>{I=Math.max(.1,Math.min(1,S)),Z()},getPlaybackSpeed:()=>I,resetForNewStream:S=>{h=!1,l=!1,d=S,E=0},resumeBotStream:()=>{h=!1,l=!1},clearAborted:()=>{h=!1},primeAudioContext:async()=>{const S=window.AudioContext||window.webkitAudioContext;s||(s=new S),s.state==="suspended"&&await s.resume(),s&&!g&&(g=s.createMediaStreamDestination())},stop:()=>{h=!0,b(),W(),n.length=0,i=!1,o=!1,a=!1,l=!1,k=0,m=0,u=0,f=0,E=0},markStreamComplete:S=>{if(E=Math.max(E,De(S)),l=!0,!a&&S.length>0){D(S);return}Y()},playAudioChunk:C,queueTimingsOnly:(S,M,V,L,P)=>{if(h)return;let U=k;P&&S.length>0&&(U=De(S)),A(S,M,V,L,U,P,d),E=Math.max(E,De(S))},getPlaybackElapsedMs:()=>a&&s&&u>0?Math.max(0,(s.currentTime-u)*1e3):o?Math.max(0,(performance.now()-f)*I):0,isSpeaking:()=>o?!0:!a||!s||u<=0?!1:(s.currentTime-u)*1e3<E,getVoiceStream:()=>g?g.stream:null,getPlaybackStartPerf:()=>f,bargeIn:()=>{h=!0,l=!1,b(),W(),n.length=0,i=!1,o=!1,a=!1,k=0,m=0,u=0,f=0,E=0},interrupt:()=>{h=!0,l=!1,b(),W(),n.length=0,i=!1,o=!1,a=!1,k=0,m=0,u=0,f=0,E=0}}}function Ve(e,t){let n=null;for(const r of e)t<r.vtime||t>=r.vtime+r.vduration||(!n||r.vtime>=n.vtime)&&(n=r);return n?n.viseme:"sil"}function Nr(e,t,n=120){const r=Ve(e,t);if(r!=="sil")return r;let s=null;for(const i of e){if(i.vtime>t)break;const o=i.vtime+i.vduration;o<=t&&t-o<=n&&(s=i)}return s?.viseme??"sil"}function Ur(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function Vr(e,t){const n=e.trim();if(!n)return null;try{const r=n.replace(/\s+/g,""),s=atob(r).length,i=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return s/2/i*1e3}catch{return null}}const wt=.5,pt=1.5,$t="Olivia",qt="f786b574-daa5-4673-aa0c-cbe3e8534c02",et=.85;function Ct(e,t=Ze,n={}){const r=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,s=[],i=[],o=[],a=[];let l="",f="neutral",u="",k=Ce;const m=new Set;let p=0,I=null;const d=()=>m.forEach(D=>D()),h=Cn({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:d},Ce),g=()=>h.getPlaybackElapsedMs(),E=(D,G=l)=>{if(!n.onLipsyncDebug)return;const _=g(),A=s;n.onLipsyncDebug({event:D,spokenText:G,mood:f,playbackAnchorPerf:h.getPlaybackStartPerf(),playbackElapsedMs:_,queueLength:A.length,activeViseme:Ve(A,_),segments:a.map(C=>({...C,visemes:[...C.visemes]}))})},b=(D,G)=>{n.onLipsyncDebug&&(I={event:D,spokenText:G},p===0&&(p=requestAnimationFrame(()=>{p=0;const _=I;I=null,_&&E(_.event,_.spokenText)})))},W=(D,G,_,A,C)=>{if(!n.onLipsyncDebug)return;const R=s.slice(G),S=_.filter(M=>String(M.symbol??M.viseme??"").trim());a.push({spokenText:D,audioOffsetMs:R[0]?.vtime??0,isNewSegment:A,chunkDurationMs:C,receivedAtMs:performance.now(),visemes:R.map((M,V)=>{const L=S[V];return{rawSymbol:String(L?.symbol??L?.viseme??M.viseme).trim(),viseme:M.viseme,startSec:Number(L?.start??0),durationSec:Number(L?.duration??0),vtimeMs:M.vtime,vdurationMs:M.vduration}})})},Z=()=>{h.stop(),s.length=0,i.length=0,o.length=0,f="neutral",a.length=0,l="",e("idle"),b("reset"),d()};return{setDeveloperToken:D=>{u=D},setTtsEngineId:D=>{k=D},getTtsEngineId:()=>k,speak:async(D,G)=>{Z(),e("loading"),l=D,a.length=0;const _=u.trim();if(!_)throw new Error("Developer token required for avatar_ttsWithPoses");const A=G?.ttsEngineId??k??Ce;h.resetForNewStream(A),h.setStreamTtsEngineId(A);const C=G?.voiceId??$t,R=G?.speakingRate??et,S=Math.max(wt,Math.min(pt,R)),M={user_query:D,ttsEngineId:A,speakingRate:S,tenant:"aiTwin"};En.has(A)&&(M.voice_id=C);try{const V=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${_}`},body:JSON.stringify(M)});if(!V.ok)throw new Error(`avatar_ttsWithPoses failed (${V.status})`);const L=V.body;if(!L)throw new Error("No response body");const P=L.getReader(),U=new TextDecoder;let K="";const ee=async Q=>{const T=In(Q);if(T){if(T.event==="audio"){const O=T.data.chunk,J=T.data.visemes??[],se=T.data.words??[],$=T.data.is_new_segment??!1,de=T.data.sample_rate,te=s.length;O?(await h.playAudioChunk(s,i,O,J,se,$,{sampleRate:de}),W(D,te,J,$,Vr(O,de)),b("audio_chunk",D)):(J.length>0||se.length>0)&&(h.queueTimingsOnly(s,i,J,se,$),W(D,te,J,$,null),b("audio_chunk",D))}else if(T.event==="metadata"){const O=T.data.mood;typeof O=="string"&&O.trim()&&(f=O.trim());const J=T.data.sentence_emotions;if(Array.isArray(J)){o.length=0;for(const se of J){if(!se||typeof se!="object")continue;const $=se;o.push({sentence_index:Number($.sentence_index??0),text:String($.text??""),sentiment:typeof $.sentiment=="string"?$.sentiment:void 0,emotion:String($.emotion??f),start_word:Number($.start_word??0),end_word:Number($.end_word??0)})}}d(),b("metadata",D)}else if(T.event==="error")throw new Error(String(T.data.message??"avatar_ttsWithPoses stream error"))}};for(;;){const{done:Q,value:T}=await P.read();T&&(K+=U.decode(T,{stream:!0}));const O=K.split(`
+"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const vt=require("react/jsx-runtime"),y=require("react"),Qn=require("@pipecat-ai/client-js"),Bt=require("@pipecat-ai/websocket-transport");var bt=typeof document<"u"?document.currentScript:null;const Zn="https://aitwin.bubu.social",kn={publicBase:Zn},er="0.3.5",tr={version:er},nr={},Ze="https://ai.aitwin.me",Ot=`${kn.publicBase}/custom-faces`,rr=`${kn.publicBase}/aitwin-workers/v${tr.version}/visemeDiffPreview.worker.js`;function sr(e){if(e)return e;const t=typeof{url:typeof document>"u"?require("url").pathToFileURL(__filename).href:bt&&bt.tagName.toUpperCase()==="SCRIPT"&&bt.src||new URL("aitwin.cjs",document.baseURI).href}<"u"?nr?.VITE_AITWIN_WORKER_URL:void 0;return t?.trim()?t.trim():rr}const ir="https://us-central1-streamoji-265f4.cloudfunctions.net/getAiTwin",Nt="eng_a7f2b9c1",ht="eng_d4e8f3a2",Pe="eng_c9b1e6d4",Ce=Pe,En=new Set([ht,Pe]),vn={[Nt]:"oculus_direct",[Pe]:"oculus_direct",[ht]:"inworld_word"},tn={google:Nt,inworld:ht,cartesia:Pe};function ar(e){return vn[e]}function or(e){return e===Pe}function cr(e){if(e.ttsEngineId&&e.ttsEngineId in vn)return e.ttsEngineId;const t=e.tts?.toLowerCase();return t&&t in tn?tn[t]:Ce}class ut extends Error{constructor(t="AI twin not found"){super(t),this.name="AiTwinNotFoundError"}}async function bn(e,t=ir){const n=new URL(t);n.searchParams.set("id",e);const r=await fetch(n.toString());if(r.status===404)throw new ut;if(!r.ok)throw new Error(`getAiTwin failed (${r.status})`);const s=await r.json();if(!s.success)throw new ut(s.error??"AI twin not found");const{tts:i,token:o,...a}=s.data,l=String(a.authToken??o??"").trim()||void 0;return{...a,ttsEngineId:cr(s.data),authToken:l}}const ur="https://us-central1-streamoji-265f4.cloudfunctions.net/getAuthToken",lr="client_6TNvp3SCs4Og0a1ijm9TommXLql1",fr="kk8Fq8EmexzP10jMIEwY3R44M5RKUEm1",dr="6TNvp3SCs4Og0a1ijm9TommXLql1",mr="Swaraj Mali";async function ct(){const e=await fetch(ur,{method:"POST",headers:{"Content-Type":"application/json","Client-Id":lr,"Client-Secret":fr},body:JSON.stringify({userId:dr,userName:mr})});if(!e.ok)throw new Error(`getAuthToken failed (${e.status})`);const t=await e.json(),n=t.authToken??t.token??"";if(!n.trim())throw new Error("getAuthToken returned no token");return n}async function hr(e,t=Ze){const n=await fetch(`${t.replace(/\/$/,"")}/api/session-value`,{headers:{Authorization:`Bearer ${e}`}});if(!n.ok)throw new Error(`/api/session-value failed (${n.status})`);const r=await n.json();if(!r.value)throw new Error("Session value response missing value");return r}function gr(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const n=new Uint8Array(t.length/2);for(let r=0;r<t.length;r+=2)n[r/2]=Number.parseInt(t.slice(r,r+2),16);return n}function st(e){return Uint8Array.from(e)}function wr(e,t){const n=new Uint8Array(e.length+t.length);return n.set(e,0),n.set(t,e.length),n}async function pr(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const n=new Uint8Array(e),r=st(n.subarray(0,12)),s=st(n.subarray(12,28)),i=st(n.subarray(28)),o=st(wr(i,s)),a=await crypto.subtle.importKey("raw",gr(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:r,tagLength:128},a,o)}let Rt=null;const Dt=new Set;function it(e){return e.includes("/custom-faces/")}function lt(e){const t=e.binBase.trim().replace(/\/$/,""),n=e.twinBase.trim().replace(/\/$/,"");if(!(e.encrypted||it(t)||it(n)))return{twinBase:n,binBase:t,encrypted:!1};const s=it(t)?t:it(n)?n:t;return{twinBase:s,binBase:s,encrypted:!0}}function Ue(e){return lt(e).encrypted}function Mt(e,t=Ot){const n=`${t.replace(/\/$/,"")}/${e}`;return{twinBase:n,binBase:n,encrypted:!0}}function Tn(e,t){return(e??t)?.trim()||void 0}function An(e){Rt=lt(e);for(const t of Dt)t()}function _r(e,t){const n=typeof globalThis.location<"u"?globalThis.location.href:void 0;if(!n)return lt(e);const r=s=>{try{return new URL(s,n).href.replace(/\/$/,"")}catch{return s}};return lt({twinBase:r(e.twinBase),binBase:r(e.binBase),encrypted:e.encrypted})}function Fe(){if(!Rt)throw new Error("Twin assets not configured; call setActiveTwinAssets first");return Rt}function yr(e){return Dt.add(e),()=>Dt.delete(e)}function nn(){const e=Fe();return Ue(e)?`${e.binBase}/sil.bin`:`${e.twinBase}/sil.png`}function Sr(){const e=Fe();return Ue(e)?`${e.binBase}/idle.bin`:`${e.twinBase}/idle.mp4`}function kr(){const e=Fe();return Ue(e)?e.binBase:e.twinBase}function gt(){return Ue(Fe())}function Er(){const e=Fe();return Ue(e)?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function vr(){const e=Fe();return Ue(e)?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function In(e){const t=e.split(/\r?\n/);let n="",r="";for(const i of t)i.startsWith("event:")?n=i.slice(6).trim():i.startsWith("data:")&&(r=i.slice(5).trim());if(!n)return null;let s={};if(r)try{s=JSON.parse(r)}catch{s={raw:r}}return{event:n,data:s}}const Rn=["aa","CH","DD","E","FF","I","O","PP","RR","SS","TH","U","kk","nn","sil"],br=new Map(Rn.map(e=>[e.toLowerCase(),e]));function Dn(e){const t=(e??"").trim();return t?br.get(t.toLowerCase())??"sil":"sil"}const Tr={aei:["aa","E","I"],o:["O","U"],ee:["I"],bmp:["PP"],fv:["FF"],l:["nn"],r:["RR"],th:["TH"],qw:["U","O"],cdgknstxyz:["DD","SS","kk","CH"]};function Ar(e){if(!e)return["sil"];const t=e.toLowerCase();return Tr[t]??["sil"]}function Ir(e,t){const n=e.word.length;if(n<=0)return 0;const r=e.wduration;if(r<=0)return 0;const s=Math.max(0,Math.min(1,(t-e.wtime)/r));return Math.min(n-1,Math.floor(s*n))}function Ne(e,t){const n=Math.max(0,Math.min(e.length-1,t));return e[n]??""}function Ut(e,t){const n=e.toLowerCase().replace(/[^a-z']/g,"");return!n||Ne(n,t)!=="w"||t>0&&Ne(n,t-1)==="q"?!1:t===0&&/^w[aeiouy]/.test(n)}function Rr(e,t){const n=e.toLowerCase(),r=Ne(n,t),s=n.slice(Math.max(0,t-1),t+2);return r==="i"||r==="y"?"I":s.includes("ee")||r==="e"||n.includes("ea")?"E":"aa"}function Dr(e,t){const n=e.toLowerCase(),r=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));if(/oo|ou|uw/.test(r))return"U";if(Ut(n,t))return"FF";const s=Ne(n,t);return s==="u"||s==="w"?"U":"O"}function Mr(e,t){const n=e.toLowerCase();if(Ut(n,t))return"FF";const r=n.slice(Math.max(0,t-2),t+1);if(/qu/.test(r))return"U";const s=n.slice(Math.max(0,t-1),Math.min(n.length,t+3));return/wo|wh/.test(s)||Ne(n,t)==="o"?"O":"U"}const at={d:"DD",t:"DD",n:"DD",s:"SS",z:"SS",k:"kk",g:"kk",c:"CH",x:"CH",j:"CH"};function Cr(e,t){const n=e.toLowerCase(),r=Ne(n,t);if(at[r])return at[r];for(let s=0;s<n.length;s++){const i=n[s];if(at[i])return at[i]}return"DD"}function xr(e,t,n){const r=Ar(e);if(r.length===1)return r[0];const s=t.trim();if(!s)return r[0];const i=(e??"").toLowerCase(),o=Math.max(0,n);switch(i){case"aei":return Rr(s,o);case"o":return Dr(s,o);case"qw":return Mr(s,o);case"cdgknstxyz":return Cr(s,o);default:return r[0]}}const Vt=16e3;function Mn(e){const t=Number(e);return Number.isFinite(t)&&t>0?t:Vt}function Pr(e){let t=e.trim();const n=t.indexOf(",");t.startsWith("data:")&&n>=0&&(t=t.slice(n+1)),t=t.replace(/\s+/g,"").replace(/-/g,"+").replace(/_/g,"/");const r=t.length%4;return r!==0&&(t+="=".repeat(4-r)),t}function Fr(e){return e.length>=4&&e[0]===82&&e[1]===73&&e[2]===70&&e[3]===70}function rn(e,t,n=Vt){const r=t.byteOffset%2===0?t:t.slice(),s=new Int16Array(r.buffer,r.byteOffset,r.byteLength/2),i=new Float32Array(s.length);for(let a=0;a<s.length;a++)i[a]=s[a]>=32768?-(65536-s[a])/32768:s[a]/32767;const o=e.createBuffer(1,i.length,n);return o.copyToChannel(i,0),o}function Lr(e){const t=Pr(e),n=window.atob(t),r=new Uint8Array(n.length);for(let s=0;s<n.length;s++)r[s]=n.charCodeAt(s);return r}async function Br(e,t,n=Vt){const r=Lr(t),s=Mn(n);if(Fr(r)){const i=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return e.decodeAudioData(i)}if(r.length>=2&&r.length%2===0)return rn(e,r,s);try{const i=r.buffer.slice(r.byteOffset,r.byteOffset+r.byteLength);return await e.decodeAudioData(i)}catch{return rn(e,r,s)}}function De(e){let t=0;for(const n of e)t=Math.max(t,n.vtime+n.vduration);return t}function Or(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function Cn(e,t){const n=[],r=[];let s=null,i=!1,o=!1,a=!1,l=!1,f=0,u=0,k=0,m=0,p=null,I=1,d=t,h=!1,g=null,E=0;const b=()=>{p!=null&&(clearTimeout(p),p=null)},W=()=>{for(const S of r){try{S.onended=null,S.stop(0)}catch{}try{S.disconnect()}catch{}}r.length=0},Z=()=>{for(const S of r)S.playbackRate.value=I},Y=()=>{if(!(h||!l||i||r.length>0||n.length>0)){if(a&&s&&u>0){const S=(s.currentTime-u)*1e3;if(S<E){b();const M=I>0?(E-S)/I:E-S;p=setTimeout(()=>{h||(o=!1,p=null,e.onSpeakingDone(),e.onNotify())},Math.max(0,M));return}}o=!1,b(),e.onSpeakingDone(),e.onNotify()}},D=S=>{const M=De(S);if(M<=0){e.onSpeakingDone(),e.onNotify();return}o=!0,f=performance.now(),e.onSpeakingStart(),e.onNotify(),b();const V=I>0?M/I:M;p=setTimeout(()=>{h||(o=!1,p=null,e.onSpeakingDone(),e.onNotify())},V)},G=(S,M,V,L)=>{M.length!==0&&(L&&(k=V),M.forEach(P=>{const K=String(P.word??"").trim();if(!K)return;const ee=Math.round((P.start??0)*1e3),Q=P.duration!=null?Math.round(P.duration*1e3):P.end!=null?Math.max(1,Math.round((P.end-(P.start??0))*1e3)):0;S.push({word:K,wtime:k+ee,wduration:Q,queueIndex:S.length,gesture:P.gesture??null,emphasis:P.emphasis===!0})}))},_=(S,M,V,L,P,U)=>{V.length!==0&&(P&&(k=L),V.forEach(K=>{const ee=String(K.symbol??K.viseme??"").trim();if(!ee)return;const Q=Math.round((K.start??0)*1e3),T=Math.max(1,Math.round((K.duration??0)*1e3)),O=k+Q,J=Or(M,O),se=J?Ir(J,O):0;let $;ar(U)==="oculus_direct"?($=Dn(ee),$==="U"&&J&&Ut(J.word,se)&&($="FF")):$=xr(ee,J?.word??"",se),S.push({viseme:$,weight:1,vtime:O,vduration:T})}),e.onNotify())},A=(S,M,V,L,P,U,K)=>{G(M,L,P,U),_(S,M,V,P,U,K)},C=async(S,M,V,L,P,U,K)=>{if(h)return;const ee=Mn(K?.sampleRate),Q=K?.newStream===!0;if(!V?.trim()){if(L.length===0&&P.length===0)return;Q&&(k=0,E=0,S.length=0,M.length=0,K?.onNewStream?.());let T=k;U&&S.length>0&&!Q&&(T=De(S)),A(S,M,L,P,T,U,d),E=Math.max(E,De(S)),!o&&E>0&&D(S),e.onNotify();return}if(i){n.push({audio:V,visemes:L,words:P,isNewSegment:U,sampleRate:ee,newStream:Q});return}i=!0;try{const T=window.AudioContext||window.webkitAudioContext;s||(s=new T);const O=s;O.state==="suspended"&&await O.resume(),g||(g=O.createMediaStreamDestination());const J=await Br(O,V,ee);a=!0;const se=O.currentTime;let $=m;const de=!o;$<se&&($=se+.1),m=$+J.duration;const te=O.createBufferSource();if(te.buffer=J,te.playbackRate.value=I,te.connect(O.destination),g&&te.connect(g),r.push(te),h){r.pop();return}if(de){o=!0,e.onSpeakingStart(),Q?(u=$,k=0,E=0,S.length=0,M.length=0,K?.onNewStream?.()):u<=0&&(u=$);const ye=($-se)*1e3;f=performance.now()+ye}te.onended=()=>{const ye=r.indexOf(te);ye>=0&&r.splice(ye,1),Y(),e.onNotify()},te.start($);const Le=($-u)*1e3;A(S,M,L,P,Le,U,d),E=Math.max(E,De(S))}catch(T){console.error("[avatarChunkPlayback] audio chunk failed:",T)}finally{if(i=!1,h)return;const T=n.shift();T?await C(S,M,T.audio,T.visemes,T.words,T.isNewSegment,{sampleRate:T.sampleRate,newStream:T.newStream,onNewStream:K?.onNewStream}):Y()}};return{setStreamTtsEngineId:S=>{d=S},setPlaybackSpeed:S=>{I=Math.max(.1,Math.min(1,S)),Z()},getPlaybackSpeed:()=>I,resetForNewStream:S=>{h=!1,l=!1,d=S,E=0},resumeBotStream:()=>{h=!1,l=!1},clearAborted:()=>{h=!1},primeAudioContext:async()=>{const S=window.AudioContext||window.webkitAudioContext;s||(s=new S),s.state==="suspended"&&await s.resume(),s&&!g&&(g=s.createMediaStreamDestination())},stop:()=>{h=!0,b(),W(),n.length=0,i=!1,o=!1,a=!1,l=!1,k=0,m=0,u=0,f=0,E=0},markStreamComplete:S=>{if(E=Math.max(E,De(S)),l=!0,!a&&S.length>0){D(S);return}Y()},playAudioChunk:C,queueTimingsOnly:(S,M,V,L,P)=>{if(h)return;let U=k;P&&S.length>0&&(U=De(S)),A(S,M,V,L,U,P,d),E=Math.max(E,De(S))},getPlaybackElapsedMs:()=>a&&s&&u>0?Math.max(0,(s.currentTime-u)*1e3):o?Math.max(0,(performance.now()-f)*I):0,isSpeaking:()=>o?!0:!a||!s||u<=0?!1:(s.currentTime-u)*1e3<E,getVoiceStream:()=>g?g.stream:null,getPlaybackStartPerf:()=>f,bargeIn:()=>{h=!0,l=!1,b(),W(),n.length=0,i=!1,o=!1,a=!1,k=0,m=0,u=0,f=0,E=0},interrupt:()=>{h=!0,l=!1,b(),W(),n.length=0,i=!1,o=!1,a=!1,k=0,m=0,u=0,f=0,E=0}}}function Ve(e,t){let n=null;for(const r of e)t<r.vtime||t>=r.vtime+r.vduration||(!n||r.vtime>=n.vtime)&&(n=r);return n?n.viseme:"sil"}function Nr(e,t,n=120){const r=Ve(e,t);if(r!=="sil")return r;let s=null;for(const i of e){if(i.vtime>t)break;const o=i.vtime+i.vduration;o<=t&&t-o<=n&&(s=i)}return s?.viseme??"sil"}function Ur(e,t){for(const n of e)if(t>=n.wtime&&t<n.wtime+n.wduration)return n;return null}function Vr(e,t){const n=e.trim();if(!n)return null;try{const r=n.replace(/\s+/g,""),s=atob(r).length,i=Number.isFinite(t)&&(t??0)>0?Number(t):16e3;return s/2/i*1e3}catch{return null}}const wt=.5,pt=1.5,$t="Olivia",qt="f786b574-daa5-4673-aa0c-cbe3e8534c02",et=.85;function Ct(e,t=Ze,n={}){const r=`${t.replace(/\/$/,"")}/avatar_ttsWithPoses`,s=[],i=[],o=[],a=[];let l="",f="neutral",u="",k=Ce;const m=new Set;let p=0,I=null;const d=()=>m.forEach(D=>D()),h=Cn({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:d},Ce),g=()=>h.getPlaybackElapsedMs(),E=(D,G=l)=>{if(!n.onLipsyncDebug)return;const _=g(),A=s;n.onLipsyncDebug({event:D,spokenText:G,mood:f,playbackAnchorPerf:h.getPlaybackStartPerf(),playbackElapsedMs:_,queueLength:A.length,activeViseme:Ve(A,_),segments:a.map(C=>({...C,visemes:[...C.visemes]}))})},b=(D,G)=>{n.onLipsyncDebug&&(I={event:D,spokenText:G},p===0&&(p=requestAnimationFrame(()=>{p=0;const _=I;I=null,_&&E(_.event,_.spokenText)})))},W=(D,G,_,A,C)=>{if(!n.onLipsyncDebug)return;const R=s.slice(G),S=_.filter(M=>String(M.symbol??M.viseme??"").trim());a.push({spokenText:D,audioOffsetMs:R[0]?.vtime??0,isNewSegment:A,chunkDurationMs:C,receivedAtMs:performance.now(),visemes:R.map((M,V)=>{const L=S[V];return{rawSymbol:String(L?.symbol??L?.viseme??M.viseme).trim(),viseme:M.viseme,startSec:Number(L?.start??0),durationSec:Number(L?.duration??0),vtimeMs:M.vtime,vdurationMs:M.vduration}})})},Z=()=>{h.stop(),s.length=0,i.length=0,o.length=0,f="neutral",a.length=0,l="",e("idle"),b("reset"),d()};return{setDeveloperToken:D=>{u=D},setTtsEngineId:D=>{k=D},getTtsEngineId:()=>k,speak:async(D,G)=>{Z(),e("loading"),l=D,a.length=0;const _=u.trim();if(!_)throw new Error("Developer token required for avatar_ttsWithPoses");const A=G?.ttsEngineId??k??Ce;h.resetForNewStream(A),h.setStreamTtsEngineId(A);const C=G?.voiceId??$t,R=G?.speakingRate??et,S=Math.max(wt,Math.min(pt,R)),M={user_query:D,ttsEngineId:A,speakingRate:S,tenant:"aiTwin"};En.has(A)&&(M.voice_id=C);try{const V=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${_}`},body:JSON.stringify(M)});if(!V.ok)throw new Error(`avatar_ttsWithPoses failed (${V.status})`);const L=V.body;if(!L)throw new Error("No response body");const P=L.getReader(),U=new TextDecoder;let K="";const ee=async Q=>{const T=In(Q);if(T){if(T.event==="audio"){const O=T.data.chunk,J=T.data.visemes??[],se=T.data.words??[],$=T.data.is_new_segment??!1,de=T.data.sample_rate,te=s.length;O?(await h.playAudioChunk(s,i,O,J,se,$,{sampleRate:de}),W(D,te,J,$,Vr(O,de)),b("audio_chunk",D)):(J.length>0||se.length>0)&&(h.queueTimingsOnly(s,i,J,se,$),W(D,te,J,$,null),b("audio_chunk",D))}else if(T.event==="metadata"){const O=T.data.mood;typeof O=="string"&&O.trim()&&(f=O.trim());const J=T.data.sentence_emotions;if(Array.isArray(J)){o.length=0;for(const se of J){if(!se||typeof se!="object")continue;const $=se;o.push({sentence_index:Number($.sentence_index??0),text:String($.text??""),sentiment:typeof $.sentiment=="string"?$.sentiment:void 0,emotion:String($.emotion??f),start_word:Number($.start_word??0),end_word:Number($.end_word??0)})}}d(),b("metadata",D)}else if(T.event==="error")throw new Error(String(T.data.message??"avatar_ttsWithPoses stream error"))}};for(;;){const{done:Q,value:T}=await P.read();T&&(K+=U.decode(T,{stream:!0}));const O=K.split(`
 
 `);K=O.pop()??"";for(const J of O)await ee(J);if(Q){K.trim()&&await ee(K.trim()),h.markStreamComplete(s),b("bot_stopped",D);break}}}catch(V){throw console.error("[avatarTtsLipsync]",V),Z(),e("error"),V}},stop:Z,getVisemeQueue:()=>s,getWordQueue:()=>i,getSentenceEmotions:()=>o,getStreamMood:()=>f,getPlaybackElapsedMs:()=>h.getPlaybackElapsedMs(),isSpeaking:()=>h.isSpeaking(),getVoiceStream:()=>h.getVoiceStream(),setPlaybackSpeed:D=>{h.setPlaybackSpeed(D)},getPlaybackSpeed:()=>h.getPlaybackSpeed(),subscribe:D=>(m.add(D),()=>m.delete(D))}}const Wt=16e3,Ht=16e3;class $r extends Bt.ProtobufFrameSerializer{async deserialize(t){try{return await super.deserialize(t)}catch(n){if(n instanceof Error&&n.message==="Unknown frame kind")return{type:"raw",message:null};throw n}}}class qr extends Bt.DailyMediaManager{constructor(){super(!0,!0,void 0,void 0,512,Wt,Ht)}bufferBotAudio(t,n){}}const Wr=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function Hr(e){return Wr.test(e.trim())}function Kr(e){const t=e.replace(/\/$/,"");return t.startsWith("https://")?`wss://${t.slice(8)}`:t.startsWith("http://")?`ws://${t.slice(7)}`:t}function xn(e){if(e.wsUrl?.trim())return e.wsUrl.trim();const t=e.authToken?.trim();if(!t)throw new Error("Voice connect requires authToken (same JWT as SSE Authorization Bearer).");const n=e.apiBase??Ze,r=new URL(`${Kr(n)}/ws/voice`);r.searchParams.set("authToken",t),r.searchParams.set("tenant","aiTwin");const s=e.voiceId?.trim();s&&Hr(s)&&r.searchParams.set("voiceId",s);const i=e.personaId?.trim();i&&r.searchParams.set("personaId",i);const o=e.speakingRate??et;return r.searchParams.set("speaking_rate",String(Math.max(wt,Math.min(pt,o)))),e.ttsStream===!1?r.searchParams.set("tts_stream","false"):r.searchParams.set("tts_stream","true"),r.toString()}function Tt(e,t){if(!Array.isArray(e))return[];const n=[];for(const r of e)!r||typeof r!="object"||n.push({sentence_index:Number(r.sentence_index??0),text:String(r.text??""),sentiment:typeof r.sentiment=="string"?r.sentiment:void 0,emotion:String(r.emotion??t),start_word:Number(r.start_word??0),end_word:Number(r.end_word??0)});return n}function Gr(e){if(!e||typeof e!="object")return null;const t=e,r=(t.data&&typeof t.data=="object"&&!Array.isArray(t.data)?t.data:null)||t;return{...r,type:String(r.type??t.type??"server-message")}}function At(e){return e.is_new_segment===!0||Number(e.audio_offset_ms??-1)===0?!0:Number(e.chunk_index??-1)===0}function jr(e){return e?.transport?._mediaManager?._wavStreamPlayer??null}function Pn(e,t={}){const n=[],r=[],s=[],i=[];let o="",a="neutral",l=Pe,f=!1,u=null,k=null,m={...t},p=0,I=null;const d=new Set,h=[];let g=!1,E=null,b=!0,W="default";const Z=new Set;let Y=null,D=0,G=!1,_=null;const A=()=>{_!=null&&(clearTimeout(_),_=null)},C=()=>d.forEach(c=>c()),R=Cn({onSpeakingStart:()=>e("speaking"),onSpeakingDone:()=>e("done"),onNotify:C},l),S=()=>{if(E)return E;const c=new MessageChannel;return c.port1.onmessage=()=>{const v=h.shift();v!==void 0&&$e(v),h.length>0?c.port2.postMessage(null):g=!1},E=c,c},M=c=>{h.push(c),!g&&(g=!0,S().port2.postMessage(null))},V=()=>R.getPlaybackElapsedMs(),L=(c,v=o)=>{if(!m.onLipsyncDebug)return;const B=V(),X=n;m.onLipsyncDebug({event:c,spokenText:v,mood:a,playbackAnchorPerf:R.getPlaybackStartPerf(),playbackElapsedMs:B,queueLength:X.length,activeViseme:Ve(X,B),segments:i.map(ie=>({...ie,visemes:[...ie.visemes]}))})},P=(c,v)=>{m.onLipsyncDebug&&(I={event:c,spokenText:v},p===0&&(p=requestAnimationFrame(()=>{p=0;const B=I;I=null,B&&L(B.event,B.spokenText)})))},U=()=>{b=!0,Z.clear(),R.clearAborted()},K=()=>{const c=jr(u);if(c){W=`bot-${Date.now()}`,c.interruptedTrackIds={[W]:!0};try{c.interrupt?.()}catch{}}const v=k;try{v?.interrupt?.()}catch{}},ee=c=>{const v=c.context_id?.trim();return v&&Z.has(v)?!0:b?!1:!(c.type==="avatar_metadata"||At(c))},Q=c=>{b=!1;const v=c.context_id?.trim();v&&Z.add(v),A(),G=!1,h.length=0,g=!1,Y?.abort(),Y=null,D+=1,K(),R.bargeIn(),T(),e("idle"),P("interrupt"),C()},T=()=>{n.length=0,r.length=0,i.length=0,o="",a="neutral",s.length=0},O=()=>{R.stop(),T(),P("reset"),C()},J=()=>{O(),e("idle"),C()},se=(c,v)=>{typeof c.mood=="string"&&c.mood.trim()&&(a=c.mood.trim()),Array.isArray(c.sentence_emotions)&&c.sentence_emotions.length>0&&(v&&(s.length=0),s.push(...Tt(c.sentence_emotions,a)))},$=c=>{if(ee({...c,type:"avatar_metadata"}))return;U(),G=!0;const v=String(c.spoken_text??c.text??"").trim();v&&(o=v),R.resetForNewStream(l),typeof c.mood=="string"&&c.mood.trim()&&(a=c.mood.trim()),s.length=0,s.push(...Tt(c.sentence_emotions,a)),Array.isArray(c.gestures)&&c.gestures,P("metadata",v),C()},de=(c,v,B,X)=>{if(!m.onLipsyncDebug)return;const ie=n.slice(B),ke=X.filter(oe=>String(oe.symbol??oe.viseme??"").trim());i.push({spokenText:v,audioOffsetMs:ie[0]?.vtime??0,isNewSegment:!!c.is_new_segment,chunkDurationMs:c.chunk_duration_ms!=null?Number(c.chunk_duration_ms):null,receivedAtMs:performance.now(),visemes:ie.map((oe,ne)=>{const pe=ke[ne];return{rawSymbol:String(pe?.symbol??pe?.viseme??oe.viseme).trim(),viseme:oe.viseme,startSec:Number(pe?.start??0),durationSec:Number(pe?.duration??0),vtimeMs:oe.vtime,vdurationMs:oe.vduration}})})},te=c=>{if(ee(c))return;A(),At(c)&&U();const v=String(c.spoken_text??c.text??"").trim(),B=At(c),X=c.visemes??[],ie=c.words??[],ke=String(c.chunk??"").trim(),oe=G;oe&&(G=!1),v&&(o=v),se(c,B),oe?R.resetForNewStream(l):B&&R.resumeBotStream();const ne=n.length;R.playAudioChunk(n,r,ke,X,ie,B,{sampleRate:c.sample_rate,newStream:oe,onNewStream:()=>{i.length=0}}).then(()=>{de(c,v,ne,X),P("audio_chunk",v)})},Le=c=>{if(ee(c))return;(c.is_new_segment||Number(c.audio_offset_ms??-1)===0)&&U(),console.warn("[realtimeVoice] avatar_lipsync is deprecated — expect avatar_audio_chunk");const v=String(c.spoken_text??c.text??"").trim();v&&(o=v),se(c,!!c.is_new_segment);const B=n.length;R.queueTimingsOnly(n,r,c.visemes??[],c.words??[],!!c.is_new_segment),de(c,v,B,c.visemes??[]),P("lipsync",v),C()},ye=async c=>{const v=String(c.text??"").trim();if(!v)return;const B=m.authToken?.trim();if(!B){console.error("[realtimeVoice] avatar_speak requires authToken");return}Y?.abort();const X=new AbortController;Y=X;const ie=++D;U(),R.resetForNewStream(l),o=v;const oe=`${(m.apiBase??Ze).replace(/\/$/,"")}/avatar_ttsWithPoses`,ne=m.voiceId?.trim()||qt,pe=m.speakingRate??et,nt=Math.max(wt,Math.min(pt,pe)),We={user_query:v,ttsEngineId:l,speakingRate:nt,tenant:"aiTwin"};En.has(l)&&(We.voice_id=ne);try{const ge=await fetch(oe,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${B}`},body:JSON.stringify(We),signal:X.signal});if(!ge.ok)throw new Error(`avatar_ttsWithPoses failed (${ge.status})`);const He=ge.body;if(!He)throw new Error("No response body");const Ae=He.getReader(),rt=new TextDecoder;let ce="";const Ee=async j=>{if(ie!==D||X.signal.aborted||!b)return;const F=In(j);if(F){if(F.event==="audio"){const me=F.data.chunk,re=F.data.visemes??[],fe=F.data.words??[],Ke=F.data.is_new_segment??!1,Ie=F.data.sample_rate;me?await R.playAudioChunk(n,r,me,re,fe,Ke,{sampleRate:Ie}):(re.length>0||fe.length>0)&&R.queueTimingsOnly(n,r,re,fe,Ke)}else if(F.event==="metadata"){const me=F.data.mood;typeof me=="string"&&me.trim()&&(a=me.trim());const re=F.data.sentence_emotions;Array.isArray(re)&&(s.length=0,s.push(...Tt(re,a))),C()}else if(F.event==="error")throw new Error(String(F.data.message??"avatar_ttsWithPoses stream error"))}};for(;!(ie!==D||X.signal.aborted||!b);){const{done:j,value:F}=await Ae.read();F&&(ce+=rt.decode(F,{stream:!0}));const me=ce.split(`
 

package/dist/aitwin.js

@@ -2,9 +2,9 @@ import { jsxs as Kn, jsx as zt } from "react/jsx-runtime";
 import { forwardRef as jn, useRef as L, useState as Je, useEffect as Me, useCallback as ge, useMemo as Yt, useImperativeHandle as Gn } from "react";
 import { PipecatClient as zn } from "@pipecat-ai/client-js";
 import { WebSocketTransport as Yn, DailyMediaManager as Jn, ProtobufFrameSerializer as Xn } from "@pipecat-ai/websocket-transport";
-const Qn = "https://pub-607ad1fc22e2400eb57d17240aab857c.r2.dev", _n = {
+const Qn = "https://aitwin.bubu.social", _n = {
   publicBase: Qn
-}, Zn = "0.3.4", er = {
+}, Zn = "0.3.5", er = {
   version: Zn
 }, tr = {}, tt = "https://ai.aitwin.me", xt = `${_n.publicBase}/custom-faces`, nr = `${_n.publicBase}/aitwin-workers/v${er.version}/visemeDiffPreview.worker.js`;
 function rr(e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@streamoji/aitwin",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "description": "Embeddable React AI twin face with TTS lipsync",
   "type": "module",
   "main": "./dist/aitwin.cjs",