@streamoji/aitwin 0.1.1

package/dist/assets/visemeDiffPreview.worker-B8Juk7ys.js

@@ -0,0 +1 @@
+(function(){"use strict";function K(e,t,r){return{width:e.crop?.source_width??t,height:e.crop?.source_height??r}}function J(e,t){const r=e.crop;return{x:t?.source_x??r?.x??0,y:t?.source_y??r?.y??0,width:t?.source_w??r?.width??e.frame_width??0,height:t?.source_h??r?.height??e.frame_height??0}}function L(e){return e?(e.w??0)>0&&(e.h??0)>0:!1}function G(e){const t=e.trim();if(t.length%2!==0)throw new Error("Invalid hex key");const r=new Uint8Array(t.length/2);for(let n=0;n<t.length;n+=2)r[n/2]=Number.parseInt(t.slice(n,n+2),16);return r}function I(e){return Uint8Array.from(e)}function j(e,t){const r=new Uint8Array(e.length+t.length);return r.set(e,0),r.set(t,e.length),r}async function Q(e,t){if(!t)throw new Error("Asset key is required for encrypted assets");if(e.byteLength<28)throw new Error("Encrypted payload too small");const r=new Uint8Array(e),n=I(r.subarray(0,12)),s=I(r.subarray(12,28)),o=I(r.subarray(28)),i=I(j(o,s)),_=await crypto.subtle.importKey("raw",G(t),{name:"AES-GCM"},!1,["decrypt"]);return crypto.subtle.decrypt({name:"AES-GCM",iv:n,tagLength:128},_,i)}async function $(e,t){const r=await fetch(e);if(!r.ok)throw new Error(`Failed to load encrypted asset: ${e}`);const n=await r.arrayBuffer();return Q(n,t)}async function X(e,t){if(!t)throw new Error("Missing asset key for encrypted JSON");const r=await $(e,t),n=new TextDecoder().decode(r);return JSON.parse(n)}async function Y(e){const t=new Blob([e],{type:"image/webp"});return createImageBitmap(t)}function Z(){throw new Error("Twin assets not configured; call setActiveTwinAssets first")}function V(){return Z().encrypted}const p=15,P=["t01_0.17","t02_0.33","t03_0.50","t04_0.67","t05_0.83"],tt=P.length,B=new Set(["aa__kk","aa__nn","aa__sil","CH__aa","CH__DD","CH__E","CH__FF","CH__I","CH__kk","CH__nn","CH__O","CH__PP","CH__RR","CH__sil","CH__SS","CH__TH","CH__U","DD__aa","DD__E","DD__FF","DD__I","DD__kk","DD__nn","DD__O","DD__PP","DD__RR","DD__sil","DD__SS","DD__TH","DD__U","E__aa","E__FF","E__I","E__kk","E__nn","E__O","E__PP","E__RR","E__sil","E__SS","E__TH","E__U","FF__aa","FF__I","FF__kk","FF__nn","FF__O","FF__PP","FF__RR","FF__sil","FF__SS","FF__TH","FF__U","I__aa","I__kk","I__nn","I__O","I__PP","I__RR","I__sil","I__SS","I__TH","I__U","kk__nn","kk__sil","nn__sil","O__aa","O__kk","O__nn","O__PP","O__RR","O__sil","O__SS","O__TH","O__U","PP__aa","PP__kk","PP__nn","PP__RR","PP__sil","PP__SS","PP__TH","PP__U","RR__aa","RR__kk","RR__nn","RR__sil","RR__SS","RR__TH","RR__U","SS__aa","SS__kk","SS__nn","SS__sil","SS__TH","SS__U","TH__aa","TH__kk","TH__nn","TH__sil","TH__U","U__aa","U__kk","U__nn","U__sil"]);function H(e,t){return`${e}__${t}`}function U(e,t){return`pairs/${e}/${t}_minus_sil.png`}function q(e){return`${e}_minus_sil.png`}function et(e,t){if(e===t)return[];const r=[],n=H(e,t);if(B.has(n)){for(const o of P)r.push(U(n,o));return r}const s=H(t,e);if(B.has(s)){for(let o=tt-1;o>=0;o--)r.push(U(s,P[o]));return r}return[]}function rt(e){return e<16?[]:e<35?[2]:e<51?[0,4]:e<67?[0,2,4]:e<83?[0,1,2,3]:[0,1,2,3,4]}function nt(e,t){return rt(t).map(n=>e[n]).filter(n=>n!=null)}function st(e,t){if(!V())return`${e}/${t}`;const r=t.split("/").pop()??t,n=r.includes(".")?r.slice(0,r.lastIndexOf(".")):r;return`${e}/${n}.bin`}function at(e,t,r,n,s=p){const o=new OffscreenCanvas(r,n),i=o.getContext("2d",{willReadFrequently:!0});i.drawImage(e,0,0,r,n);const _=i.getImageData(0,0,r,n),a=i.createImageData(r,n);a.data.set(_.data);const l=new OffscreenCanvas(r,n).getContext("2d",{willReadFrequently:!0});for(const E of t){l.clearRect(0,0,r,n),l.drawImage(E,0,0,r,n);const u=l.getImageData(0,0,r,n);for(let c=0;c<a.data.length;c+=4){const D=u.data[c],R=u.data[c+1],z=u.data[c+2];Math.max(D,R,z)>s&&(a.data[c]=D,a.data[c+1]=R,a.data[c+2]=z,a.data[c+3]=255)}}return i.putImageData(a,0,0),o}const S=-1;let f=null,y=null,d=null,O=null,F="sil";const A=new Map;function g(){if(!O)throw new Error("Worker assets not configured");return O}function ot(){return`${g().twinBase}/sil.png`}function it(){const e=g();return e.encrypted?`${e.binBase}/atlas.bin`:`${e.twinBase}/atlas.json`}function ct(){const e=g();return e.encrypted?`${e.binBase}/expression_atlas.bin`:`${e.twinBase}/expression_atlas.json`}function _t(){const e=g();return e.encrypted?e.binBase:e.twinBase}function b(){return g().encrypted}async function M(e){const t=await fetch(e);if(!t.ok)throw new Error(`Failed to load image: ${e}`);const r=await t.blob();return createImageBitmap(r)}async function v(e,t,r){const n=b()?await X(e,r):await fetch(e).then(a=>{if(!a.ok)throw new Error(`Failed to load ${e}`);return a.json()}),s=n.sheets?.[0];if(!s?.path)throw new Error(`${e} has no sheets[0].path`);const o=b()?await Y(await $(st(t,s.path),r??"")):await M(`${t}/${s.path}`),i=new Map;for(const a of n.cells??[])a.path&&i.set(a.path,a);const _=new Map;for(const a of n.sheets??[])_.set(a.index,a);return{atlas:o,atlasMeta:n,cellByPath:i,sheetByIndex:_,diffBase:t}}async function m(e){if(b()&&!e)throw new Error("Encrypted assets enabled but no key provided");if(y&&(!b()||d===(e??null)))return y;d=e??null,A.clear();const t=_t();return y=(async()=>{const r=await M(ot()),n=await v(it(),t,e);let s=null;try{s=await v(ct(),t,e)}catch{s=null}return{sil:r,viseme:n,expression:s}})(),y}function x(e){return K(e.viseme.atlasMeta,e.sil.width,e.sil.height)}function lt(e,t){if(!e.sheetByIndex.get(t.sheet??0))throw new Error(`Unknown sheet index: ${t.sheet}`);const n=new OffscreenCanvas(t.w,t.h);return n.getContext("2d",{willReadFrequently:!0}).drawImage(e.atlas,t.x,t.y,t.w,t.h,0,0,t.w,t.h),n}async function ft(e){return createImageBitmap(e)}async function N(e,t,r){const n=`${t.diffBase}::${r}`,s=A.get(n);if(s)return s;const o=(async()=>{const i=t.cellByPath.get(r);if(!i)throw new Error(`No atlas cell for path: ${r}`);const _=J(t.atlasMeta,i),a=x(e);if(L(i)){const E=lt(t,i),u=new OffscreenCanvas(a.width,a.height),c=u.getContext("2d");return c.fillStyle="#000",c.fillRect(0,0,a.width,a.height),c.drawImage(E,_.x,_.y),ft(u)}const h=`${t.diffBase}/${r}`,l=await M(h);if(l.width===a.width&&l.height===a.height)return l;throw l.close(),new Error(`Diff PNG wrong size for ${r}`)})();return A.set(n,o),o}function k(e,t){const r={type:"status",label:e,generation:t};self.postMessage(r)}function ut(e,t){const r={type:"error",requestId:e,message:t};self.postMessage(r)}async function C(e){if(!f||e!==S&&e!==w)return;const t=await createImageBitmap(f);if(e!==S&&e!==w){t.close();return}const r={type:"frame",bitmap:t,generation:e};self.postMessage(r,{transfer:[t]})}function dt(e,t,r){return new Promise((n,s)=>{setTimeout(()=>{t!==r?s(new DOMException("Aborted","AbortError")):n()},e)})}async function W(e,t,r,n=p){const s=await m(d??void 0),{width:o,height:i}=x(s);(e.width!==o||e.height!==i)&&(e.width=o,e.height=i);const _=[];if(t){if(!s.viseme.cellByPath.get(t))throw new Error(`No viseme atlas cell for path: ${t}`);_.push(await N(s,s.viseme,t))}if(r&&s.expression){if(!s.expression.cellByPath.get(r))throw new Error(`No expression atlas cell for path: ${r}`);_.push(await N(s,s.expression,r))}const a=e.getContext("2d");if(a.clearRect(0,0,o,i),_.length===0){a.drawImage(s.sil,0,0,o,i);return}const h=at(s.sil,_,o,i,n);a.drawImage(h,0,0)}async function T(e,t,r,n,s=p){await W(e,t,r,s),await C(n)}async function wt(e,t,r){F="sil";const n=await m(d??void 0);if(t!==r)throw new DOMException("Aborted","AbortError");const{width:s,height:o}=x(n);e.width=s,e.height=o;const i=e.getContext("2d");i.clearRect(0,0,s,o),i.drawImage(n.sil,0,0,s,o),await C(t)}async function ht(e,t,r,n,s={}){const o=s.from??F,i=s.transitionDurationMs??400,_=s.gapMs??i,a=s.threshold??p;if(await m(d??void 0),o===t){if(k(t,r),await T(e,q(t),null,r,a),r!==n)throw new DOMException("Aborted","AbortError");F=t;return}const h=et(o,t),l=nt(h,_),E=q(t),u=l.length>0?i/l.length:0;l.length===0&&k(`${o} → ${t} (no pair; final only)`,r);for(let c=0;c<l.length;c++){if(r!==n)throw new DOMException("Aborted","AbortError");const D=l[c],R=D.split("/").pop()?.replace("_minus_sil.png","")??`frame ${c}`;if(k(`${o} → ${t} (${R})`,r),await T(e,D,null,r,a),r!==n)throw new DOMException("Aborted","AbortError");u>0&&await dt(u,r,n)}if(r!==n)throw new DOMException("Aborted","AbortError");if(k(t,r),await T(e,E,null,r,a),r!==n)throw new DOMException("Aborted","AbortError");F=t}let w=0;self.onmessage=e=>{const t=e.data;(async()=>{try{switch(t.type){case"init":{f=new OffscreenCanvas(1,1);const r={type:"ready",requestId:t.requestId};self.postMessage(r);break}case"loadAssets":{O=t.urls,y=null,d=null,A.clear(),await m(t.keyHex);const r={type:"loadAssetsDone",requestId:t.requestId};self.postMessage(r);break}case"drawFrame":{if(!f)throw new Error("Worker not initialized");await m(d??void 0),await W(f,t.diffPath,t.expressionDiffPath??null,t.threshold??p),await C(S);const r={type:"renderDone",requestId:t.requestId,generation:S};self.postMessage(r);break}case"renderSil":{if(w=t.generation,!f)throw new Error("Worker not initialized");await wt(f,t.generation,w);const r={type:"renderDone",requestId:t.requestId,generation:t.generation};self.postMessage(r);break}case"renderViseme":{if(w=t.generation,!f)throw new Error("Worker not initialized");await ht(f,t.to,t.generation,w,{from:t.from,transitionDurationMs:t.transitionDurationMs,gapMs:t.gapMs,threshold:t.threshold});const r={type:"renderDone",requestId:t.requestId,generation:t.generation};self.postMessage(r);break}default:break}}catch(r){if(r instanceof DOMException&&r.name==="AbortError"){const n={type:"renderAborted",requestId:t.requestId,generation:"generation"in t?t.generation:w};self.postMessage(n);return}ut(t.requestId,r instanceof Error?r.message:String(r))}})()}})();

package/dist/index.d.ts

@@ -0,0 +1,176 @@
+import { CSSProperties } from 'react';
+import { ForwardRefExoticComponent } from 'react';
+import { RefAttributes } from 'react';
+
+export declare const AiTwin: ForwardRefExoticComponent<AiTwinProps & RefAttributes<AiTwinHandle>>;
+
+export declare type AiTwinHandle = {
+    speakText: (text: string, options?: AvatarTtsSpeakOptions) => Promise<void>;
+    stop: () => void;
+    setTtsProvider: (provider: TtsProvider) => void;
+    renderViseme: (toViseme: string, options?: AiTwinRenderVisemeOptions) => Promise<void>;
+    isReady: () => boolean;
+    getStatus: () => AvatarTtsLipsyncStatus;
+};
+
+export declare class AiTwinNotFoundError extends Error {
+    constructor(message?: string);
+}
+
+export declare type AiTwinProps = {
+    /** Twin id for getAiTwin (e.g. `olivia`). Omit when `assets` is set. */
+    id?: string;
+    /**
+     * Preconfigured asset URLs (e.g. local `twins/blondelady`).
+     * When set, skips getAiTwin.
+     */
+    assets?: TwinAssetUrls;
+    /**
+     * Bearer token for TTS and encrypted asset decryption.
+     * When omitted, a dev token is fetched via getAuthToken.
+     */
+    authToken?: string;
+    /** Streamoji API base (default: https://ai.streamoji.com). */
+    apiBase?: string;
+    /** R2 CDN base for custom face bundles (default: pub R2 custom-faces). */
+    facesCdnBase?: string;
+    /** TTS provider when using `assets` (default: `google`). */
+    tts?: TtsProvider;
+    /** TTS voice id override (provider-specific). */
+    voiceId?: string;
+    /** API speakingRate (default: 0.85). */
+    speakingRate?: number;
+    className?: string;
+    style?: CSSProperties;
+    /** Optional canvas style (e.g. maxHeight for lab pages). */
+    canvasStyle?: CSSProperties;
+    onStatusChange?: (status: AvatarTtsLipsyncStatus) => void;
+    /** Canvas compositor label (viseme id, transition, idle, etc.). */
+    onDisplayStatus?: (label: string) => void;
+    onError?: (message: string) => void;
+    /** Called when twin metadata, auth, and face assets are ready. */
+    onReady?: () => void;
+    /** When false, errors only go to `onError` (no overlay on the canvas). Default: true. */
+    showErrorOverlay?: boolean;
+};
+
+export declare type AiTwinRecord = {
+    id: string;
+    faceId: string;
+    tts: TtsProvider_2;
+    voiceId?: string;
+    gender?: string;
+    createdAt?: string;
+    updatedAt?: string;
+};
+
+export declare type AiTwinRenderVisemeOptions = {
+    from?: string;
+    transitionDurationMs?: number;
+    gapMs?: number;
+};
+
+export declare type AvatarTtsLipsyncController = {
+    setDeveloperToken: (token: string) => void;
+    setTtsProvider: (provider: TtsProvider) => void;
+    getTtsProvider: () => TtsProvider;
+    speak: (userQuery: string, options?: AvatarTtsSpeakOptions) => Promise<void>;
+    stop: () => void;
+    getVisemeQueue: () => VisemeQueueItem[];
+    getWordQueue: () => WordQueueItem[];
+    getSentenceEmotions: () => SentenceEmotion[];
+    getStreamMood: () => string;
+    getPlaybackElapsedMs: () => number;
+    isSpeaking: () => boolean;
+    setPlaybackSpeed: (rate: number) => void;
+    getPlaybackSpeed: () => number;
+    subscribe: (listener: () => void) => () => void;
+};
+
+export declare type AvatarTtsLipsyncStatus = "idle" | "loading" | "speaking" | "done" | "error";
+
+export declare type AvatarTtsSpeakOptions = {
+    voiceId?: string;
+    /** Passed to API as speakingRate (optional; server defaults to 1.0). */
+    speakingRate?: number;
+    /** Overrides controller default for this request. */
+    tts?: TtsProvider;
+};
+
+export declare function createAvatarTtsLipsyncController(onStatus: (status: AvatarTtsLipsyncStatus) => void, apiBase?: string): AvatarTtsLipsyncController;
+
+export declare const DEFAULT_API_BASE = "https://ai.streamoji.com";
+
+export declare const DEFAULT_FACES_CDN_BASE = "https://pub-607ad1fc22e2400eb57d17240aab857c.r2.dev/custom-faces";
+
+export declare function fetchAiTwin(id: string, baseUrl?: string): Promise<AiTwinRecord>;
+
+/** Bearer token for /avatar_ttsWithPoses and /api/session-value. */
+export declare function fetchDevAuthToken(): Promise<string>;
+
+/** Google TTS streams Oculus viseme symbols; normalize casing and unknown ids. */
+export declare function normalizeOculusViseme(symbol: string | null | undefined): string;
+
+/** Oculus viseme ids for Talking Lady stills and pair transitions (15 standard). */
+export declare const OCULUS_VISEME_IDS: readonly ["aa", "CH", "DD", "E", "FF", "I", "O", "PP", "RR", "SS", "TH", "U", "kk", "nn", "sil"];
+
+/** Pick the active viseme at playback offset (latest vtime wins on overlap). */
+export declare function resolveVisemeAtTime(queue: VisemeQueueItem[], elapsedMs: number): string;
+
+/** Word active at playback offset (ms). */
+export declare function resolveWordAtTime(words: WordQueueItem[], timeMs: number): WordQueueItem | null;
+
+export declare interface SentenceEmotion {
+    sentence_index: number;
+    text: string;
+    sentiment?: string;
+    emotion: string;
+    start_word: number;
+    end_word: number;
+}
+
+export declare const SPEAKING_RATE_MAX = 1.5;
+
+export declare const SPEAKING_RATE_MIN = 0.5;
+
+export declare type TtsProvider = "google" | "inworld" | "cartesia";
+
+declare type TtsProvider_2 = "google" | "inworld" | "cartesia";
+
+/** Runtime face bundle URLs (R2 custom-faces or local /twins/…). */
+export declare type TwinAssetUrls = {
+    /** sil.png, idle.mp4 */
+    twinBase: string;
+    /** atlas.bin and encrypted sheets */
+    binBase: string;
+    encrypted: boolean;
+};
+
+export declare const VISEME_IDS: readonly ["aa", "E", "I", "O", "U", "PP", "FF", "DD", "SS", "TH", "CH", "RR", "kk", "nn"];
+
+/** Preset Cartesia voice (Katie) for testing without a custom clone. */
+export declare const VISEME_TEST_CARTESIA_VOICE_ID = "f786b574-daa5-4673-aa0c-cbe3e8534c02";
+
+export declare const VISEME_TEST_SPEAKING_RATE = 0.85;
+
+/** Viseme-test page: fixed female voice + speech speed for /avatar_ttsWithPoses. */
+export declare const VISEME_TEST_VOICE_ID = "Olivia";
+
+export declare interface VisemeQueueItem {
+    viseme: string;
+    weight: number;
+    vtime: number;
+    vduration: number;
+}
+
+export declare interface WordQueueItem {
+    word: string;
+    wtime: number;
+    wduration: number;
+    /** Index in utterance word list (for sentence_emotions ranges). */
+    queueIndex: number;
+    gesture?: string | null;
+    emotion?: string;
+}
+
+export { }

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@streamoji/aitwin",
+  "version": "0.1.1",
+  "description": "Embeddable React AI twin face with TTS lipsync",
+  "type": "module",
+  "main": "./dist/aitwin.cjs",
+  "module": "./dist/aitwin.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/aitwin.js",
+      "require": "./dist/aitwin.cjs"
+    }
+  },
+  "files": [
+    "dist/aitwin.js",
+    "dist/aitwin.cjs",
+    "dist/index.d.ts",
+    "dist/assets",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "vite build",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "prepublishOnly": "npm run build"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "react": "^18.0.0 || ^19.0.0",
+    "react-dom": "^18.0.0 || ^19.0.0"
+  },
+  "devDependencies": {
+    "@types/react": "^19.2.5",
+    "@types/react-dom": "^19.2.3",
+    "@vitejs/plugin-react": "^5.1.1",
+    "react": "^19.2.0",
+    "react-dom": "^19.2.0",
+    "typescript": "^5.9.3",
+    "vite": "^7.2.4",
+    "vite-plugin-dts": "^4.5.4"
+  }
+}