@streamoji/aitwin 0.7.4 → 0.7.6

package/dist/index.d.ts

@@ -15,6 +15,26 @@ export declare type AiTwinCaptionsOptions = {
     color?: string;
 };
 
+export declare type AiTwinGestureCatalog = {
+    success: boolean;
+    fullBody: boolean;
+    kind?: string;
+    poses: string[];
+    poseDetails?: Array<{
+        id: string;
+        label: string;
+        description?: string;
+        keywords?: string[];
+        category?: string;
+        speechMapped?: boolean;
+        cdnPath?: string;
+    }>;
+    encrypted: boolean;
+    pathTemplate: string;
+    cropMetaPath: string;
+    notes?: string;
+};
+
 export declare type AiTwinHandle = {
     speakText: (text: string, options?: AvatarTtsSpeakOptions) => Promise<void>;
     /**
@@ -40,6 +60,8 @@ export declare type AiTwinHandle = {
     clearEmotionPreview: () => void;
     isReady: () => boolean;
     getStatus: () => AvatarTtsLipsyncStatus;
+    /** Current utterance word queue (includes `gesture` when fullBody TTS mapped poses). */
+    getWordQueue: () => WordQueueItem[];
     /** Connect mic + WebSocket voice session over `/ws/voice`. */
     connect: (options?: VoiceConnectOptions) => Promise<void>;
     /** Disconnect voice session. */
@@ -146,6 +168,18 @@ export declare type AiTwinProps = {
      * Default: 500.
      */
     idleHandoffHoldMs?: number;
+    /**
+     * When true, skip loading/playing the cloud head idle MP4 (still sil only
+     * between speech). Demo pages can overlay a custom full-body idle instead.
+     * Default: false.
+     */
+    disableIdleVideo?: boolean;
+    /**
+     * Full-body (waist-up) twin: disable head idle and overlay encrypted CDN
+     * body idle + gesture videos under `custom-faces/{faceId}/ges/` as `G_00N.bin`.
+     * Create with `createAiTwinFace({ fullBody: true, … })`.
+     */
+    fullBody?: boolean;
     /**
      * Pixi only: cap WebGL MAX_TEXTURE_SIZE (e.g. 4096) to preview mobile-GPU
      * atlas handling. Omit to use the device limit.
@@ -153,6 +187,16 @@ export declare type AiTwinProps = {
     pixiMaxTextureSize?: number;
     /** Plain subtitles during TTS/voice playback. */
     captions?: boolean | AiTwinCaptionsOptions;
+    /** Logged-in visitor display name for conversation history / billing attribution. */
+    userName?: string;
+    /** Logged-in visitor email for conversation history. */
+    userEmail?: string;
+    /**
+     * Visitor note for the twin (what they want it to know).
+     * Passed through to speakWithReply / connect as `userContext`; backend
+     * injects into the LLM prompt for that request (not Redis-persisted).
+     */
+    userContext?: string;
 };
 
 export declare type AiTwinRecord = {
@@ -241,6 +285,11 @@ export declare type AiTwinWidgetProps = {
      * Default: false (inline in the mount target).
      */
     float?: boolean;
+    /**
+     * Full-body twin: body idle + gesture overlays from CDN.
+     * Create via `createAiTwinFace({ fullBody: true })`.
+     */
+    fullBody?: boolean;
     /**
      * Brain id for `speakWithReply` (Text mode) and `/ws/voice`.
      * When omitted, uses getAiTwin `brainId` if available.
@@ -286,6 +335,8 @@ export declare type AvatarTtsLipsyncController = {
      */
     speakWithReply: (message: string, options?: SpeakWithReplyOptions) => Promise<SpeakWithReplyResult>;
     stop: () => void;
+    /** Let a `delayPlayback` request become audible. */
+    releasePlaybackHold: () => void;
     getVisemeQueue: () => VisemeQueueItem[];
     getWordQueue: () => WordQueueItem[];
     /** Full utterance viseme queue (all SSE chunks, cumulative vtimes) for debug. */
@@ -310,6 +361,11 @@ export declare type AvatarTtsLipsyncController = {
 
 declare type AvatarTtsLipsyncControllerOptions = {
     onLipsyncDebug?: (snapshot: RealtimeLipsyncDebugSnapshot) => void;
+    /**
+     * When true, `/avatar_ttsWithPoses` uses the full-body PixVerse pose vocab
+     * and stamps `words[].gesture` with CDN codes (G_002, G_004, …).
+     */
+    fullBody?: boolean;
 };
 
 export declare type AvatarTtsLipsyncStatus = "idle" | "loading" | "speaking" | "done" | "error";
@@ -325,6 +381,16 @@ export declare type AvatarTtsSpeakOptions = {
      * Sent as `language` on `/avatar_ttsWithPoses`.
      */
     language?: TtsLanguage;
+    /**
+     * Override controller `fullBody` for this request. When true, backend maps
+     * speech to PixVerse gesture codes (G_002, …) on `words[].gesture`.
+     */
+    fullBody?: boolean;
+    /**
+     * Buffer audio and visemes, but do not start playback until
+     * `releasePlaybackHold()`. Used so full-body speech starts on an idle boundary.
+     */
+    delayPlayback?: boolean;
 };
 
 /** Pair transitions for playback, including sil→first and last→sil bookends. */
@@ -345,6 +411,23 @@ export declare function compactSkippableVisemes(series: VisemeQueueItem[]): Vise
 
 export declare const CONSONANT_SKIP_GROUP: Set<string>;
 
+/** Upload a portrait and enqueue the twin-face pipeline (poll with getAiTwinFaceStatus). */
+export declare function createAiTwinFace(options: CreateAiTwinFaceOptions): Promise<TwinFaceJobResponse>;
+
+export declare type CreateAiTwinFaceOptions = {
+    /** Bearer token (Firebase ID token or `client_…` JWT). */
+    authToken: string;
+    image: Blob | File;
+    name?: string;
+    /**
+     * Create a full-body (waist-up) twin: NanoBanana front-facing correction,
+     * then PixVerse idle + gesture videos (encrypted on CDN like idle.bin).
+     */
+    fullBody?: boolean;
+    /** Default: package `DEFAULT_API_BASE` (https://ai.aitwin.me). */
+    apiBase?: string;
+};
+
 export declare function createAvatarTtsLipsyncController(onStatus: (status: AvatarTtsLipsyncStatus) => void, apiBase?: string, options?: AvatarTtsLipsyncControllerOptions): AvatarTtsLipsyncController;
 
 export declare function createRealtimeVoiceLipsyncController(onStatus: (status: AvatarTtsLipsyncStatus) => void, initialOptions?: RealtimeVoiceOptions): RealtimeVoiceLipsyncController;
@@ -440,12 +523,43 @@ declare type FrameRenderTransitionMeta = {
     omittedByGap: number;
 };
 
+/** Default PixVerse full-body pose catalog (matches pipeline). */
+export declare const FULLBODY_GESTURE_POSES: readonly ["open_palm", "emphasis", "thumbs_up", "friendly_wave", "point_camera"];
+
+export declare type FullBodyCropMeta = {
+    y_end: number;
+    silHeight: number;
+    silWidth: number;
+    seamLockPx?: number;
+    poses?: string[];
+    encrypted?: boolean;
+    videoBin?: string;
+};
+
+export declare type FullBodyGesturePose = (typeof FULLBODY_GESTURE_POSES)[number];
+
+export declare function gesturesBaseForFace(faceId: string, cdnBase?: string): string;
+
+export declare function getAiTwinFaceStatus(avatarId: string, options: {
+    authToken: string;
+    apiBase?: string;
+}): Promise<TwinFaceJobResponse>;
+
+/** Server catalog for full-body PixVerse poses (idle + gestures). */
+export declare function getAiTwinGestureCatalog(apiBase?: string): Promise<AiTwinGestureCatalog>;
+
 /** Public CDN original portrait for custom-faces/{faceId}/ (same id as pipeline avatarId). */
 export declare function getAiTwinSourceUrl(faceId: string, cdnBase?: string, extension?: string): string;
 
 /** Public CDN thumbnail for custom-faces/{faceId}/ (same id as pipeline avatarId). */
 export declare function getAiTwinThumbnailUrl(faceId: string, cdnBase?: string): string;
 
+export declare function getFullBodyCropMetaUrl(gesturesBase: string): string;
+
+export declare function getFullBodyGestureBinUrl(gesturesBase: string, pose: string): string;
+
+export declare function isFullBodyGesturePose(id: string | null | undefined): id is FullBodyGesturePose;
+
 export declare function isTtsLanguageCode(raw: string): raw is TtsLanguage;
 
 export declare function isTtsLanguageSupported(engineId: TtsEngineId, language: string): boolean;
@@ -457,6 +571,12 @@ export declare function listSkippedVisemes(queue: VisemeQueueItem[]): string[];
 
 export declare const MIC_SAMPLE_RATE = 16000;
 
+/**
+ * Map a TTS `words[].gesture` token to a pose id.
+ * Accepts pose names (`open_palm`) and CDN codes (`G_002`).
+ */
+export declare function normalizeFullBodyGesturePose(id: string | null | undefined): FullBodyGesturePose | null;
+
 /** Google TTS streams Oculus viseme symbols; normalize casing and unknown ids. */
 export declare function normalizeOculusViseme(symbol: string | null | undefined): string;
 
@@ -598,6 +718,15 @@ export declare type SpeakWithReplyOptions = AvatarTtsSpeakOptions & {
     brainId?: string;
     brainUrl?: string;
     history?: SpeakWithReplyMessage[];
+    /** Logged-in visitor display name — persisted on non-meeting conversation history. */
+    userName?: string;
+    /** Logged-in visitor email — persisted on non-meeting conversation history. */
+    userEmail?: string;
+    /**
+     * Visitor note for the twin (what they want the twin to know).
+     * Sent each turn; backend injects into the LLM prompt (not Redis-persisted).
+     */
+    userContext?: string;
     /** Called as soon as spokenText arrives on the SSE stream. */
     onSpokenText?: (spokenText: string) => void;
     /** Called as soon as displayText arrives on the SSE stream. */
@@ -905,6 +1034,33 @@ declare type TwinBrainFields = {
     brainUrl?: string | null;
 };
 
+export declare type TwinFaceJobResponse = {
+    avatarId: string;
+    twinName: string;
+    status: TwinFaceJobStatus;
+    message: string;
+    success: boolean;
+    createdAt?: string;
+    updatedAt?: string;
+    error?: string;
+    deliverables?: string[];
+    twinPath?: string;
+    r2PublicBase?: string;
+    thumbnailUrl?: string;
+    sourceUrl?: string;
+    gender?: string;
+    genderConfidence?: number;
+    genderError?: string;
+    /** When true, pipeline runs waist-up NanoBanana + PixVerse gestures. */
+    fullBody?: boolean;
+    gesturesReady?: boolean;
+    gestures?: string[];
+    gesturesBase?: string;
+    gesturesError?: string;
+};
+
+export declare type TwinFaceJobStatus = "queued" | "running" | "succeeded" | "failed";
+
 export declare const VISEME_IDS: readonly ["aa", "E", "I", "O", "U", "PP", "FF", "DD", "SS", "TH", "CH", "RR", "kk", "nn"];
 
 /** Minimum gap (ms) between consecutive vtimes to keep a skippable viseme. */
@@ -943,6 +1099,20 @@ export declare type VoiceConnectOptions = {
      * Sent as `language` on `/ws/voice` and redirected `/avatar_ttsWithPoses` calls.
      */
     language?: TtsLanguage;
+    /**
+     * When true, redirected `/avatar_ttsWithPoses` uses full-body PixVerse pose vocab.
+     */
+    fullBody?: boolean;
+    /**
+     * Full-body only: resolve when the current body-idle clip ends so speech
+     * does not start mid-loop. Null when the body layer is not mounted.
+     */
+    waitForBodyIdle?: () => {
+        promise: Promise<void>;
+        cancel: () => void;
+    } | null;
+    /** Unpark body idle if a full-body speak is cancelled before playback. */
+    onBodyIdleCancelled?: () => void;
     /** Redis brain id (from getAiTwin `brainId`). Sent as `brainId` on `/ws/voice`. */
     brainId?: string;
     /** @deprecated Use `brainId`. */
@@ -962,6 +1132,15 @@ export declare type VoiceConnectOptions = {
     speakOnlyWhenSpokenTo?: boolean;
     /** Wake name used with `speakOnlyWhenSpokenTo` (e.g. twin.agentName). */
     avatarName?: string;
+    /** Logged-in visitor display name — persisted on non-meeting conversation history. */
+    userName?: string;
+    /** Logged-in visitor email — persisted on non-meeting conversation history. */
+    userEmail?: string;
+    /**
+     * Visitor note for the twin. Sent on connect; backend stores lead-scoped
+     * Sent on connect; backend injects into the LLM prompt (not Redis-persisted).
+     */
+    userContext?: string;
     /** HTTP API base; default `https://ai.aitwin.me`. */
     apiBase?: string;
     /** Full WebSocket URL override. Query params must be included. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@streamoji/aitwin",
-  "version": "0.7.4",
+  "version": "0.7.6",
   "description": "AiTwin.me Player SDK",
   "type": "module",
   "main": "./dist/aitwin.cjs",