@omote/babylon 0.2.0

package/dist/index.d.cts

@@ -0,0 +1,214 @@
+import * as _omote_core from '@omote/core';
+import { FaceCompositorConfig, CharacterControllerConfig, EmotionWeights, ConversationalState, A2EOrchestratorConfig } from '@omote/core';
+import { AbstractMesh, TransformNode, Scene, Camera } from '@babylonjs/core';
+
+/**
+ * SceneDiscovery — Discover avatar structure from a Babylon.js scene graph.
+ *
+ * Finds morph targets, head/neck/eye bones, and pre-computes a fast index
+ * mapping from LAM_BLENDSHAPES order to per-mesh morph target indices.
+ *
+ * @category Babylon
+ */
+
+interface MorphIndexEntry {
+    mesh: AbstractMesh;
+    /** indices[lamIndex] = morphTargetIndex (or -1 if not found) */
+    indices: Int16Array;
+}
+interface SceneDiscoveryResult {
+    /** All meshes under the root that have a MorphTargetManager */
+    meshes: AbstractMesh[];
+    /** Head bone (TransformNode named 'Head') or null */
+    headBone: TransformNode | null;
+    /** Neck bone (TransformNode named 'Neck') or null */
+    neckBone: TransformNode | null;
+    /** Left eye bone (TransformNode named 'LeftEye') or null */
+    leftEyeBone: TransformNode | null;
+    /** Right eye bone (TransformNode named 'RightEye') or null */
+    rightEyeBone: TransformNode | null;
+    /** Pre-computed morph index mapping per mesh */
+    morphEntries: MorphIndexEntry[];
+    /** Primary face mesh (prefers 'Head_Mesh', falls back to first with morph targets) */
+    faceMesh: AbstractMesh | null;
+    /** Total number of ARKit blendshapes successfully mapped across all meshes */
+    mappedBlendshapeCount: number;
+}
+/**
+ * Discover avatar structure from a Babylon.js root node.
+ *
+ * Traverses the scene graph under `root` to find:
+ * - Meshes with MorphTargetManagers
+ * - Head, neck, and eye bones
+ * - Pre-computed LAM_BLENDSHAPES -> morph target index mapping
+ *
+ * @param root - The root AbstractMesh (typically loaded via SceneLoader)
+ * @returns SceneDiscoveryResult with all discovered components
+ */
+declare function discoverScene(root: AbstractMesh): SceneDiscoveryResult;
+
+/** Generic frame source -- any object that emits 'frame' events */
+interface FrameSource {
+    on(event: 'frame', callback: (frame: {
+        blendshapes: Float32Array;
+    }) => void): void;
+    off?(event: 'frame', callback: (...args: any[]) => void): void;
+}
+interface OmoteAvatarOptions {
+    /** Root mesh of the avatar (typically loaded via SceneLoader) */
+    target: AbstractMesh;
+    /** Babylon.js scene */
+    scene: Scene;
+    /** FaceCompositor configuration */
+    compositor?: FaceCompositorConfig;
+    /** Gaze tracking configuration */
+    gaze?: CharacterControllerConfig['gaze'];
+    /**
+     * Register scene.registerBeforeRender() for automatic update.
+     * Requires setCamera() to be called before the first frame.
+     * Default: false
+     */
+    autoUpdate?: boolean;
+}
+declare class OmoteAvatar {
+    private readonly controller;
+    private readonly discovery;
+    private readonly scene;
+    private currentBlendshapes;
+    private _emotion;
+    private _isSpeaking;
+    private _state;
+    private _audioEnergy;
+    private _camera;
+    private frameSourceCallback;
+    private connectedSource;
+    private renderCallback;
+    private lastTime;
+    constructor(options: OmoteAvatarOptions);
+    /**
+     * Call each frame with delta time and camera.
+     *
+     * Runs CharacterController (compositor, gaze, procedural life) then writes
+     * blendshapes to morph targets and applies head rotation.
+     *
+     * If using autoUpdate, this is called automatically via scene.registerBeforeRender().
+     *
+     * @param delta - Time since last frame in seconds
+     * @param camera - Active Babylon.js camera (for gaze tracking)
+     * @param avatarRotationY - Optional avatar Y rotation in radians for gaze compensation
+     */
+    update(delta: number, camera: Camera, avatarRotationY?: number): void;
+    /**
+     * Connect a frame source (PlaybackPipeline, MicLipSync, etc.).
+     *
+     * Automatically listens for 'frame' events and stores the latest blendshapes.
+     * Only one source can be connected at a time; calling again disconnects the previous.
+     */
+    connectFrameSource(source: FrameSource): void;
+    /** Disconnect the current frame source (if any). */
+    disconnectFrameSource(): void;
+    /** Set blendshapes directly (alternative to connectFrameSource). */
+    setFrame(blendshapes: Float32Array): void;
+    /** Set emotion (string preset like 'happy' or EmotionWeights object). */
+    setEmotion(emotion: string | EmotionWeights): void;
+    /** Set whether the avatar is currently speaking (drives mouth emphasis). */
+    setSpeaking(speaking: boolean): void;
+    /** Set conversational state (idle, listening, thinking, speaking). */
+    setState(state: ConversationalState): void;
+    /** Set audio energy level (0-1, drives emphasis/gesture intensity). */
+    setAudioEnergy(energy: number): void;
+    /**
+     * Set the active camera for gaze tracking.
+     * Required when using autoUpdate. Can also be passed directly to update().
+     */
+    setCamera(camera: Camera): void;
+    /** Access underlying FaceCompositor for advanced use. */
+    get compositor(): _omote_core.FaceCompositor;
+    /** Access SceneDiscoveryResult (meshes, bones, morph entries). */
+    get parts(): SceneDiscoveryResult;
+    /** Whether the scene has any mapped morph targets. */
+    get hasMorphTargets(): boolean;
+    /** Number of successfully mapped ARKit blendshapes. */
+    get mappedBlendshapeCount(): number;
+    /** Reset all state (smoothing, life layer, emotions). */
+    reset(): void;
+    /** Clean up all resources: disconnect frame source, unregister render loop, dispose controller. */
+    dispose(): void;
+    private registerAutoUpdate;
+}
+
+/**
+ * BlendshapeWriter — Zero-lookup hot-path blendshape application.
+ *
+ * Uses pre-computed Int16Array indices from SceneDiscovery for O(1) lookups
+ * per blendshape per mesh. No Map.get(), no string comparison on the hot path.
+ *
+ * @category Babylon
+ */
+
+/**
+ * Write 52 ARKit blendshapes to Babylon.js morph targets.
+ *
+ * Uses pre-computed indices for zero-lookup hot path.
+ * Call this every frame after CharacterController.update().
+ *
+ * @param blendshapes - 52-element Float32Array in LAM_BLENDSHAPES order
+ * @param morphEntries - Pre-computed index mappings from discoverScene()
+ */
+declare function writeBlendshapes(blendshapes: Float32Array, morphEntries: MorphIndexEntry[]): void;
+
+interface BlendshapeControllerOptions {
+    /** Blendshape names in order (default: LAM_BLENDSHAPES, 52 ARKit) */
+    names?: readonly string[];
+    /** Smoothing factor 0-1 (0 = no change, 1 = snap to target). Default: 0.7 */
+    smoothing?: number;
+    /** Traverse target for child meshes with MorphTargetManager automatically. Default: true */
+    autoFind?: boolean;
+    /** Register scene.registerBeforeRender() callback automatically. Default: false */
+    autoRegister?: boolean;
+    /** Called when meshes with morph targets are found */
+    onMeshesFound?: (meshes: AbstractMesh[]) => void;
+}
+declare class BlendshapeController {
+    private _meshes;
+    private nameToIndex;
+    private currentWeights;
+    private blendshapeNames;
+    private smoothing;
+    private scene;
+    private renderObserver;
+    private pendingWeights;
+    private onMeshesFound?;
+    constructor(target: AbstractMesh, scene: Scene, options?: BlendshapeControllerOptions);
+    get meshes(): AbstractMesh[];
+    /** Normalize Babylon morph target names (strip common prefixes) */
+    private normalizeName;
+    setTarget(target: AbstractMesh): void;
+    update(weights: Float32Array | number[]): void;
+    /** Buffer weights for the next render loop tick (used with autoRegister) */
+    setWeightsForNextFrame(weights: Float32Array | number[]): void;
+    private registerRenderLoop;
+    dispose(): void;
+}
+
+interface OmoteA2EOptions extends A2EOrchestratorConfig {
+    target: AbstractMesh;
+    scene: Scene;
+    controllerOptions?: BlendshapeControllerOptions;
+}
+/** @deprecated Use {@link OmoteAvatar} instead. OmoteA2E will be removed in v0.8.0. */
+declare class OmoteA2E {
+    private orchestrator;
+    private controller;
+    constructor(options: OmoteA2EOptions);
+    load(): Promise<void>;
+    start(): Promise<void>;
+    stop(): void;
+    update(): void;
+    dispose(): Promise<void>;
+    get isReady(): boolean;
+    get isStreaming(): boolean;
+    get backend(): string | null;
+}
+
+export { BlendshapeController, type BlendshapeControllerOptions, type FrameSource, type MorphIndexEntry, OmoteA2E, type OmoteA2EOptions, OmoteAvatar, type OmoteAvatarOptions, type SceneDiscoveryResult, discoverScene, writeBlendshapes };

package/dist/index.d.ts

@@ -0,0 +1,214 @@
+import * as _omote_core from '@omote/core';
+import { FaceCompositorConfig, CharacterControllerConfig, EmotionWeights, ConversationalState, A2EOrchestratorConfig } from '@omote/core';
+import { AbstractMesh, TransformNode, Scene, Camera } from '@babylonjs/core';
+
+/**
+ * SceneDiscovery — Discover avatar structure from a Babylon.js scene graph.
+ *
+ * Finds morph targets, head/neck/eye bones, and pre-computes a fast index
+ * mapping from LAM_BLENDSHAPES order to per-mesh morph target indices.
+ *
+ * @category Babylon
+ */
+
+interface MorphIndexEntry {
+    mesh: AbstractMesh;
+    /** indices[lamIndex] = morphTargetIndex (or -1 if not found) */
+    indices: Int16Array;
+}
+interface SceneDiscoveryResult {
+    /** All meshes under the root that have a MorphTargetManager */
+    meshes: AbstractMesh[];
+    /** Head bone (TransformNode named 'Head') or null */
+    headBone: TransformNode | null;
+    /** Neck bone (TransformNode named 'Neck') or null */
+    neckBone: TransformNode | null;
+    /** Left eye bone (TransformNode named 'LeftEye') or null */
+    leftEyeBone: TransformNode | null;
+    /** Right eye bone (TransformNode named 'RightEye') or null */
+    rightEyeBone: TransformNode | null;
+    /** Pre-computed morph index mapping per mesh */
+    morphEntries: MorphIndexEntry[];
+    /** Primary face mesh (prefers 'Head_Mesh', falls back to first with morph targets) */
+    faceMesh: AbstractMesh | null;
+    /** Total number of ARKit blendshapes successfully mapped across all meshes */
+    mappedBlendshapeCount: number;
+}
+/**
+ * Discover avatar structure from a Babylon.js root node.
+ *
+ * Traverses the scene graph under `root` to find:
+ * - Meshes with MorphTargetManagers
+ * - Head, neck, and eye bones
+ * - Pre-computed LAM_BLENDSHAPES -> morph target index mapping
+ *
+ * @param root - The root AbstractMesh (typically loaded via SceneLoader)
+ * @returns SceneDiscoveryResult with all discovered components
+ */
+declare function discoverScene(root: AbstractMesh): SceneDiscoveryResult;
+
+/** Generic frame source -- any object that emits 'frame' events */
+interface FrameSource {
+    on(event: 'frame', callback: (frame: {
+        blendshapes: Float32Array;
+    }) => void): void;
+    off?(event: 'frame', callback: (...args: any[]) => void): void;
+}
+interface OmoteAvatarOptions {
+    /** Root mesh of the avatar (typically loaded via SceneLoader) */
+    target: AbstractMesh;
+    /** Babylon.js scene */
+    scene: Scene;
+    /** FaceCompositor configuration */
+    compositor?: FaceCompositorConfig;
+    /** Gaze tracking configuration */
+    gaze?: CharacterControllerConfig['gaze'];
+    /**
+     * Register scene.registerBeforeRender() for automatic update.
+     * Requires setCamera() to be called before the first frame.
+     * Default: false
+     */
+    autoUpdate?: boolean;
+}
+declare class OmoteAvatar {
+    private readonly controller;
+    private readonly discovery;
+    private readonly scene;
+    private currentBlendshapes;
+    private _emotion;
+    private _isSpeaking;
+    private _state;
+    private _audioEnergy;
+    private _camera;
+    private frameSourceCallback;
+    private connectedSource;
+    private renderCallback;
+    private lastTime;
+    constructor(options: OmoteAvatarOptions);
+    /**
+     * Call each frame with delta time and camera.
+     *
+     * Runs CharacterController (compositor, gaze, procedural life) then writes
+     * blendshapes to morph targets and applies head rotation.
+     *
+     * If using autoUpdate, this is called automatically via scene.registerBeforeRender().
+     *
+     * @param delta - Time since last frame in seconds
+     * @param camera - Active Babylon.js camera (for gaze tracking)
+     * @param avatarRotationY - Optional avatar Y rotation in radians for gaze compensation
+     */
+    update(delta: number, camera: Camera, avatarRotationY?: number): void;
+    /**
+     * Connect a frame source (PlaybackPipeline, MicLipSync, etc.).
+     *
+     * Automatically listens for 'frame' events and stores the latest blendshapes.
+     * Only one source can be connected at a time; calling again disconnects the previous.
+     */
+    connectFrameSource(source: FrameSource): void;
+    /** Disconnect the current frame source (if any). */
+    disconnectFrameSource(): void;
+    /** Set blendshapes directly (alternative to connectFrameSource). */
+    setFrame(blendshapes: Float32Array): void;
+    /** Set emotion (string preset like 'happy' or EmotionWeights object). */
+    setEmotion(emotion: string | EmotionWeights): void;
+    /** Set whether the avatar is currently speaking (drives mouth emphasis). */
+    setSpeaking(speaking: boolean): void;
+    /** Set conversational state (idle, listening, thinking, speaking). */
+    setState(state: ConversationalState): void;
+    /** Set audio energy level (0-1, drives emphasis/gesture intensity). */
+    setAudioEnergy(energy: number): void;
+    /**
+     * Set the active camera for gaze tracking.
+     * Required when using autoUpdate. Can also be passed directly to update().
+     */
+    setCamera(camera: Camera): void;
+    /** Access underlying FaceCompositor for advanced use. */
+    get compositor(): _omote_core.FaceCompositor;
+    /** Access SceneDiscoveryResult (meshes, bones, morph entries). */
+    get parts(): SceneDiscoveryResult;
+    /** Whether the scene has any mapped morph targets. */
+    get hasMorphTargets(): boolean;
+    /** Number of successfully mapped ARKit blendshapes. */
+    get mappedBlendshapeCount(): number;
+    /** Reset all state (smoothing, life layer, emotions). */
+    reset(): void;
+    /** Clean up all resources: disconnect frame source, unregister render loop, dispose controller. */
+    dispose(): void;
+    private registerAutoUpdate;
+}
+
+/**
+ * BlendshapeWriter — Zero-lookup hot-path blendshape application.
+ *
+ * Uses pre-computed Int16Array indices from SceneDiscovery for O(1) lookups
+ * per blendshape per mesh. No Map.get(), no string comparison on the hot path.
+ *
+ * @category Babylon
+ */
+
+/**
+ * Write 52 ARKit blendshapes to Babylon.js morph targets.
+ *
+ * Uses pre-computed indices for zero-lookup hot path.
+ * Call this every frame after CharacterController.update().
+ *
+ * @param blendshapes - 52-element Float32Array in LAM_BLENDSHAPES order
+ * @param morphEntries - Pre-computed index mappings from discoverScene()
+ */
+declare function writeBlendshapes(blendshapes: Float32Array, morphEntries: MorphIndexEntry[]): void;
+
+interface BlendshapeControllerOptions {
+    /** Blendshape names in order (default: LAM_BLENDSHAPES, 52 ARKit) */
+    names?: readonly string[];
+    /** Smoothing factor 0-1 (0 = no change, 1 = snap to target). Default: 0.7 */
+    smoothing?: number;
+    /** Traverse target for child meshes with MorphTargetManager automatically. Default: true */
+    autoFind?: boolean;
+    /** Register scene.registerBeforeRender() callback automatically. Default: false */
+    autoRegister?: boolean;
+    /** Called when meshes with morph targets are found */
+    onMeshesFound?: (meshes: AbstractMesh[]) => void;
+}
+declare class BlendshapeController {
+    private _meshes;
+    private nameToIndex;
+    private currentWeights;
+    private blendshapeNames;
+    private smoothing;
+    private scene;
+    private renderObserver;
+    private pendingWeights;
+    private onMeshesFound?;
+    constructor(target: AbstractMesh, scene: Scene, options?: BlendshapeControllerOptions);
+    get meshes(): AbstractMesh[];
+    /** Normalize Babylon morph target names (strip common prefixes) */
+    private normalizeName;
+    setTarget(target: AbstractMesh): void;
+    update(weights: Float32Array | number[]): void;
+    /** Buffer weights for the next render loop tick (used with autoRegister) */
+    setWeightsForNextFrame(weights: Float32Array | number[]): void;
+    private registerRenderLoop;
+    dispose(): void;
+}
+
+interface OmoteA2EOptions extends A2EOrchestratorConfig {
+    target: AbstractMesh;
+    scene: Scene;
+    controllerOptions?: BlendshapeControllerOptions;
+}
+/** @deprecated Use {@link OmoteAvatar} instead. OmoteA2E will be removed in v0.8.0. */
+declare class OmoteA2E {
+    private orchestrator;
+    private controller;
+    constructor(options: OmoteA2EOptions);
+    load(): Promise<void>;
+    start(): Promise<void>;
+    stop(): void;
+    update(): void;
+    dispose(): Promise<void>;
+    get isReady(): boolean;
+    get isStreaming(): boolean;
+    get backend(): string | null;
+}
+
+export { BlendshapeController, type BlendshapeControllerOptions, type FrameSource, type MorphIndexEntry, OmoteA2E, type OmoteA2EOptions, OmoteAvatar, type OmoteAvatarOptions, type SceneDiscoveryResult, discoverScene, writeBlendshapes };