@elizaos/app-core 2.0.0-alpha.37

package/src/components/avatar/VrmFootShadow.ts

@@ -0,0 +1,96 @@
+import type { VRM } from "@pixiv/three-vrm";
+import * as THREE from "three";
+
+/**
+ * Manages a soft radial shadow disc placed beneath the VRM avatar's feet.
+ */
+export class VrmFootShadow {
+  private mesh: THREE.Mesh | null = null;
+  private readonly tempLeft = new THREE.Vector3();
+  private readonly tempRight = new THREE.Vector3();
+  private readonly tempCenter = new THREE.Vector3();
+
+  /** Create (or recreate) the shadow disc and add it to `scene`. */
+  create(scene: THREE.Scene): void {
+    this.dispose(scene);
+
+    const shadowCanvas = document.createElement("canvas");
+    shadowCanvas.width = 128;
+    shadowCanvas.height = 128;
+
+    const context = shadowCanvas.getContext("2d");
+    if (!context) return;
+
+    const gradient = context.createRadialGradient(64, 64, 0, 64, 64, 64);
+    gradient.addColorStop(0, "rgba(0, 0, 0, 0.4)");
+    gradient.addColorStop(0.5, "rgba(0, 0, 0, 0.2)");
+    gradient.addColorStop(1, "rgba(0, 0, 0, 0)");
+
+    context.fillStyle = gradient;
+    context.fillRect(0, 0, 128, 128);
+
+    const shadowTexture = new THREE.CanvasTexture(shadowCanvas);
+    const shadowGeometry = new THREE.PlaneGeometry(2.2, 2.2);
+    const shadowMaterial = new THREE.MeshBasicMaterial({
+      map: shadowTexture,
+      transparent: true,
+      alphaTest: 0.001,
+      depthWrite: false,
+    });
+
+    const shadow = new THREE.Mesh(shadowGeometry, shadowMaterial);
+    shadow.rotation.x = -Math.PI / 2;
+    shadow.position.set(0, 0, 0);
+    scene.add(shadow);
+    this.mesh = shadow;
+  }
+
+  /** Update shadow position to follow the VRM on the ground plane. */
+  update(vrm: VRM): void {
+    if (!this.mesh) return;
+    vrm.scene.updateMatrixWorld(true);
+    const leftFoot = vrm.humanoid?.getNormalizedBoneNode("leftFoot");
+    const rightFoot = vrm.humanoid?.getNormalizedBoneNode("rightFoot");
+
+    if (leftFoot && rightFoot) {
+      leftFoot.getWorldPosition(this.tempLeft);
+      rightFoot.getWorldPosition(this.tempRight);
+      this.tempCenter
+        .copy(this.tempLeft)
+        .add(this.tempRight)
+        .multiplyScalar(0.5);
+      this.mesh.position.copy(this.tempCenter);
+      return;
+    }
+
+    const bounds = new THREE.Box3().setFromObject(vrm.scene);
+    if (!bounds.isEmpty()) {
+      bounds.getCenter(this.tempCenter);
+      this.mesh.position.set(
+        this.tempCenter.x,
+        bounds.min.y,
+        this.tempCenter.z,
+      );
+    }
+  }
+
+  /** Remove and dispose of the shadow mesh, releasing GPU resources. */
+  dispose(scene: THREE.Scene): void {
+    if (!this.mesh) return;
+    scene.remove(this.mesh);
+    this.mesh.geometry.dispose();
+    const material = this.mesh.material;
+    if (Array.isArray(material)) {
+      for (const mat of material) {
+        const meshMat = mat as THREE.MeshBasicMaterial;
+        meshMat.map?.dispose();
+        meshMat.dispose();
+      }
+    } else {
+      const meshMat = material as THREE.MeshBasicMaterial;
+      meshMat.map?.dispose();
+      meshMat.dispose();
+    }
+    this.mesh = null;
+  }
+}

package/src/components/avatar/VrmViewer.tsx

@@ -0,0 +1,421 @@
+/**
+ * VRM avatar canvas component.
+ *
+ * Renders a VRM model with idle animation and mouth-sync driven by
+ * the `mouthOpen` prop. Sized to fill its parent container.
+ */
+
+import { resolveAppAssetUrl } from "@elizaos/app-core/utils";
+import { useEffect, useEffectEvent, useRef } from "react";
+import {
+  type CameraProfile,
+  type InteractionMode,
+  VrmEngine,
+  type VrmEngineDebugInfo,
+  type VrmEngineState,
+} from "./VrmEngine";
+
+const DEFAULT_VRM_PATH = resolveAppAssetUrl("vrms/milady-1.vrm.gz");
+
+export type VrmViewerProps = {
+  /** When false the loaded scene stays resident but the render loop is paused */
+  active?: boolean;
+  /** Path to the VRM file to load (default: bundled Miwaifus #1) */
+  vrmPath?: string;
+  mouthOpen: number;
+  /** When true the engine generates mouth animation internally */
+  isSpeaking?: boolean;
+  /** Enable drag-rotate + wheel/pinch zoom camera controls */
+  interactive?: boolean;
+  /** Camera profile preset (chat default, companion for hero-stage framing) */
+  cameraProfile?: CameraProfile;
+  /** Interaction behavior for camera controls */
+  interactiveMode?: InteractionMode;
+  /** Optional Gaussian splat world behind the avatar */
+  worldUrl?: string;
+  /** Enable springy drag/touch camera offset instead of orbit controls */
+  pointerParallax?: boolean;
+  onEngineState?: (state: VrmEngineState) => void;
+  onEngineReady?: (engine: VrmEngine) => void;
+  onRevealStart?: () => void;
+};
+
+type VrmEngineDebugRegistryEntry = {
+  id: string;
+  role: "world-stage" | "chat-avatar";
+  vrmPath: string;
+  worldUrl: string | null;
+  engine: VrmEngine;
+  getDebugInfo: () => VrmEngineDebugInfo;
+};
+
+declare global {
+  interface Window {
+    __MILADY_VRM_ENGINES__?: VrmEngineDebugRegistryEntry[];
+  }
+}
+
+export function VrmViewer(props: VrmViewerProps) {
+  const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const engineRef = useRef<VrmEngine | null>(null);
+  const mouthOpenRef = useRef<number>(props.mouthOpen);
+  const activeRef = useRef<boolean>(props.active ?? true);
+  const isSpeakingRef = useRef<boolean>(props.isSpeaking ?? false);
+  const interactiveRef = useRef<boolean>(props.interactive ?? false);
+  const cameraProfileRef = useRef<CameraProfile>(props.cameraProfile ?? "chat");
+  const interactionModeRef = useRef<InteractionMode>(
+    props.interactiveMode ?? "free",
+  );
+  const pointerParallaxRef = useRef<boolean>(props.pointerParallax ?? false);
+  const worldUrlRef = useRef<string>(props.worldUrl ?? "");
+  const prefersWorldRendererRef = useRef<boolean>(Boolean(props.worldUrl));
+  const lastStateEmitMsRef = useRef<number>(0);
+  const mountedRef = useRef(true);
+  const currentVrmPathRef = useRef<string>("");
+  const currentWorldPathRef = useRef<string>("");
+  const worldLoadPromiseRef = useRef<Promise<void> | null>(null);
+  const pointerStateRef = useRef<{
+    active: boolean;
+    id: number | null;
+    startX: number;
+    startY: number;
+  }>({
+    active: false,
+    id: null,
+    startX: 0,
+    startY: 0,
+  });
+  const onEngineReadyRef = useRef(props.onEngineReady);
+  const onEngineStateRef = useRef(props.onEngineState);
+  const onRevealStartRef = useRef(props.onRevealStart);
+  const revealStartedRef = useRef(false);
+  const debugRegistryIdRef = useRef(
+    `vrm-viewer-${Math.random().toString(36).slice(2, 10)}`,
+  );
+
+  mouthOpenRef.current = props.mouthOpen;
+  activeRef.current = props.active ?? true;
+  isSpeakingRef.current = props.isSpeaking ?? false;
+  interactiveRef.current = props.interactive ?? false;
+  cameraProfileRef.current = props.cameraProfile ?? "chat";
+  interactionModeRef.current = props.interactiveMode ?? "free";
+  pointerParallaxRef.current = props.pointerParallax ?? false;
+  worldUrlRef.current = props.worldUrl ?? "";
+  prefersWorldRendererRef.current = Boolean(props.worldUrl);
+  onEngineReadyRef.current = props.onEngineReady;
+  onEngineStateRef.current = props.onEngineState;
+  onRevealStartRef.current = props.onRevealStart;
+
+  const syncDebugRegistry = useEffectEvent(() => {
+    if (!import.meta.env.DEV) return;
+    if (typeof window === "undefined") return;
+    const engine = engineRef.current;
+    const registry = window.__MILADY_VRM_ENGINES__ ?? [];
+    const id = debugRegistryIdRef.current;
+    const nextEntry: VrmEngineDebugRegistryEntry | null = engine
+      ? {
+          id,
+          role: props.worldUrl ? "world-stage" : "chat-avatar",
+          vrmPath: props.vrmPath ?? DEFAULT_VRM_PATH,
+          worldUrl: props.worldUrl ?? null,
+          engine,
+          getDebugInfo: () => engine.getDebugInfo(),
+        }
+      : null;
+
+    window.__MILADY_VRM_ENGINES__ = nextEntry
+      ? [...registry.filter((entry) => entry.id !== id), nextEntry]
+      : registry.filter((entry) => entry.id !== id);
+  });
+
+  // Setup engine once
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    if (!canvas) return;
+
+    mountedRef.current = true;
+
+    let engine = engineRef.current;
+    if (!engine || !engine.isInitialized()) {
+      engine = new VrmEngine();
+      engineRef.current = engine;
+    }
+
+    engine.setup(
+      canvas,
+      () => {
+        // Frame loop: guard all state-setting calls against unmount.
+        if (!mountedRef.current) return;
+        engine.setMouthOpen(mouthOpenRef.current);
+        engine.setSpeaking(isSpeakingRef.current);
+        const now = performance.now();
+        if (now - lastStateEmitMsRef.current >= 250) {
+          lastStateEmitMsRef.current = now;
+          const state = engine.getState();
+          if (state.revealStarted && !revealStartedRef.current) {
+            revealStartedRef.current = true;
+            onRevealStartRef.current?.();
+          }
+          onEngineStateRef.current?.(state);
+        }
+      },
+      {
+        rendererPreference: prefersWorldRendererRef.current ? "webgl" : "auto",
+        sparkOptimized: prefersWorldRendererRef.current,
+      },
+    );
+    engine.setPaused(!activeRef.current);
+
+    // One-time initial camera/control setup (subsequent changes handled by effects).
+    engine.setCameraProfile(cameraProfileRef.current);
+    engine.setInteractionMode(interactionModeRef.current);
+    engine.setInteractionEnabled(interactiveRef.current);
+    engine.setPointerParallaxEnabled(pointerParallaxRef.current);
+
+    const resize = () => {
+      const el = canvasRef.current;
+      if (!el) return;
+      const rect = el.getBoundingClientRect();
+      engine.resize(rect.width, rect.height);
+    };
+    const resizeObserver =
+      typeof ResizeObserver !== "undefined"
+        ? new ResizeObserver(() => resize())
+        : null;
+    resizeObserver?.observe(canvas);
+    window.addEventListener("resize", resize);
+    void engine.whenReady().then(
+      () => {
+        if (!mountedRef.current) return;
+        resize();
+        syncDebugRegistry();
+        onEngineReadyRef.current?.(engine);
+      },
+      (error) => {
+        console.warn("Failed to initialize VRM renderer:", error);
+      },
+    );
+
+    return () => {
+      mountedRef.current = false;
+      window.removeEventListener("resize", resize);
+      resizeObserver?.disconnect();
+
+      engine.dispose();
+      if (engineRef.current === engine) {
+        engineRef.current = null;
+      }
+      syncDebugRegistry();
+    };
+  }, []);
+
+  useEffect(() => {
+    syncDebugRegistry();
+  });
+
+  useEffect(() => {
+    const engine = engineRef.current;
+    if (!engine) return;
+    engine.setPaused(!(props.active ?? true));
+  }, [props.active]);
+
+  useEffect(() => {
+    const engine = engineRef.current;
+    if (!engine) return;
+    engine.setInteractionEnabled(props.interactive ?? false);
+  }, [props.interactive]);
+
+  useEffect(() => {
+    const engine = engineRef.current;
+    if (!engine) return;
+    engine.setCameraProfile(props.cameraProfile ?? "chat");
+  }, [props.cameraProfile]);
+
+  useEffect(() => {
+    const engine = engineRef.current;
+    if (!engine) return;
+    engine.setInteractionMode(props.interactiveMode ?? "free");
+  }, [props.interactiveMode]);
+
+  useEffect(() => {
+    const engine = engineRef.current;
+    if (!engine) return;
+    engine.setPointerParallaxEnabled(props.pointerParallax ?? false);
+    if (!(props.pointerParallax ?? false)) {
+      engine.resetPointerParallax();
+    }
+  }, [props.pointerParallax]);
+
+  // Load VRM when path changes
+  useEffect(() => {
+    const engine = engineRef.current;
+    if (!engine) return;
+
+    const vrmUrl = props.vrmPath ?? DEFAULT_VRM_PATH;
+    if (vrmUrl === currentVrmPathRef.current) return;
+    currentVrmPathRef.current = vrmUrl;
+    revealStartedRef.current = false;
+
+    const abortController = new AbortController();
+
+    void (async () => {
+      try {
+        await engine.whenReady();
+        if (!mountedRef.current || abortController.signal.aborted) return;
+        const worldUrl = worldUrlRef.current;
+        if (worldUrl) {
+          if (worldUrl !== currentWorldPathRef.current) {
+            currentWorldPathRef.current = worldUrl;
+            const worldLoadPromise = (async () => {
+              await engine.setWorldUrl(worldUrl);
+            })();
+            worldLoadPromiseRef.current = worldLoadPromise;
+            try {
+              await worldLoadPromise;
+            } finally {
+              if (worldLoadPromiseRef.current === worldLoadPromise) {
+                worldLoadPromiseRef.current = null;
+              }
+            }
+          } else if (worldLoadPromiseRef.current) {
+            await worldLoadPromiseRef.current;
+          }
+        }
+        if (!mountedRef.current || abortController.signal.aborted) return;
+        await engine.loadVrmFromUrl(
+          vrmUrl,
+          vrmUrl.split("/").pop() ?? "avatar.vrm",
+        );
+        if (!mountedRef.current || abortController.signal.aborted) return;
+        const state = engine.getState();
+        if (state.revealStarted && !revealStartedRef.current) {
+          revealStartedRef.current = true;
+          onRevealStartRef.current?.();
+        }
+        onEngineStateRef.current?.(state);
+      } catch (err) {
+        if (err instanceof Error && err.name === "AbortError") return;
+        if (currentVrmPathRef.current === vrmUrl) {
+          currentVrmPathRef.current = "";
+        }
+        console.warn("Failed to load VRM:", err);
+      }
+    })();
+
+    return () => {
+      abortController.abort();
+      if (currentVrmPathRef.current === vrmUrl) {
+        currentVrmPathRef.current = "";
+      }
+    };
+  }, [props.vrmPath]);
+
+  useEffect(() => {
+    const engine = engineRef.current;
+    if (!engine) return;
+
+    const worldUrl = props.worldUrl ?? "";
+    if (worldUrl === currentWorldPathRef.current) return;
+    currentWorldPathRef.current = worldUrl;
+    const abortController = new AbortController();
+
+    let worldLoadPromise: Promise<void> | null = null;
+    worldLoadPromise = (async () => {
+      try {
+        await engine.whenReady();
+        if (!mountedRef.current || abortController.signal.aborted) return;
+        await engine.setWorldUrl(worldUrl || null);
+      } catch (err) {
+        console.warn("Failed to load splat world:", err);
+      } finally {
+        if (worldLoadPromiseRef.current === worldLoadPromise) {
+          worldLoadPromiseRef.current = null;
+        }
+      }
+    })();
+    worldLoadPromiseRef.current = worldLoadPromise;
+
+    return () => {
+      abortController.abort();
+      if (currentWorldPathRef.current === worldUrl) {
+        currentWorldPathRef.current = "";
+      }
+      if (worldLoadPromiseRef.current === worldLoadPromise) {
+        worldLoadPromiseRef.current = null;
+      }
+    };
+  }, [props.worldUrl]);
+
+  const updateParallaxFromPointer = (
+    clientX: number,
+    clientY: number,
+    release = false,
+  ) => {
+    const engine = engineRef.current;
+    const canvas = canvasRef.current;
+    const pointerState = pointerStateRef.current;
+    if (!engine || !canvas || !pointerParallaxRef.current) return;
+    if (release) {
+      engine.resetPointerParallax();
+      return;
+    }
+    const rect = canvas.getBoundingClientRect();
+    const deltaX = clientX - pointerState.startX;
+    const deltaY = clientY - pointerState.startY;
+    const normalizedX = rect.width > 0 ? deltaX / rect.width : 0;
+    const normalizedY = rect.height > 0 ? deltaY / rect.height : 0;
+    engine.setPointerParallaxTarget(normalizedX * 2.2, -normalizedY * 2.2);
+  };
+
+  return (
+    <canvas
+      ref={canvasRef}
+      onPointerDown={(event) => {
+        if (!pointerParallaxRef.current) return;
+        pointerStateRef.current = {
+          active: true,
+          id: event.pointerId,
+          startX: event.clientX,
+          startY: event.clientY,
+        };
+        event.currentTarget.setPointerCapture(event.pointerId);
+      }}
+      onPointerMove={(event) => {
+        const pointerState = pointerStateRef.current;
+        if (
+          !pointerParallaxRef.current ||
+          !pointerState.active ||
+          pointerState.id !== event.pointerId
+        ) {
+          return;
+        }
+        updateParallaxFromPointer(event.clientX, event.clientY);
+      }}
+      onPointerUp={(event) => {
+        const pointerState = pointerStateRef.current;
+        if (pointerState.id !== event.pointerId) return;
+        pointerStateRef.current.active = false;
+        event.currentTarget.releasePointerCapture(event.pointerId);
+        updateParallaxFromPointer(event.clientX, event.clientY, true);
+      }}
+      onPointerCancel={(event) => {
+        const pointerState = pointerStateRef.current;
+        if (pointerState.id !== event.pointerId) return;
+        pointerStateRef.current.active = false;
+        updateParallaxFromPointer(event.clientX, event.clientY, true);
+      }}
+      style={{
+        position: "absolute",
+        inset: 0,
+        display: "block",
+        width: "100vw",
+        height: "100vh",
+        minWidth: "100vw",
+        minHeight: "100vh",
+        background: "transparent",
+        cursor: props.pointerParallax || props.interactive ? "grab" : "default",
+        touchAction: props.pointerParallax ? "none" : "auto",
+      }}
+    />
+  );
+}

package/src/components/avatar/__tests__/VrmCameraManager.test.ts

@@ -0,0 +1,168 @@
+import type { VRM } from "@pixiv/three-vrm";
+import * as THREE from "three";
+import { describe, expect, it } from "vitest";
+import { VrmCameraManager } from "../VrmCameraManager";
+
+function createVrmWithBones(options: {
+  neckY?: number;
+  headY?: number;
+  chestY?: number;
+}): VRM {
+  const scene = new THREE.Group();
+  const body = new THREE.Mesh(
+    new THREE.BoxGeometry(0.6, 2, 0.4),
+    new THREE.MeshBasicMaterial(),
+  );
+  body.position.y = 1;
+  scene.add(body);
+
+  const bones = new Map<string, THREE.Object3D>();
+
+  if (typeof options.neckY === "number") {
+    const neck = new THREE.Object3D();
+    neck.position.y = options.neckY;
+    scene.add(neck);
+    bones.set("neck", neck);
+  }
+  if (typeof options.headY === "number") {
+    const head = new THREE.Object3D();
+    head.position.y = options.headY;
+    scene.add(head);
+    bones.set("head", head);
+  }
+  if (typeof options.chestY === "number") {
+    const chest = new THREE.Object3D();
+    chest.position.y = options.chestY;
+    scene.add(chest);
+    bones.set("chest", chest);
+  }
+
+  return {
+    scene,
+    humanoid: {
+      getNormalizedBoneNode: (name: string) => bones.get(name) ?? null,
+    },
+  } as VRM;
+}
+
+describe("VrmCameraManager", () => {
+  it("frames companion_close avatars at their own neck height", () => {
+    const manager = new VrmCameraManager();
+    const camera = new THREE.PerspectiveCamera(22, 1, 0.1, 20);
+    const lookAtShort = new THREE.Vector3();
+    const lookAtTall = new THREE.Vector3();
+    const baseCameraPosition = new THREE.Vector3();
+
+    const shortVrm = createVrmWithBones({ neckY: 1.15 });
+    const tallVrm = createVrmWithBones({ neckY: 1.55 });
+
+    manager.centerAndFrame(
+      shortVrm,
+      camera,
+      null,
+      "companion_close",
+      lookAtShort,
+      baseCameraPosition,
+      () => {},
+    );
+    const shortCameraY = camera.position.y;
+
+    manager.centerAndFrame(
+      tallVrm,
+      camera,
+      null,
+      "companion_close",
+      lookAtTall,
+      baseCameraPosition,
+      () => {},
+    );
+
+    expect(lookAtTall.y).toBeGreaterThan(lookAtShort.y);
+    expect(shortCameraY).toBeGreaterThan(lookAtShort.y);
+    expect(camera.position.y).toBeGreaterThan(lookAtTall.y);
+  });
+
+  it("falls back to head/chest midpoint when neck bone is missing", () => {
+    const manager = new VrmCameraManager();
+    const camera = new THREE.PerspectiveCamera(28, 1, 0.1, 20);
+    const lookAtTarget = new THREE.Vector3();
+
+    const vrm = createVrmWithBones({ headY: 1.8, chestY: 1.2 });
+    manager.centerAndFrame(
+      vrm,
+      camera,
+      null,
+      "companion",
+      lookAtTarget,
+      new THREE.Vector3(),
+      () => {},
+    );
+
+    expect(lookAtTarget.y).toBeGreaterThan(1.1);
+    expect(lookAtTarget.y).toBeLessThan(1.6);
+    expect(camera.position.y).toBeGreaterThan(lookAtTarget.y);
+  });
+
+  it("normalizes the avatar so its feet rest at stage origin", () => {
+    const manager = new VrmCameraManager();
+    const camera = new THREE.PerspectiveCamera(28, 1, 0.1, 20);
+    const lookAtTarget = new THREE.Vector3();
+
+    const vrm = createVrmWithBones({ neckY: 1.4 });
+    manager.centerAndFrame(
+      vrm,
+      camera,
+      null,
+      "companion",
+      lookAtTarget,
+      new THREE.Vector3(),
+      () => {},
+    );
+
+    const bounds = new THREE.Box3().setFromObject(vrm.scene);
+    expect(bounds.min.y).toBeCloseTo(0, 5);
+    expect(Math.abs(bounds.getCenter(new THREE.Vector3()).x)).toBeLessThan(
+      1e-6,
+    );
+  });
+
+  it("animates companion camera as a shallow front orbit around the focus point", () => {
+    const manager = new VrmCameraManager();
+    const camera = new THREE.PerspectiveCamera(28, 1, 0.1, 20);
+    const lookAtTarget = new THREE.Vector3(0, 1.4, 0);
+    const baseCameraPosition = new THREE.Vector3(0, 1.48, 5.1);
+    const animation = {
+      enabled: true,
+      swayAmplitude: 0.04,
+      bobAmplitude: 0.022,
+      rotationAmplitude: 0.012,
+      speed: 0.42,
+    };
+
+    manager.applyCameraMotion(
+      camera,
+      baseCameraPosition,
+      lookAtTarget,
+      animation,
+      2,
+    );
+    const positionA = camera.position.clone();
+    const offsetA = positionA.clone().sub(lookAtTarget);
+
+    manager.applyCameraMotion(
+      camera,
+      baseCameraPosition,
+      lookAtTarget,
+      animation,
+      18,
+    );
+    const positionB = camera.position.clone();
+    const offsetB = positionB.clone().sub(lookAtTarget);
+
+    expect(positionA.distanceTo(positionB)).toBeGreaterThan(0.05);
+    expect(offsetA.z).toBeGreaterThan(4.7);
+    expect(offsetB.z).toBeGreaterThan(4.7);
+    expect(Math.abs(offsetA.x)).toBeLessThan(1.2);
+    expect(Math.abs(offsetB.x)).toBeLessThan(1.2);
+  });
+});