@elizaos/app-core 2.0.0-alpha.73 → 2.0.0-alpha.75

package/src/components/avatar/VrmFootShadow.ts

@@ -1,96 +0,0 @@
-import type { VRM } from "@pixiv/three-vrm";
-import * as THREE from "three";
-
-/**
- * Manages a soft radial shadow disc placed beneath the VRM avatar's feet.
- */
-export class VrmFootShadow {
-  private mesh: THREE.Mesh | null = null;
-  private readonly tempLeft = new THREE.Vector3();
-  private readonly tempRight = new THREE.Vector3();
-  private readonly tempCenter = new THREE.Vector3();
-
-  /** Create (or recreate) the shadow disc and add it to `scene`. */
-  create(scene: THREE.Scene): void {
-    this.dispose(scene);
-
-    const shadowCanvas = document.createElement("canvas");
-    shadowCanvas.width = 128;
-    shadowCanvas.height = 128;
-
-    const context = shadowCanvas.getContext("2d");
-    if (!context) return;
-
-    const gradient = context.createRadialGradient(64, 64, 0, 64, 64, 64);
-    gradient.addColorStop(0, "rgba(0, 0, 0, 0.4)");
-    gradient.addColorStop(0.5, "rgba(0, 0, 0, 0.2)");
-    gradient.addColorStop(1, "rgba(0, 0, 0, 0)");
-
-    context.fillStyle = gradient;
-    context.fillRect(0, 0, 128, 128);
-
-    const shadowTexture = new THREE.CanvasTexture(shadowCanvas);
-    const shadowGeometry = new THREE.PlaneGeometry(2.2, 2.2);
-    const shadowMaterial = new THREE.MeshBasicMaterial({
-      map: shadowTexture,
-      transparent: true,
-      alphaTest: 0.001,
-      depthWrite: false,
-    });
-
-    const shadow = new THREE.Mesh(shadowGeometry, shadowMaterial);
-    shadow.rotation.x = -Math.PI / 2;
-    shadow.position.set(0, 0, 0);
-    scene.add(shadow);
-    this.mesh = shadow;
-  }
-
-  /** Update shadow position to follow the VRM on the ground plane. */
-  update(vrm: VRM): void {
-    if (!this.mesh) return;
-    vrm.scene.updateMatrixWorld(true);
-    const leftFoot = vrm.humanoid?.getNormalizedBoneNode("leftFoot");
-    const rightFoot = vrm.humanoid?.getNormalizedBoneNode("rightFoot");
-
-    if (leftFoot && rightFoot) {
-      leftFoot.getWorldPosition(this.tempLeft);
-      rightFoot.getWorldPosition(this.tempRight);
-      this.tempCenter
-        .copy(this.tempLeft)
-        .add(this.tempRight)
-        .multiplyScalar(0.5);
-      this.mesh.position.copy(this.tempCenter);
-      return;
-    }
-
-    const bounds = new THREE.Box3().setFromObject(vrm.scene);
-    if (!bounds.isEmpty()) {
-      bounds.getCenter(this.tempCenter);
-      this.mesh.position.set(
-        this.tempCenter.x,
-        bounds.min.y,
-        this.tempCenter.z,
-      );
-    }
-  }
-
-  /** Remove and dispose of the shadow mesh, releasing GPU resources. */
-  dispose(scene: THREE.Scene): void {
-    if (!this.mesh) return;
-    scene.remove(this.mesh);
-    this.mesh.geometry.dispose();
-    const material = this.mesh.material;
-    if (Array.isArray(material)) {
-      for (const mat of material) {
-        const meshMat = mat as THREE.MeshBasicMaterial;
-        meshMat.map?.dispose();
-        meshMat.dispose();
-      }
-    } else {
-      const meshMat = material as THREE.MeshBasicMaterial;
-      meshMat.map?.dispose();
-      meshMat.dispose();
-    }
-    this.mesh = null;
-  }
-}

package/src/components/avatar/VrmViewer.tsx

@@ -1,421 +0,0 @@
-/**
- * VRM avatar canvas component.
- *
- * Renders a VRM model with idle animation and mouth-sync driven by
- * the `mouthOpen` prop. Sized to fill its parent container.
- */
-
-import { useEffect, useEffectEvent, useRef } from "react";
-import { getVrmUrl } from "../../state/vrm";
-import {
-  type CameraProfile,
-  type InteractionMode,
-  VrmEngine,
-  type VrmEngineDebugInfo,
-  type VrmEngineState,
-} from "./VrmEngine";
-
-const DEFAULT_VRM_PATH = getVrmUrl(1);
-
-export type VrmViewerProps = {
-  /** When false the loaded scene stays resident but the render loop is paused */
-  active?: boolean;
-  /** Path to the VRM file to load (default: bundled Miwaifus #1) */
-  vrmPath?: string;
-  mouthOpen: number;
-  /** When true the engine generates mouth animation internally */
-  isSpeaking?: boolean;
-  /** Enable drag-rotate + wheel/pinch zoom camera controls */
-  interactive?: boolean;
-  /** Camera profile preset (chat default, companion for hero-stage framing) */
-  cameraProfile?: CameraProfile;
-  /** Interaction behavior for camera controls */
-  interactiveMode?: InteractionMode;
-  /** Optional Gaussian splat world behind the avatar */
-  worldUrl?: string;
-  /** Enable springy drag/touch camera offset instead of orbit controls */
-  pointerParallax?: boolean;
-  onEngineState?: (state: VrmEngineState) => void;
-  onEngineReady?: (engine: VrmEngine) => void;
-  onRevealStart?: () => void;
-};
-
-type VrmEngineDebugRegistryEntry = {
-  id: string;
-  role: "world-stage" | "chat-avatar";
-  vrmPath: string;
-  worldUrl: string | null;
-  engine: VrmEngine;
-  getDebugInfo: () => VrmEngineDebugInfo;
-};
-
-declare global {
-  interface Window {
-    __ELIZA_VRM_ENGINES__?: VrmEngineDebugRegistryEntry[];
-  }
-}
-
-export function VrmViewer(props: VrmViewerProps) {
-  const canvasRef = useRef<HTMLCanvasElement | null>(null);
-  const engineRef = useRef<VrmEngine | null>(null);
-  const mouthOpenRef = useRef<number>(props.mouthOpen);
-  const activeRef = useRef<boolean>(props.active ?? true);
-  const isSpeakingRef = useRef<boolean>(props.isSpeaking ?? false);
-  const interactiveRef = useRef<boolean>(props.interactive ?? false);
-  const cameraProfileRef = useRef<CameraProfile>(props.cameraProfile ?? "chat");
-  const interactionModeRef = useRef<InteractionMode>(
-    props.interactiveMode ?? "free",
-  );
-  const pointerParallaxRef = useRef<boolean>(props.pointerParallax ?? false);
-  const worldUrlRef = useRef<string>(props.worldUrl ?? "");
-  const prefersWorldRendererRef = useRef<boolean>(Boolean(props.worldUrl));
-  const lastStateEmitMsRef = useRef<number>(0);
-  const mountedRef = useRef(true);
-  const currentVrmPathRef = useRef<string>("");
-  const currentWorldPathRef = useRef<string>("");
-  const worldLoadPromiseRef = useRef<Promise<void> | null>(null);
-  const pointerStateRef = useRef<{
-    active: boolean;
-    id: number | null;
-    startX: number;
-    startY: number;
-  }>({
-    active: false,
-    id: null,
-    startX: 0,
-    startY: 0,
-  });
-  const onEngineReadyRef = useRef(props.onEngineReady);
-  const onEngineStateRef = useRef(props.onEngineState);
-  const onRevealStartRef = useRef(props.onRevealStart);
-  const revealStartedRef = useRef(false);
-  const debugRegistryIdRef = useRef(
-    `vrm-viewer-${Math.random().toString(36).slice(2, 10)}`,
-  );
-
-  mouthOpenRef.current = props.mouthOpen;
-  activeRef.current = props.active ?? true;
-  isSpeakingRef.current = props.isSpeaking ?? false;
-  interactiveRef.current = props.interactive ?? false;
-  cameraProfileRef.current = props.cameraProfile ?? "chat";
-  interactionModeRef.current = props.interactiveMode ?? "free";
-  pointerParallaxRef.current = props.pointerParallax ?? false;
-  worldUrlRef.current = props.worldUrl ?? "";
-  prefersWorldRendererRef.current = Boolean(props.worldUrl);
-  onEngineReadyRef.current = props.onEngineReady;
-  onEngineStateRef.current = props.onEngineState;
-  onRevealStartRef.current = props.onRevealStart;
-
-  const syncDebugRegistry = useEffectEvent(() => {
-    if (!import.meta.env.DEV) return;
-    if (typeof window === "undefined") return;
-    const engine = engineRef.current;
-    const registry = window.__ELIZA_VRM_ENGINES__ ?? [];
-    const id = debugRegistryIdRef.current;
-    const nextEntry: VrmEngineDebugRegistryEntry | null = engine
-      ? {
-          id,
-          role: props.worldUrl ? "world-stage" : "chat-avatar",
-          vrmPath: props.vrmPath ?? DEFAULT_VRM_PATH,
-          worldUrl: props.worldUrl ?? null,
-          engine,
-          getDebugInfo: () => engine.getDebugInfo(),
-        }
-      : null;
-
-    window.__ELIZA_VRM_ENGINES__ = nextEntry
-      ? [...registry.filter((entry) => entry.id !== id), nextEntry]
-      : registry.filter((entry) => entry.id !== id);
-  });
-
-  // Setup engine once
-  useEffect(() => {
-    const canvas = canvasRef.current;
-    if (!canvas) return;
-
-    mountedRef.current = true;
-
-    let engine = engineRef.current;
-    if (!engine || !engine.isInitialized()) {
-      engine = new VrmEngine();
-      engineRef.current = engine;
-    }
-
-    engine.setup(
-      canvas,
-      () => {
-        // Frame loop: guard all state-setting calls against unmount.
-        if (!mountedRef.current) return;
-        engine.setMouthOpen(mouthOpenRef.current);
-        engine.setSpeaking(isSpeakingRef.current);
-        const now = performance.now();
-        if (now - lastStateEmitMsRef.current >= 250) {
-          lastStateEmitMsRef.current = now;
-          const state = engine.getState();
-          if (state.revealStarted && !revealStartedRef.current) {
-            revealStartedRef.current = true;
-            onRevealStartRef.current?.();
-          }
-          onEngineStateRef.current?.(state);
-        }
-      },
-      {
-        rendererPreference: prefersWorldRendererRef.current ? "webgl" : "auto",
-        sparkOptimized: prefersWorldRendererRef.current,
-      },
-    );
-    engine.setPaused(!activeRef.current);
-
-    // One-time initial camera/control setup (subsequent changes handled by effects).
-    engine.setCameraProfile(cameraProfileRef.current);
-    engine.setInteractionMode(interactionModeRef.current);
-    engine.setInteractionEnabled(interactiveRef.current);
-    engine.setPointerParallaxEnabled(pointerParallaxRef.current);
-
-    const resize = () => {
-      const el = canvasRef.current;
-      if (!el) return;
-      const rect = el.getBoundingClientRect();
-      engine.resize(rect.width, rect.height);
-    };
-    const resizeObserver =
-      typeof ResizeObserver !== "undefined"
-        ? new ResizeObserver(() => resize())
-        : null;
-    resizeObserver?.observe(canvas);
-    window.addEventListener("resize", resize);
-    void engine.whenReady().then(
-      () => {
-        if (!mountedRef.current) return;
-        resize();
-        syncDebugRegistry();
-        onEngineReadyRef.current?.(engine);
-      },
-      (error) => {
-        console.warn("Failed to initialize VRM renderer:", error);
-      },
-    );
-
-    return () => {
-      mountedRef.current = false;
-      window.removeEventListener("resize", resize);
-      resizeObserver?.disconnect();
-
-      engine.dispose();
-      if (engineRef.current === engine) {
-        engineRef.current = null;
-      }
-      syncDebugRegistry();
-    };
-  }, []);
-
-  useEffect(() => {
-    syncDebugRegistry();
-  });
-
-  useEffect(() => {
-    const engine = engineRef.current;
-    if (!engine) return;
-    engine.setPaused(!(props.active ?? true));
-  }, [props.active]);
-
-  useEffect(() => {
-    const engine = engineRef.current;
-    if (!engine) return;
-    engine.setInteractionEnabled(props.interactive ?? false);
-  }, [props.interactive]);
-
-  useEffect(() => {
-    const engine = engineRef.current;
-    if (!engine) return;
-    engine.setCameraProfile(props.cameraProfile ?? "chat");
-  }, [props.cameraProfile]);
-
-  useEffect(() => {
-    const engine = engineRef.current;
-    if (!engine) return;
-    engine.setInteractionMode(props.interactiveMode ?? "free");
-  }, [props.interactiveMode]);
-
-  useEffect(() => {
-    const engine = engineRef.current;
-    if (!engine) return;
-    engine.setPointerParallaxEnabled(props.pointerParallax ?? false);
-    if (!(props.pointerParallax ?? false)) {
-      engine.resetPointerParallax();
-    }
-  }, [props.pointerParallax]);
-
-  // Load VRM when path changes
-  useEffect(() => {
-    const engine = engineRef.current;
-    if (!engine) return;
-
-    const vrmUrl = props.vrmPath ?? DEFAULT_VRM_PATH;
-    if (vrmUrl === currentVrmPathRef.current) return;
-    currentVrmPathRef.current = vrmUrl;
-    revealStartedRef.current = false;
-
-    const abortController = new AbortController();
-
-    void (async () => {
-      try {
-        await engine.whenReady();
-        if (!mountedRef.current || abortController.signal.aborted) return;
-        const worldUrl = worldUrlRef.current;
-        if (worldUrl) {
-          if (worldUrl !== currentWorldPathRef.current) {
-            currentWorldPathRef.current = worldUrl;
-            const worldLoadPromise = (async () => {
-              await engine.setWorldUrl(worldUrl);
-            })();
-            worldLoadPromiseRef.current = worldLoadPromise;
-            try {
-              await worldLoadPromise;
-            } finally {
-              if (worldLoadPromiseRef.current === worldLoadPromise) {
-                worldLoadPromiseRef.current = null;
-              }
-            }
-          } else if (worldLoadPromiseRef.current) {
-            await worldLoadPromiseRef.current;
-          }
-        }
-        if (!mountedRef.current || abortController.signal.aborted) return;
-        await engine.loadVrmFromUrl(
-          vrmUrl,
-          vrmUrl.split("/").pop() ?? "avatar.vrm",
-        );
-        if (!mountedRef.current || abortController.signal.aborted) return;
-        const state = engine.getState();
-        if (state.revealStarted && !revealStartedRef.current) {
-          revealStartedRef.current = true;
-          onRevealStartRef.current?.();
-        }
-        onEngineStateRef.current?.(state);
-      } catch (err) {
-        if (err instanceof Error && err.name === "AbortError") return;
-        if (currentVrmPathRef.current === vrmUrl) {
-          currentVrmPathRef.current = "";
-        }
-        console.warn("Failed to load VRM:", err);
-      }
-    })();
-
-    return () => {
-      abortController.abort();
-      if (currentVrmPathRef.current === vrmUrl) {
-        currentVrmPathRef.current = "";
-      }
-    };
-  }, [props.vrmPath]);
-
-  useEffect(() => {
-    const engine = engineRef.current;
-    if (!engine) return;
-
-    const worldUrl = props.worldUrl ?? "";
-    if (worldUrl === currentWorldPathRef.current) return;
-    currentWorldPathRef.current = worldUrl;
-    const abortController = new AbortController();
-
-    let worldLoadPromise: Promise<void> | null = null;
-    worldLoadPromise = (async () => {
-      try {
-        await engine.whenReady();
-        if (!mountedRef.current || abortController.signal.aborted) return;
-        await engine.setWorldUrl(worldUrl || null);
-      } catch (err) {
-        console.warn("Failed to load splat world:", err);
-      } finally {
-        if (worldLoadPromiseRef.current === worldLoadPromise) {
-          worldLoadPromiseRef.current = null;
-        }
-      }
-    })();
-    worldLoadPromiseRef.current = worldLoadPromise;
-
-    return () => {
-      abortController.abort();
-      if (currentWorldPathRef.current === worldUrl) {
-        currentWorldPathRef.current = "";
-      }
-      if (worldLoadPromiseRef.current === worldLoadPromise) {
-        worldLoadPromiseRef.current = null;
-      }
-    };
-  }, [props.worldUrl]);
-
-  const updateParallaxFromPointer = (
-    clientX: number,
-    clientY: number,
-    release = false,
-  ) => {
-    const engine = engineRef.current;
-    const canvas = canvasRef.current;
-    const pointerState = pointerStateRef.current;
-    if (!engine || !canvas || !pointerParallaxRef.current) return;
-    if (release) {
-      engine.resetPointerParallax();
-      return;
-    }
-    const rect = canvas.getBoundingClientRect();
-    const deltaX = clientX - pointerState.startX;
-    const deltaY = clientY - pointerState.startY;
-    const normalizedX = rect.width > 0 ? deltaX / rect.width : 0;
-    const normalizedY = rect.height > 0 ? deltaY / rect.height : 0;
-    engine.setPointerParallaxTarget(normalizedX * 2.2, -normalizedY * 2.2);
-  };
-
-  return (
-    <canvas
-      ref={canvasRef}
-      onPointerDown={(event) => {
-        if (!pointerParallaxRef.current) return;
-        pointerStateRef.current = {
-          active: true,
-          id: event.pointerId,
-          startX: event.clientX,
-          startY: event.clientY,
-        };
-        event.currentTarget.setPointerCapture(event.pointerId);
-      }}
-      onPointerMove={(event) => {
-        const pointerState = pointerStateRef.current;
-        if (
-          !pointerParallaxRef.current ||
-          !pointerState.active ||
-          pointerState.id !== event.pointerId
-        ) {
-          return;
-        }
-        updateParallaxFromPointer(event.clientX, event.clientY);
-      }}
-      onPointerUp={(event) => {
-        const pointerState = pointerStateRef.current;
-        if (pointerState.id !== event.pointerId) return;
-        pointerStateRef.current.active = false;
-        event.currentTarget.releasePointerCapture(event.pointerId);
-        updateParallaxFromPointer(event.clientX, event.clientY, true);
-      }}
-      onPointerCancel={(event) => {
-        const pointerState = pointerStateRef.current;
-        if (pointerState.id !== event.pointerId) return;
-        pointerStateRef.current.active = false;
-        updateParallaxFromPointer(event.clientX, event.clientY, true);
-      }}
-      style={{
-        position: "absolute",
-        inset: 0,
-        display: "block",
-        width: "100vw",
-        height: "100vh",
-        minWidth: "100vw",
-        minHeight: "100vh",
-        background: "transparent",
-        cursor: props.pointerParallax || props.interactive ? "grab" : "default",
-        touchAction: props.pointerParallax ? "none" : "auto",
-      }}
-    />
-  );
-}

package/src/components/avatar/__tests__/VrmCameraManager.test.ts

@@ -1,168 +0,0 @@
-import type { VRM } from "@pixiv/three-vrm";
-import * as THREE from "three";
-import { describe, expect, it } from "vitest";
-import { VrmCameraManager } from "../VrmCameraManager";
-
-function createVrmWithBones(options: {
-  neckY?: number;
-  headY?: number;
-  chestY?: number;
-}): VRM {
-  const scene = new THREE.Group();
-  const body = new THREE.Mesh(
-    new THREE.BoxGeometry(0.6, 2, 0.4),
-    new THREE.MeshBasicMaterial(),
-  );
-  body.position.y = 1;
-  scene.add(body);
-
-  const bones = new Map<string, THREE.Object3D>();
-
-  if (typeof options.neckY === "number") {
-    const neck = new THREE.Object3D();
-    neck.position.y = options.neckY;
-    scene.add(neck);
-    bones.set("neck", neck);
-  }
-  if (typeof options.headY === "number") {
-    const head = new THREE.Object3D();
-    head.position.y = options.headY;
-    scene.add(head);
-    bones.set("head", head);
-  }
-  if (typeof options.chestY === "number") {
-    const chest = new THREE.Object3D();
-    chest.position.y = options.chestY;
-    scene.add(chest);
-    bones.set("chest", chest);
-  }
-
-  return {
-    scene,
-    humanoid: {
-      getNormalizedBoneNode: (name: string) => bones.get(name) ?? null,
-    },
-  } as VRM;
-}
-
-describe("VrmCameraManager", () => {
-  it("frames companion_close avatars at their own neck height", () => {
-    const manager = new VrmCameraManager();
-    const camera = new THREE.PerspectiveCamera(22, 1, 0.1, 20);
-    const lookAtShort = new THREE.Vector3();
-    const lookAtTall = new THREE.Vector3();
-    const baseCameraPosition = new THREE.Vector3();
-
-    const shortVrm = createVrmWithBones({ neckY: 1.15 });
-    const tallVrm = createVrmWithBones({ neckY: 1.55 });
-
-    manager.centerAndFrame(
-      shortVrm,
-      camera,
-      null,
-      "companion_close",
-      lookAtShort,
-      baseCameraPosition,
-      () => {},
-    );
-    const shortCameraY = camera.position.y;
-
-    manager.centerAndFrame(
-      tallVrm,
-      camera,
-      null,
-      "companion_close",
-      lookAtTall,
-      baseCameraPosition,
-      () => {},
-    );
-
-    expect(lookAtTall.y).toBeGreaterThan(lookAtShort.y);
-    expect(shortCameraY).toBeGreaterThan(lookAtShort.y);
-    expect(camera.position.y).toBeGreaterThan(lookAtTall.y);
-  });
-
-  it("falls back to head/chest midpoint when neck bone is missing", () => {
-    const manager = new VrmCameraManager();
-    const camera = new THREE.PerspectiveCamera(28, 1, 0.1, 20);
-    const lookAtTarget = new THREE.Vector3();
-
-    const vrm = createVrmWithBones({ headY: 1.8, chestY: 1.2 });
-    manager.centerAndFrame(
-      vrm,
-      camera,
-      null,
-      "companion",
-      lookAtTarget,
-      new THREE.Vector3(),
-      () => {},
-    );
-
-    expect(lookAtTarget.y).toBeGreaterThan(1.1);
-    expect(lookAtTarget.y).toBeLessThan(1.6);
-    expect(camera.position.y).toBeGreaterThan(lookAtTarget.y);
-  });
-
-  it("normalizes the avatar so its feet rest at stage origin", () => {
-    const manager = new VrmCameraManager();
-    const camera = new THREE.PerspectiveCamera(28, 1, 0.1, 20);
-    const lookAtTarget = new THREE.Vector3();
-
-    const vrm = createVrmWithBones({ neckY: 1.4 });
-    manager.centerAndFrame(
-      vrm,
-      camera,
-      null,
-      "companion",
-      lookAtTarget,
-      new THREE.Vector3(),
-      () => {},
-    );
-
-    const bounds = new THREE.Box3().setFromObject(vrm.scene);
-    expect(bounds.min.y).toBeCloseTo(0, 5);
-    expect(Math.abs(bounds.getCenter(new THREE.Vector3()).x)).toBeLessThan(
-      1e-6,
-    );
-  });
-
-  it("animates companion camera as a shallow front orbit around the focus point", () => {
-    const manager = new VrmCameraManager();
-    const camera = new THREE.PerspectiveCamera(28, 1, 0.1, 20);
-    const lookAtTarget = new THREE.Vector3(0, 1.4, 0);
-    const baseCameraPosition = new THREE.Vector3(0, 1.48, 5.1);
-    const animation = {
-      enabled: true,
-      swayAmplitude: 0.04,
-      bobAmplitude: 0.022,
-      rotationAmplitude: 0.012,
-      speed: 0.42,
-    };
-
-    manager.applyCameraMotion(
-      camera,
-      baseCameraPosition,
-      lookAtTarget,
-      animation,
-      2,
-    );
-    const positionA = camera.position.clone();
-    const offsetA = positionA.clone().sub(lookAtTarget);
-
-    manager.applyCameraMotion(
-      camera,
-      baseCameraPosition,
-      lookAtTarget,
-      animation,
-      18,
-    );
-    const positionB = camera.position.clone();
-    const offsetB = positionB.clone().sub(lookAtTarget);
-
-    expect(positionA.distanceTo(positionB)).toBeGreaterThan(0.05);
-    expect(offsetA.z).toBeGreaterThan(4.7);
-    expect(offsetB.z).toBeGreaterThan(4.7);
-    expect(Math.abs(offsetA.x)).toBeLessThan(1.2);
-    expect(Math.abs(offsetB.x)).toBeLessThan(1.2);
-  });
-});