@elizaos/app-core 2.0.0-alpha.37

package/src/components/avatar/VrmAnimationLoader.test.ts

@@ -0,0 +1,164 @@
+import type { VRM } from "@pixiv/three-vrm";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const hoisted = vi.hoisted(() => ({
+  fetchMock: vi.fn(),
+  gltfParseAsyncMock: vi.fn(),
+  fbxParseMock: vi.fn(),
+  retargetGltfMock: vi.fn(),
+  retargetFbxMock: vi.fn(),
+}));
+
+vi.mock("three/examples/jsm/loaders/GLTFLoader.js", () => ({
+  GLTFLoader: class MockGLTFLoader {
+    parseAsync = hoisted.gltfParseAsyncMock;
+  },
+}));
+
+vi.mock("three/examples/jsm/loaders/FBXLoader.js", () => ({
+  FBXLoader: class MockFBXLoader {
+    parse = hoisted.fbxParseMock;
+  },
+}));
+
+vi.mock("./retargetMixamoGltfToVrm", () => ({
+  retargetMixamoGltfToVrm: hoisted.retargetGltfMock,
+}));
+
+vi.mock("./retargetMixamoFbxToVrm", () => ({
+  retargetMixamoFbxToVrm: hoisted.retargetFbxMock,
+}));
+
+import { loadEmoteClip, loadIdleClip } from "./VrmAnimationLoader";
+
+describe("VrmAnimationLoader", () => {
+  const originalFetch = globalThis.fetch;
+  const originalDecompressionStream = globalThis.DecompressionStream;
+
+  beforeEach(() => {
+    vi.restoreAllMocks();
+    hoisted.fetchMock.mockReset();
+    hoisted.gltfParseAsyncMock.mockReset();
+    hoisted.fbxParseMock.mockReset();
+    hoisted.retargetGltfMock.mockReset();
+    hoisted.retargetFbxMock.mockReset();
+    Object.assign(globalThis, {
+      fetch: hoisted.fetchMock,
+    });
+  });
+
+  afterEach(() => {
+    Object.assign(globalThis, {
+      fetch: originalFetch,
+      DecompressionStream: originalDecompressionStream,
+    });
+  });
+
+  it("loads a gzipped idle GLB animation", async () => {
+    const compressed = new Uint8Array([0x1f, 0x8b, 0x08, 0x00]).buffer;
+    const decompressed = new ArrayBuffer(32);
+    hoisted.fetchMock.mockResolvedValueOnce({
+      ok: true,
+      arrayBuffer: vi.fn().mockResolvedValue(compressed),
+    });
+
+    class MockDecompressionStream {
+      readable = new ReadableStream<Uint8Array>();
+      writable = new WritableStream<Uint8Array>();
+    }
+
+    Object.assign(globalThis, {
+      DecompressionStream:
+        MockDecompressionStream as unknown as typeof DecompressionStream,
+    });
+
+    const pipeThroughSpy = vi.spyOn(Blob.prototype, "stream").mockReturnValue({
+      pipeThrough: vi.fn(() => new ReadableStream<Uint8Array>()),
+    } as unknown as ReturnType<Blob["stream"]>);
+    const responseArrayBufferSpy = vi
+      .spyOn(Response.prototype, "arrayBuffer")
+      .mockResolvedValueOnce(decompressed);
+
+    const vrm = {
+      scene: { updateMatrixWorld: vi.fn() },
+    } as unknown as VRM;
+    const gltf = {
+      scene: { updateMatrixWorld: vi.fn() },
+      animations: [],
+    };
+    const clip = { name: "idle" };
+    hoisted.gltfParseAsyncMock.mockResolvedValueOnce(gltf);
+    hoisted.retargetGltfMock.mockReturnValueOnce(clip);
+
+    const result = await loadIdleClip(vrm, "/animations/idle.glb.gz", {
+      isAborted: () => false,
+      isCurrentVrm: () => true,
+    });
+
+    expect(result).toBe(clip);
+    expect(hoisted.fetchMock).toHaveBeenCalledWith("/animations/idle.glb.gz");
+    expect(hoisted.gltfParseAsyncMock).toHaveBeenCalledWith(
+      decompressed,
+      "/animations/",
+    );
+    expect(hoisted.retargetGltfMock).toHaveBeenCalledWith(gltf, vrm);
+
+    responseArrayBufferSpy.mockRestore();
+    pipeThroughSpy.mockRestore();
+  });
+
+  it("loads a gzipped Mixamo FBX emote", async () => {
+    const compressed = new Uint8Array([0x1f, 0x8b, 0x08, 0x00]).buffer;
+    const decompressed = new ArrayBuffer(64);
+    hoisted.fetchMock.mockResolvedValueOnce({
+      ok: true,
+      arrayBuffer: vi.fn().mockResolvedValue(compressed),
+    });
+
+    class MockDecompressionStream {
+      readable = new ReadableStream<Uint8Array>();
+      writable = new WritableStream<Uint8Array>();
+    }
+
+    Object.assign(globalThis, {
+      DecompressionStream:
+        MockDecompressionStream as unknown as typeof DecompressionStream,
+    });
+
+    const pipeThroughSpy = vi.spyOn(Blob.prototype, "stream").mockReturnValue({
+      pipeThrough: vi.fn(() => new ReadableStream<Uint8Array>()),
+    } as unknown as ReturnType<Blob["stream"]>);
+    const responseArrayBufferSpy = vi
+      .spyOn(Response.prototype, "arrayBuffer")
+      .mockResolvedValueOnce(decompressed);
+
+    const vrm = {
+      scene: { updateMatrixWorld: vi.fn() },
+    } as unknown as VRM;
+    const sourceClip = { name: "happy" };
+    const fbx = {
+      animations: [sourceClip],
+    };
+    const clip = { name: "happy" };
+    hoisted.fbxParseMock.mockReturnValueOnce(fbx);
+    hoisted.retargetFbxMock.mockReturnValueOnce(clip);
+
+    const result = await loadEmoteClip("/animations/mixamo/Happy.fbx.gz", vrm, {
+      isAborted: () => false,
+      isCurrentVrm: () => true,
+    });
+
+    expect(result).toBe(clip);
+    expect(hoisted.fetchMock).toHaveBeenCalledWith(
+      "/animations/mixamo/Happy.fbx.gz",
+    );
+    expect(hoisted.fbxParseMock).toHaveBeenCalledWith(
+      decompressed,
+      "/animations/mixamo/",
+    );
+    expect(hoisted.retargetFbxMock).toHaveBeenCalledWith(fbx, sourceClip, vrm);
+
+    responseArrayBufferSpy.mockRestore();
+    pipeThroughSpy.mockRestore();
+  });
+});

package/src/components/avatar/VrmAnimationLoader.ts

@@ -0,0 +1,151 @@
+import type { VRM } from "@pixiv/three-vrm";
+import type * as THREE from "three";
+import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js";
+import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
+
+/**
+ * Context needed by the animation loader to check whether the owning engine
+ * is still alive / relevant for the load request.
+ */
+export type AnimationLoaderContext = {
+  /** Returns `true` if the loading sequence was aborted (engine disposed). */
+  isAborted: () => boolean;
+  /** Returns `true` if `vrm` is still the active model in the engine. */
+  isCurrentVrm: (vrm: VRM) => boolean;
+};
+
+function isGzipBuffer(buffer: ArrayBuffer): boolean {
+  if (buffer.byteLength < 2) return false;
+  const bytes = new Uint8Array(buffer, 0, 2);
+  return bytes[0] === 0x1f && bytes[1] === 0x8b;
+}
+
+async function decompressGzipBuffer(buffer: ArrayBuffer): Promise<ArrayBuffer> {
+  if (typeof DecompressionStream !== "function") {
+    throw new Error(
+      "This runtime does not support gzip-compressed animation assets.",
+    );
+  }
+
+  const stream = new Blob([buffer])
+    .stream()
+    .pipeThrough(new DecompressionStream("gzip"));
+  return await new Response(stream).arrayBuffer();
+}
+
+async function fetchAnimationBuffer(path: string): Promise<ArrayBuffer> {
+  const response = await fetch(path);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch animation asset: ${response.status}`);
+  }
+
+  const buffer = await response.arrayBuffer();
+  return isGzipBuffer(buffer) ? await decompressGzipBuffer(buffer) : buffer;
+}
+
+function getAssetResourcePath(path: string): string {
+  const withoutQuery = path.split(/[?#]/, 1)[0] ?? path;
+  const lastSlash = withoutQuery.lastIndexOf("/");
+  return lastSlash >= 0 ? withoutQuery.slice(0, lastSlash + 1) : "./";
+}
+
+function isFbxAsset(path: string): boolean {
+  const normalized = path.toLowerCase();
+  return normalized.endsWith(".fbx") || normalized.endsWith(".fbx.gz");
+}
+
+/**
+ * Load and return an idle {@link THREE.AnimationClip} for the given VRM.
+ *
+ * Loads the GLB idle animation and retargets it to VRM bones.
+ * Returns `null` when the load was aborted (engine disposed / VRM replaced).
+ * Throws when the idle clip cannot be loaded.
+ */
+export async function loadIdleClip(
+  vrm: VRM,
+  idleGlbUrl: string,
+  ctx: AnimationLoaderContext,
+): Promise<THREE.AnimationClip | null> {
+  const { retargetMixamoGltfToVrm } = await import("./retargetMixamoGltfToVrm");
+  if (ctx.isAborted() || !ctx.isCurrentVrm(vrm)) return null;
+
+  const gltfLoader = new GLTFLoader();
+  const gltf = await gltfLoader.parseAsync(
+    await fetchAnimationBuffer(idleGlbUrl),
+    getAssetResourcePath(idleGlbUrl),
+  );
+  if (ctx.isAborted() || !ctx.isCurrentVrm(vrm)) return null;
+
+  gltf.scene.updateMatrixWorld(true);
+  vrm.scene.updateMatrixWorld(true);
+  const clip = retargetMixamoGltfToVrm(
+    { scene: gltf.scene, animations: gltf.animations },
+    vrm,
+  );
+
+  if (!clip) {
+    throw new Error("No usable idle animation (idle.glb)");
+  }
+
+  if (ctx.isAborted() || !ctx.isCurrentVrm(vrm)) return null;
+  return clip;
+}
+
+/**
+ * Load a single emote animation clip (GLB/FBX) and retarget it to
+ * the supplied VRM. Returns `null` when the load was aborted or the file
+ * format could not be processed.
+ *
+ * Supports both `.glb` / `.gltf` files (retargeted via retargetMixamoGltfToVrm)
+ * and Mixamo `.fbx` files (retargeted via retargetMixamoFbxToVrm).
+ */
+export async function loadEmoteClip(
+  path: string,
+  vrm: VRM,
+  ctx: AnimationLoaderContext,
+): Promise<THREE.AnimationClip | null> {
+  try {
+    if (!ctx.isCurrentVrm(vrm)) return null;
+
+    const isFbx = isFbxAsset(path);
+
+    if (isFbx) {
+      const { retargetMixamoFbxToVrm } = await import(
+        "./retargetMixamoFbxToVrm"
+      );
+      const fbxLoader = new FBXLoader();
+      const fbx = fbxLoader.parse(
+        await fetchAnimationBuffer(path),
+        getAssetResourcePath(path),
+      );
+      if (!ctx.isCurrentVrm(vrm)) return null;
+
+      const sourceClip = fbx.animations[0];
+      if (!sourceClip) {
+        console.warn(`[VrmEngine] FBX has no animations: ${path}`);
+        return null;
+      }
+      return retargetMixamoFbxToVrm(fbx, sourceClip, vrm);
+    }
+
+    const { retargetMixamoGltfToVrm } = await import(
+      "./retargetMixamoGltfToVrm"
+    );
+    const loader = new GLTFLoader();
+    const gltf = await loader.parseAsync(
+      await fetchAnimationBuffer(path),
+      getAssetResourcePath(path),
+    );
+    if (!ctx.isCurrentVrm(vrm)) return null;
+
+    gltf.scene.updateMatrixWorld(true);
+    vrm.scene.updateMatrixWorld(true);
+    return retargetMixamoGltfToVrm(
+      { scene: gltf.scene, animations: gltf.animations },
+      vrm,
+    );
+  } catch (err) {
+    console.error(`[VrmEngine] Failed to load emote: ${path}`, err);
+    return null;
+  }
+}

package/src/components/avatar/VrmBlinkController.ts

@@ -0,0 +1,118 @@
+/** Blink animation phase */
+type BlinkPhase = "idle" | "closing" | "closed" | "opening";
+
+/**
+ * Self-contained blink state machine for VRM eye-blink animation.
+ *
+ * State flow: idle -> closing -> closed -> opening -> idle
+ * Random interval between blinks with occasional double-blinks.
+ */
+export class VrmBlinkController {
+  private blinkPhase: BlinkPhase = "idle";
+  private blinkTimer = 0;
+  private blinkPhaseTimer = 0;
+  private _blinkValue = 0;
+  private nextBlinkDelay = 2 + Math.random() * 3;
+
+  /** Duration (seconds) for eyelids to close */
+  private static readonly BLINK_CLOSE_DURATION = 0.06;
+  /** Duration (seconds) eyelids stay fully closed */
+  private static readonly BLINK_HOLD_DURATION = 0.04;
+  /** Duration (seconds) for eyelids to re-open */
+  private static readonly BLINK_OPEN_DURATION = 0.12;
+  /** Minimum seconds between blinks */
+  private static readonly BLINK_MIN_INTERVAL = 1.8;
+  /** Maximum seconds between blinks */
+  private static readonly BLINK_MAX_INTERVAL = 5.5;
+  /** Probability of a quick double-blink */
+  private static readonly DOUBLE_BLINK_CHANCE = 0.15;
+
+  /** Current blink expression weight (0 = open, 1 = closed). */
+  get blinkValue(): number {
+    return this._blinkValue;
+  }
+
+  /**
+   * Advance the blink state machine by `delta` seconds.
+   * Returns the current blink expression weight (0..1).
+   */
+  update(delta: number): number {
+    switch (this.blinkPhase) {
+      case "idle":
+        this.blinkTimer += delta;
+        if (this.blinkTimer >= this.nextBlinkDelay) {
+          this.blinkPhase = "closing";
+          this.blinkPhaseTimer = 0;
+        }
+        break;
+
+      case "closing": {
+        this.blinkPhaseTimer += delta;
+        const t = Math.min(
+          1,
+          this.blinkPhaseTimer / VrmBlinkController.BLINK_CLOSE_DURATION,
+        );
+        // Ease-in (accelerate) — eyelids speed up as they close
+        this._blinkValue = t * t;
+        if (t >= 1) {
+          this.blinkPhase = "closed";
+          this.blinkPhaseTimer = 0;
+          this._blinkValue = 1;
+        }
+        break;
+      }
+
+      case "closed":
+        this.blinkPhaseTimer += delta;
+        if (this.blinkPhaseTimer >= VrmBlinkController.BLINK_HOLD_DURATION) {
+          this.blinkPhase = "opening";
+          this.blinkPhaseTimer = 0;
+        }
+        break;
+
+      case "opening": {
+        this.blinkPhaseTimer += delta;
+        const t = Math.min(
+          1,
+          this.blinkPhaseTimer / VrmBlinkController.BLINK_OPEN_DURATION,
+        );
+        // Ease-out (decelerate) — eyelids slow down as they finish opening
+        const eased = 1 - (1 - t) * (1 - t);
+        this._blinkValue = 1 - eased;
+        if (t >= 1) {
+          this.blinkPhase = "idle";
+          this.blinkPhaseTimer = 0;
+          this._blinkValue = 0;
+          this.blinkTimer = 0;
+          this.scheduleNextBlink();
+        }
+        break;
+      }
+    }
+
+    return this._blinkValue;
+  }
+
+  /** Reset blink state (called when a new VRM is loaded). */
+  reset(): void {
+    this.blinkPhase = "idle";
+    this.blinkTimer = 0;
+    this.blinkPhaseTimer = 0;
+    this._blinkValue = 0;
+    this.nextBlinkDelay = 1.5 + Math.random() * 2;
+  }
+
+  /** Pick the delay (seconds) until the next blink. */
+  private scheduleNextBlink(): void {
+    const range =
+      VrmBlinkController.BLINK_MAX_INTERVAL -
+      VrmBlinkController.BLINK_MIN_INTERVAL;
+    this.nextBlinkDelay =
+      VrmBlinkController.BLINK_MIN_INTERVAL + Math.random() * range;
+
+    // Occasional quick double-blink
+    if (Math.random() < VrmBlinkController.DOUBLE_BLINK_CHANCE) {
+      this.nextBlinkDelay = 0.12 + Math.random() * 0.08;
+    }
+  }
+}