@usamanawaz/react-native-ai-avatar 0.0.0-stage → 1.0.0

package/src/avatar/model/blendshapes.ts

@@ -0,0 +1,70 @@
+/**
+ * Face contract for every GLB this library loads.
+ * Names match Apple's ARKit blend shapes so one mesh works for blink,
+ * gaze, expression, and lip-sync.
+ */
+export const BLENDSHAPES = [
+  'browDownLeft',
+  'browDownRight',
+  'browInnerUp',
+  'browOuterUpLeft',
+  'browOuterUpRight',
+  'cheekPuff',
+  'cheekSquintLeft',
+  'cheekSquintRight',
+  'eyeBlinkLeft',
+  'eyeBlinkRight',
+  'eyeLookDownLeft',
+  'eyeLookDownRight',
+  'eyeLookInLeft',
+  'eyeLookInRight',
+  'eyeLookOutLeft',
+  'eyeLookOutRight',
+  'eyeLookUpLeft',
+  'eyeLookUpRight',
+  'eyeSquintLeft',
+  'eyeSquintRight',
+  'eyeWideLeft',
+  'eyeWideRight',
+  'jawForward',
+  'jawLeft',
+  'jawRight',
+  'jawOpen',
+  'mouthClose',
+  'mouthFunnel',
+  'mouthPucker',
+  'mouthLeft',
+  'mouthRight',
+  'mouthSmileLeft',
+  'mouthSmileRight',
+  'mouthFrownLeft',
+  'mouthFrownRight',
+  'mouthDimpleLeft',
+  'mouthDimpleRight',
+  'mouthStretchLeft',
+  'mouthStretchRight',
+  'mouthRollLower',
+  'mouthRollUpper',
+  'mouthShrugLower',
+  'mouthShrugUpper',
+  'mouthPressLeft',
+  'mouthPressRight',
+  'mouthLowerDownLeft',
+  'mouthLowerDownRight',
+  'mouthUpperUpLeft',
+  'mouthUpperUpRight',
+  'noseSneerLeft',
+  'noseSneerRight',
+  'tongueOut',
+] as const;
+
+type AssertLength<T extends readonly unknown[], N extends number> = T['length'] extends N
+  ? T
+  : never;
+
+export const ARKIT_BLENDSHAPES: AssertLength<typeof BLENDSHAPES, 52> = BLENDSHAPES;
+
+export type BlendshapeName = (typeof BLENDSHAPES)[number];
+
+/** Morph target weights. Missing keys stay at 0. Values are 0 to 1. */
+export type BlendshapeWeights = Partial<Record<BlendshapeName, number>>;

package/src/avatar/model/boneMap.ts

@@ -0,0 +1,17 @@
+import type { BoneName } from './skeleton';
+
+/**
+ * MPFB default-rig names, then the names Three.js actually puts on the
+ * objects. Dots are stripped on load (`eye.L` becomes `eyeL`) because they
+ * are reserved in animation paths. The GLB itself is left unchanged.
+ */
+export const MPFB_BONE_SOURCES: Record<BoneName, readonly string[]> = {
+  Head: ['head', 'Head', 'mixamorigHead'],
+  Neck: ['neck01', 'neck02', 'neck03', 'Neck', 'mixamorigNeck'],
+  LeftEye: ['eye.L', 'eyeL', 'LeftEye'],
+  RightEye: ['eye.R', 'eyeR', 'RightEye'],
+  Jaw: ['jaw', 'Jaw'],
+};
+
+/** Neck rotation is shared across the three neck bones so one control does not triple the bend. */
+export const NECK_BONE_SHARE = 1 / 3;

package/src/avatar/model/catalog.ts

@@ -0,0 +1,15 @@
+import type { AvatarModelId, AvatarOutfit } from '../types';
+
+const MODELS: Partial<Record<AvatarModelId, number>> = {
+  male: require('../../../assets/models/male-avatar.glb'),
+};
+
+const OUTFITS: Record<AvatarOutfit, number> = {
+  casual: require('../../../assets/models/male-avatar-clothed.glb'),
+};
+
+export function avatarModelSource(id: AvatarModelId, outfit?: AvatarOutfit): number | null {
+  if (outfit) return OUTFITS[outfit] ?? null;
+  return MODELS[id] ?? null;
+}
+

package/src/avatar/model/cpuMorphs.ts

@@ -0,0 +1,124 @@
+import * as THREE from 'three';
+
+import type { BlendshapeWeights } from './blendshapes';
+
+/**
+ * Expo GL on iOS (especially the Simulator) often cannot sample the float
+ * DataArrayTexture Three.js uses for morph targets. We strip GPU morphs and
+ * apply the same deltas on the CPU so the body mesh still draws.
+ */
+export type CpuMorphMesh = {
+  mesh: THREE.Mesh;
+  dictionary: Record<string, number>;
+  basePosition: Float32Array;
+  baseNormal: Float32Array | null;
+  positionDeltas: Float32Array[];
+  normalDeltas: Float32Array[] | null;
+  position: THREE.BufferAttribute;
+  normal: THREE.BufferAttribute | null;
+  scratchPosition: Float32Array;
+  scratchNormal: Float32Array | null;
+  lastKey: string;
+  /** When set, expression offsets are stored here so CPU skinning can run after them. */
+  morphOut?: Float32Array;
+};
+
+export function extractCpuMorphs(mesh: THREE.Mesh): CpuMorphMesh | null {
+  const dictionary = mesh.morphTargetDictionary;
+  const geometry = mesh.geometry;
+  const morphPositions = geometry.morphAttributes.position;
+  if (!dictionary || !morphPositions || morphPositions.length === 0) return null;
+
+  const position = geometry.getAttribute('position') as THREE.BufferAttribute;
+  const normal = geometry.getAttribute('normal') as THREE.BufferAttribute | null;
+  const morphNormals = geometry.morphAttributes.normal;
+
+  const basePosition = floatCopy(position);
+  const baseNormal = normal ? floatCopy(normal) : null;
+  const positionDeltas = morphPositions.map((attr) => floatCopy(attr as THREE.BufferAttribute));
+  const normalDeltas =
+    normal && morphNormals && morphNormals.length === morphPositions.length
+      ? morphNormals.map((attr) => floatCopy(attr as THREE.BufferAttribute))
+      : null;
+
+  // Drop GPU morphs so Three.js never uploads a float morph texture.
+  geometry.morphAttributes = {};
+  delete mesh.morphTargetDictionary;
+  delete mesh.morphTargetInfluences;
+
+  return {
+    mesh,
+    dictionary: { ...dictionary },
+    basePosition,
+    baseNormal,
+    positionDeltas,
+    normalDeltas,
+    position,
+    normal,
+    scratchPosition: new Float32Array(basePosition.length),
+    scratchNormal: baseNormal ? new Float32Array(baseNormal.length) : null,
+    lastKey: '',
+  };
+}
+
+export function applyCpuMorphs(targets: CpuMorphMesh[], weights: BlendshapeWeights) {
+  const key = weightKey(weights);
+  for (const target of targets) {
+    if (target.lastKey === key) continue;
+    target.lastKey = key;
+    const { scratchPosition, basePosition, positionDeltas, dictionary } = target;
+    scratchPosition.set(basePosition);
+
+    for (const [name, weight] of Object.entries(weights)) {
+      if (!weight) continue;
+      const index = dictionary[name];
+      if (index === undefined) continue;
+      const delta = positionDeltas[index];
+      if (!delta) continue;
+      const amount = weight;
+      for (let i = 0; i < scratchPosition.length; i += 1) {
+        scratchPosition[i] += delta[i] * amount;
+      }
+    }
+
+    if (target.morphOut) {
+      const morph = target.morphOut;
+      const base = target.basePosition;
+      for (let i = 0; i < scratchPosition.length; i += 1) morph[i] = scratchPosition[i] - base[i];
+    } else {
+      (target.position.array as Float32Array).set(scratchPosition);
+      target.position.needsUpdate = true;
+    }
+
+    if (target.scratchNormal && target.baseNormal && target.normal && target.normalDeltas) {
+      const scratchNormal = target.scratchNormal;
+      scratchNormal.set(target.baseNormal);
+      for (const [name, weight] of Object.entries(weights)) {
+        if (!weight) continue;
+        const index = dictionary[name];
+        if (index === undefined) continue;
+        const delta = target.normalDeltas[index];
+        if (!delta) continue;
+        for (let i = 0; i < scratchNormal.length; i += 1) {
+          scratchNormal[i] += delta[i] * weight;
+        }
+      }
+      (target.normal.array as Float32Array).set(scratchNormal);
+      target.normal.needsUpdate = true;
+    }
+  }
+}
+
+function floatCopy(attr: THREE.BufferAttribute): Float32Array {
+  const array = attr.array;
+  return array instanceof Float32Array ? new Float32Array(array) : Float32Array.from(array as ArrayLike<number>);
+}
+
+function weightKey(weights: BlendshapeWeights): string {
+  const parts: string[] = [];
+  for (const [name, weight] of Object.entries(weights)) {
+    if (!weight || weight < 0.001) continue;
+    parts.push(`${name}:${weight.toFixed(3)}`);
+  }
+  return parts.join('|');
+}

package/src/avatar/model/cpuSkin.ts

@@ -0,0 +1,105 @@
+import * as THREE from 'three';
+
+/**
+ * Expo GL cannot sample the float bone texture Three.js uses for SkinnedMesh.
+ * The mesh is drawn as a plain Mesh, and bone motion is applied on the CPU.
+ */
+export type CpuSkin = {
+  position: THREE.BufferAttribute;
+  bind: Float32Array;
+  /** Added to the bind position before skinning. Face shapes write this. */
+  morph: Float32Array;
+  skinIndex: THREE.BufferAttribute;
+  skinWeight: THREE.BufferAttribute;
+  skeleton: THREE.Skeleton;
+  bindMatrix: THREE.Matrix4;
+  bindMatrixInverse: THREE.Matrix4;
+  boneMatrices: THREE.Matrix4[];
+};
+
+const composed = new THREE.Matrix4();
+
+export function createCpuSkins(scene: THREE.Object3D): CpuSkin[] {
+  const skinned: THREE.SkinnedMesh[] = [];
+  scene.traverse((object) => {
+    if ((object as THREE.SkinnedMesh).isSkinnedMesh) skinned.push(object as THREE.SkinnedMesh);
+  });
+
+  const skins: CpuSkin[] = [];
+  for (const source of skinned) {
+    const parent = source.parent;
+    const geometry = source.geometry;
+    const position = geometry.getAttribute('position') as THREE.BufferAttribute | undefined;
+    const skinIndex = geometry.getAttribute('skinIndex') as THREE.BufferAttribute | undefined;
+    const skinWeight = geometry.getAttribute('skinWeight') as THREE.BufferAttribute | undefined;
+    if (!parent || !source.skeleton || !position || !skinIndex || !skinWeight) continue;
+
+    source.updateMatrixWorld(true);
+    const mesh = new THREE.Mesh(geometry, source.material);
+    mesh.name = source.name;
+    mesh.frustumCulled = false;
+    mesh.position.copy(source.position);
+    mesh.quaternion.copy(source.quaternion);
+    mesh.scale.copy(source.scale);
+    mesh.morphTargetDictionary = source.morphTargetDictionary ? { ...source.morphTargetDictionary } : undefined;
+    mesh.morphTargetInfluences = source.morphTargetInfluences ? source.morphTargetInfluences.slice() : undefined;
+    parent.add(mesh);
+    parent.remove(source);
+
+    skins.push({
+      position,
+      bind: Float32Array.from(position.array as ArrayLike<number>),
+      morph: new Float32Array(position.count * 3),
+      skinIndex,
+      skinWeight,
+      skeleton: source.skeleton,
+      bindMatrix: source.bindMatrix.clone(),
+      bindMatrixInverse: source.bindMatrixInverse.clone(),
+      boneMatrices: source.skeleton.bones.map(() => new THREE.Matrix4()),
+    });
+  }
+  return skins;
+}
+
+export function updateCpuSkins(skins: CpuSkin[]) {
+  for (const skin of skins) {
+    const { skeleton, boneMatrices, bindMatrix, bindMatrixInverse } = skin;
+    const bones = skeleton.bones;
+    const inverses = skeleton.boneInverses;
+    for (let bone = 0; bone < bones.length; bone += 1) {
+      composed.multiplyMatrices(bones[bone].matrixWorld, inverses[bone]);
+      composed.multiply(bindMatrix);
+      boneMatrices[bone].multiplyMatrices(bindMatrixInverse, composed);
+    }
+
+    const out = skin.position.array as Float32Array;
+    const bind = skin.bind;
+    const morph = skin.morph;
+    const indices = skin.skinIndex.array as ArrayLike<number>;
+    const weights = skin.skinWeight.array as ArrayLike<number>;
+    const stride = skin.skinIndex.itemSize;
+    const count = skin.position.count;
+
+    for (let vertex = 0; vertex < count; vertex += 1) {
+      const x = bind[vertex * 3] + morph[vertex * 3];
+      const y = bind[vertex * 3 + 1] + morph[vertex * 3 + 1];
+      const z = bind[vertex * 3 + 2] + morph[vertex * 3 + 2];
+      let ox = 0;
+      let oy = 0;
+      let oz = 0;
+      const base = vertex * stride;
+      for (let influence = 0; influence < stride; influence += 1) {
+        const weight = weights[base + influence];
+        if (weight === 0) continue;
+        const elements = boneMatrices[indices[base + influence]].elements;
+        ox += weight * (elements[0] * x + elements[4] * y + elements[8] * z + elements[12]);
+        oy += weight * (elements[1] * x + elements[5] * y + elements[9] * z + elements[13]);
+        oz += weight * (elements[2] * x + elements[6] * y + elements[10] * z + elements[14]);
+      }
+      out[vertex * 3] = ox;
+      out[vertex * 3 + 1] = oy;
+      out[vertex * 3 + 2] = oz;
+    }
+    skin.position.needsUpdate = true;
+  }
+}

package/src/avatar/model/jpeg-js.d.ts

@@ -0,0 +1,6 @@
+declare module 'jpeg-js' {
+  export function decode(
+    jpegData: Uint8Array,
+    opts?: { useTArray?: boolean; formatAsRGBA?: boolean },
+  ): { width: number; height: number; data: Uint8Array };
+}