@convai/embody 0.2.1 → 0.3.0

package/CHANGELOG.md

@@ -1,5 +1,55 @@
 # @convai/embody
 
+## 0.3.0
+
+### Minor Changes
+
+- f9d8f06: Characters now look at you, and their mouths move by default.
+
+  Four fixes, all of which change what a character does with no code change on
+  the consumer's side. Minor rather than patch because default behaviour moves
+  and one interface gained a member.
+
+  **Lipsync worked only if you brought your own client.** `createSession` never
+  set `enableLipsync`, and the server sends no blendshape frames unless asked —
+  so the zero-config path, `<ConvaiCharacter apiKey characterId />`, connected,
+  replied, played audio and never moved its mouth. It worked when an app built
+  its own `ConvaiClient` and passed `client`, because then the app supplied the
+  flag. That asymmetry was recorded for weeks as "staging emits no blendshapes";
+  it was never the environment. `createSession` now defaults `enableLipsync:
+true` and `blendshapeConfig: { format: 'mha' }`, with your config spread last
+  so you can still disable it or choose another format.
+
+  **The head aimed past its target by the rig's rest tilt.** The look angle was
+  solved in the head's parent frame but applied composed onto the head's rest
+  pose — two different frames. On a rig whose bones rest at identity they
+  coincide, which is why no test caught it; on a real MetaHuman the head settled
+  17–20° above a level camera, chin up, while reporting itself on target.
+
+  **The eyes tracked the head's own turn instead of the target.**
+  `EyeGazeSystem` read `GazeSystem.direction`, which is how far the head has
+  turned from neutral, when the eyes need how far the head is _still_ off target.
+  Those diverge exactly when the head is aimed at you — so the eyes rolled
+  further the way the head had already gone, and a character looking straight at
+  you stared past you. Measured on a live body, `eyeLookUp` sat pinned at 0.203
+  for ever.
+
+  **New:** a face material pass, on by default. The defect it fixes is in the
+  exports rather than in any app: the teeth material arrives with its albedo
+  unbound, so glTF loaders never materialise the `Teeth` image the file ships and
+  the mouth renders as one flat white slab. The pass fetches it through the glTF
+  parser, tags it sRGB, makes the mouth unlit so it reads the same in sun and
+  shade, and adds a catchlight to eyeballs that have a lighting model. Pass
+  `faceMaterials: false` to leave every material exactly as the GLB shipped it.
+
+  ### Breaking
+
+  `GazeSource` gained a required `residual` member. If you implement that
+  interface yourself — rather than consuming `GazeSystem` — you now need to
+  supply it. `direction` is unchanged, and `BlinkSystem` still reads it: lids
+  follow the head's deflection from neutral, eyes follow the residual, and one
+  field could not serve both.
+
 ## 0.2.1
 
 ### Patch Changes

package/dist/character/createCharacter.d.ts

@@ -16,6 +16,7 @@ import { type ResolvedRig } from '../rig/resolve.js';
 import { GazeSystem, type GazeConfig } from '../motion/gaze.js';
 import { type BreathingConfig } from '../motion/breathe.js';
 import { type EyeGazeConfig } from '../motion/eyeGaze.js';
+import { type FaceMaterialTuning } from '../render/faceMaterials.js';
 import { type BlinkConfig } from '../motion/blink.js';
 import { type ReducedMotion } from '../motion/reducedMotion.js';
 /**
@@ -110,6 +111,18 @@ export interface CreateCharacterOptions {
      * systems — no gaze, breathing, eye-look or blink. Default `true`.
      */
     motion?: boolean;
+    /**
+     * Eye, teeth and socket shading. On by default, because the defect it fixes
+     * is in the exports rather than in any app: eyeballs ship with no catchlight
+     * and read as plastic, and the teeth material arrives with its albedo
+     * unbound so the mouth renders as one flat white slab.
+     *
+     * Pass `false` to leave every material exactly as the GLB shipped it — for a
+     * pipeline that does its own look development, or a rig whose materials are
+     * not named on the MetaHuman convention this matches (`Eyes*`, `Teeth*`,
+     * `Eyeocclusion*`).
+     */
+    faceMaterials?: boolean | FaceMaterialTuning;
 }
 export interface CharacterHandle {
     root: THREE.Object3D;

package/dist/character/createCharacter.js

@@ -12,6 +12,7 @@ import { resolveRig } from '../rig/resolve.js';
 import { GazeSystem } from '../motion/gaze.js';
 import { BreathingSystem } from '../motion/breathe.js';
 import { EyeGazeSystem } from '../motion/eyeGaze.js';
+import { applyFaceMaterials } from '../render/faceMaterials.js';
 import { BlinkSystem } from '../motion/blink.js';
 import { createReducedMotion } from '../motion/reducedMotion.js';
 /**
@@ -196,6 +197,12 @@ export async function createCharacter(opts) {
         draco.dispose();
     }
     const root = modelGltf.scene;
+    // Shading pass, before anything reads the scene. Not awaited: the teeth
+    // texture it rescues resolves on its own and lands a frame or two later,
+    // and the mouth is correctly shaped and coloured in the meantime.
+    if (opts.faceMaterials !== false) {
+        applyFaceMaterials(root, opts.faceMaterials === true || opts.faceMaterials === undefined ? {} : opts.faceMaterials, modelGltf.parser);
+    }
     // gather every clip whose name the manifest/overrides declared
     const wanted = new Set(clipUrls.keys());
     const clips = [];

package/dist/connect/session.js

@@ -23,7 +23,21 @@ import { AudioRenderer } from '@convai/web-sdk/vanilla';
  */
 export async function createSession(opts) {
     const { audio = true, ...config } = opts;
-    const client = new ConvaiClient(config);
+    // Ask for the frames embody exists to render. The server sends NO
+    // blendshapes unless `enableLipsync` is set, and embody's only registered
+    // rig is convai-mha-1, which wants MHA-format frames — so the zero-config
+    // path `<ConvaiCharacter apiKey characterId />` produced a character that
+    // connected, replied, played audio and never moved its mouth. It only
+    // worked when an app built its own client and passed `client`, because
+    // then the APP supplied these, which is exactly what this demo used to do.
+    //
+    // `config` spreads LAST so an app can still disable lipsync or choose
+    // another blendshape format.
+    const client = new ConvaiClient({
+        enableLipsync: true,
+        blendshapeConfig: { format: 'mha' },
+        ...config,
+    });
     await client.connect();
     const renderer = audio ? new AudioRenderer(client.room) : null;
     return wrapSession(client, renderer, true);

package/dist/motion/eyeGaze.js

@@ -36,7 +36,13 @@ export class EyeGazeSystem {
         const lerpSpeed = this.cfg.lerpSpeed ?? 0.2;
         const verticalOffset = this.cfg.verticalOffset ?? -0.15;
         const gain = this.cfg.morphGain ?? 0.65;
-        const d = this.gaze.direction;
+        // RESIDUAL, not `direction`. `direction` is how far the HEAD has turned
+        // from neutral; the eyes need how far the head is STILL off target. They
+        // differ exactly when the head is on target, which is most of the time:
+        // driving the eyes from `direction` rolls them further the way the head
+        // already went, so a character looking straight at you stares past you.
+        // On the live convai-mha-1 body that pinned eyeLookUp at 0.20 forever.
+        const d = this.gaze.residual;
         const targetH = ctx.reducedMotion ? 0 : THREE.MathUtils.clamp(d.horizontal, -1, 1);
         const targetV = ctx.reducedMotion
             ? 0

package/dist/motion/gaze.d.ts

@@ -1,5 +1,5 @@
 import type { FrameContext, System } from '../character/pipeline.js';
-import type { GazeDirection, GazeSource } from './types.js';
+import type { EyeGazeDirection, GazeDirection, GazeSource } from './types.js';
 /**
  * Config surface for `GazeSystem`. Task 6 landed the geometric core —
  * target -> yaw/pitch -> head/neck split -> time-based, velocity-capped
@@ -110,6 +110,8 @@ export declare class GazeSystem implements System, GazeSource {
     private readonly cfg;
     readonly stage: 2;
     readonly direction: GazeDirection;
+    /** World-space angle still between the head's forward and the target. */
+    readonly residual: EyeGazeDirection;
     private gate;
     private yaw;
     private pitch;
@@ -141,6 +143,12 @@ export declare class GazeSystem implements System, GazeSource {
     private readonly vHead;
     private readonly vLocal;
     private readonly mInv;
+    private readonly vAim;
+    private readonly vFwdW;
+    private readonly vRightW;
+    private readonly vUpW;
+    private readonly qHeadW;
+    private readonly qRestInv;
     /** What the mixer wrote to the head this frame, before we touch it. */
     private readonly qClip;
     /** The camera-locked look pose computed this frame. */

package/dist/motion/gaze.js

@@ -38,6 +38,8 @@ export class GazeSystem {
             right: { horizontal: 0, vertical: 0 },
             horizontal: 0, vertical: 0,
         };
+        /** World-space angle still between the head's forward and the target. */
+        this.residual = { horizontal: 0, vertical: 0 };
         this.yaw = 0;
         this.pitch = 0;
         /** Idle-roll + turn-bank, written by the aliveness layer below. */
@@ -64,6 +66,13 @@ export class GazeSystem {
         this.vHead = new THREE.Vector3();
         this.vLocal = new THREE.Vector3();
         this.mInv = new THREE.Matrix4();
+        // Scratch for the world-space residual published to EyeGazeSystem.
+        this.vAim = new THREE.Vector3();
+        this.vFwdW = new THREE.Vector3();
+        this.vRightW = new THREE.Vector3();
+        this.vUpW = new THREE.Vector3();
+        this.qHeadW = new THREE.Quaternion();
+        this.qRestInv = new THREE.Quaternion();
         /** What the mixer wrote to the head this frame, before we touch it. */
         this.qClip = new THREE.Quaternion();
         /** The camera-locked look pose computed this frame. */
@@ -176,6 +185,25 @@ export class GazeSystem {
         const parent = head.parent ?? ctx.root;
         this.mInv.copy(parent.matrixWorld).invert();
         this.vLocal.subVectors(this.vTarget, this.vHead).transformDirection(this.mInv);
+        // Into the head's REST frame. The yaw/pitch solved below are applied as
+        // `restHead * qLook` (the premultiply further down), i.e. RELATIVE to the
+        // head's rest orientation — so they must be MEASURED relative to it too.
+        //
+        //   want:  P · restHead · qLook · fwd = aim        (P = parent world quat)
+        //   so:    qLook solves against restHead⁻¹ · P⁻¹ · aim
+        //
+        // `transformDirection(mInv)` above supplies only P⁻¹ · aim. Without the
+        // restHead⁻¹ the solve measures in one frame and applies in another, and
+        // the head lands mis-aimed by exactly the rest tilt.
+        //
+        // Invisible on a rig whose bones rest at identity — which is every older
+        // fixture in this file. On the japanese-f fixture the head rests 11.42°
+        // tilted; measured on the live convai-mha-1 body, a level camera left the
+        // head aiming 17-20° ABOVE it, chin up, while the solve reported itself on
+        // target. That is the "it never looks at me directly" symptom.
+        const headRestFrame = ctx.rig.restPose.head;
+        if (headRestFrame)
+            this.vLocal.applyQuaternion(this.qRestInv.copy(headRestFrame).invert());
         const basis = ctx.rig.profile.basis;
         // Yaw/pitch of the target about the rig's own bone-local axes.
         const right = this.vLocal.dot(basis.right);
@@ -463,5 +491,40 @@ export class GazeSystem {
         this.direction.left.vertical = v;
         this.direction.right.horizontal = h;
         this.direction.right.vertical = v;
+        // The EYES need a different quantity, measured in WORLD space from the
+        // head's ACTUAL orientation after the writes above — not from
+        // `this.yaw`/`this.pitch`, which are bone-space CORRECTION angles.
+        //
+        // Those correction angles also carry the head's own rest tilt and the
+        // neck chain's orientation. On a rig whose bones rest at identity the
+        // two coincide, which is why this was invisible in every test here. On a
+        // real MetaHuman it is badly wrong: measured in a browser on the live
+        // convai-mha-1 body, a camera 3cm above the head (true angle 0.48°)
+        // published vertical = 0.456, 42x too large, because the head bone rests
+        // ~11.5° tilted and the solve must undo that before it aims anywhere.
+        // EyeGazeSystem read that as "looking up at 46% of maximum" and pinned
+        // eyeLookUp at 0.20 — the eyes never tracked, they were stuck up, and a
+        // character looking straight at you appeared to stare over your shoulder.
+        //
+        // `direction` above is deliberately NOT changed: BlinkSystem wants the
+        // head's deflection from neutral (lids follow the head), and two tests
+        // pin that meaning. One field cannot serve both, so the eyes get their
+        // own. `getWorldQuaternion` refreshes the matrix chain, so this reads
+        // this frame's pose rather than last frame's.
+        head.getWorldQuaternion(this.qHeadW);
+        head.getWorldPosition(this.vHead);
+        ctx.camera.getWorldPosition(this.vTarget);
+        this.vAim.subVectors(this.vTarget, this.vHead);
+        if (this.vAim.lengthSq() > 1e-12) {
+            this.vAim.normalize();
+            this.vFwdW.copy(basis.fwd).applyQuaternion(this.qHeadW);
+            this.vRightW.copy(basis.right).applyQuaternion(this.qHeadW);
+            this.vUpW.copy(basis.up).applyQuaternion(this.qHeadW);
+            const aimR = this.vAim.dot(this.vRightW);
+            const aimU = this.vAim.dot(this.vUpW);
+            const aimF = this.vAim.dot(this.vFwdW);
+            this.residual.horizontal = THREE.MathUtils.clamp(Math.atan2(aimR, aimF) / maxTrack, -1, 1);
+            this.residual.vertical = THREE.MathUtils.clamp(Math.atan2(aimU, Math.hypot(aimR, aimF)) / maxPitch, -1, 1);
+        }
     }
 }

package/dist/motion/types.d.ts

@@ -12,5 +12,23 @@ export interface GazeDirection {
 }
 /** Anything that can tell a downstream system where the character is looking. */
 export interface GazeSource {
+    /**
+     * How far the HEAD has turned from its neutral, normalised by the tracking
+     * limits. This is what "the character is looking up/right" means, and it is
+     * what BlinkSystem wants: lids follow the head's own deflection.
+     *
+     * It is NOT what the eyes want. See `residual`.
+     */
     readonly direction: GazeDirection;
+    /**
+     * How far the head STILL points away from the gaze target, in world space,
+     * after this frame's turn. Zero when the head is aimed at the target.
+     *
+     * This is what EyeGazeSystem wants, and the two genuinely differ: a head
+     * that has finished turning to face you has a large `direction` (it turned
+     * a long way) and a `residual` of zero (it is now on target). Driving the
+     * eyes from `direction` rolls them further in the direction the head has
+     * already gone, so a character looking straight at you stares past you.
+     */
+    readonly residual: EyeGazeDirection;
 }

package/dist/render/faceMaterials.d.ts

@@ -0,0 +1,103 @@
+import * as THREE from 'three';
+/**
+ * Face material pass — the half of "the eyes and teeth look wrong" that no
+ * amount of animation can fix.
+ *
+ * Ported from the pagoda-act scene, where every value below was arrived at by
+ * looking at a rendered face rather than by reasoning about shading. The
+ * lipsync layer in `src/speech/lipsync.ts` is the other half of the same work;
+ * neither is sufficient alone, and the teeth in particular need both:
+ *
+ *   SHAPE — the arches must retract and tuck as the jaw moves. That lives in
+ *           the lipsync layer (teethRetract / teethUpperLift / teethLowerTuck)
+ *           and is already here.
+ *   SHADE — the arches must be textured and lit like a mouth interior. That is
+ *           this file, and without it the mouth renders as one flat slab.
+ *
+ * Two distinct defects, both shading:
+ *
+ *  · THE EYES READ DEAD. Exports ship the eyeball as a matte-ish dielectric
+ *    with no wet dome, so there is no catchlight. A real eye is a glossy
+ *    sphere under a near-mirror cornea; with no specular highlight the viewer
+ *    reads plastic no matter how well the gaze tracks.
+ *  · THE TEETH READ TOO BRIGHT. A real mouth sits in shadow. Darkening the
+ *    enamel and cutting its lighting response is what finally allows an honest
+ *    jaw opening — open vowels then show a shadowed glimpse of the upper row
+ *    instead of a bright white band.
+ *
+ * TWO DELIBERATE DEVIATIONS from the original, both because this is a library
+ * and that was one scene:
+ *
+ *  1. No cross-character texture sharing. The original keeps a module-level
+ *     `sharedTeethMap` so a character whose export ships no teeth image can
+ *     borrow one from whoever else is mounted. Two characters in one scene can
+ *     afford that. A package cannot: `CharacterHandle.dispose()` frees every
+ *     texture it finds, so a borrowed map would be disposed out from under a
+ *     character that is still using it, and the failure would appear on an
+ *     unrelated character at an unrelated time. Each character gets its own
+ *     map or none.
+ *  2. Replaced materials are disposed here, not leaked. Swapping a lit teeth
+ *     material for an unlit one orphans the original — `dispose()` walks the
+ *     scene graph and would only ever see the replacement.
+ */
+/** What this actually needs of `GLTF.parser`, structurally. */
+export type GltfParserLike = {
+    json?: {
+        images?: Array<{
+            name?: string;
+        }>;
+        textures?: Array<{
+            source?: number;
+        }>;
+    };
+    getDependency?: (type: string, index: number) => Promise<unknown>;
+};
+export interface FaceMaterialTuning {
+    /**
+     * Per-character multiplier on the teeth albedo.
+     *
+     * Exists because characters stand in different light and nothing occludes a
+     * mouth. In the original scene the two characters' teeth materials were
+     * measured IDENTICAL and only the lighting differed, so one shared value
+     * could not satisfy both — every attempt to find one made whichever
+     * character was currently wrong, right, and the other one wrong.
+     *
+     * 1 is the original's tuning and the default, so nothing silently changes.
+     */
+    teethScale?: number;
+}
+export interface FaceMaterialReport {
+    eyes: number;
+    teeth: number;
+    occlusion: number;
+}
+/**
+ * Pull this model's teeth albedo out of the file, bound or not.
+ *
+ * THE V2 EXPORTS ORPHAN IT. The GLB ships a `Teeth` image — alongside
+ * Teeth_spec, Teeth_normal and Teeth_roughness — and a texture entry pointing
+ * at it. What it does not ship is a material that USES it: the teeth material
+ * arrives with `baseColorFactor: [0,0,0,1]` and no `baseColorTexture`. glTF
+ * loaders only materialise textures some material references, so the map is
+ * present in the file, never becomes a THREE.Texture, and the mouth renders as
+ * one flat untextured slab with no tooth separation. That is the "everything
+ * is white" report.
+ *
+ * `parser.getDependency('texture', i)` loads any texture entry, referenced or
+ * not, which is exactly the hole.
+ *
+ * Matches the image name EXACTLY — `Teeth`, never `Teeth_spec`,
+ * `Teeth_normal` or `Teeth_roughness`, all three of which sit next to it and
+ * none of which is a colour map. Getting that wrong puts a normal map in the
+ * albedo slot, which reads as a lilac mouth rather than as an error.
+ */
+export declare function loadTeethTexture(parser: GltfParserLike | undefined): Promise<THREE.Texture | null>;
+/**
+ * Apply the face pass. Synchronous for the material work; the teeth texture
+ * resolves on its own and lands a frame or two later.
+ *
+ * Deliberately not awaited by the caller: the teeth are correctly shaped and
+ * correctly coloured in the meantime, they simply have no tooth separation
+ * until the image decodes.
+ */
+export declare function applyFaceMaterials(root: THREE.Object3D, tuning?: FaceMaterialTuning, parser?: GltfParserLike): FaceMaterialReport;

package/dist/render/faceMaterials.js

@@ -0,0 +1,210 @@
+import * as THREE from 'three';
+/**
+ * Pull this model's teeth albedo out of the file, bound or not.
+ *
+ * THE V2 EXPORTS ORPHAN IT. The GLB ships a `Teeth` image — alongside
+ * Teeth_spec, Teeth_normal and Teeth_roughness — and a texture entry pointing
+ * at it. What it does not ship is a material that USES it: the teeth material
+ * arrives with `baseColorFactor: [0,0,0,1]` and no `baseColorTexture`. glTF
+ * loaders only materialise textures some material references, so the map is
+ * present in the file, never becomes a THREE.Texture, and the mouth renders as
+ * one flat untextured slab with no tooth separation. That is the "everything
+ * is white" report.
+ *
+ * `parser.getDependency('texture', i)` loads any texture entry, referenced or
+ * not, which is exactly the hole.
+ *
+ * Matches the image name EXACTLY — `Teeth`, never `Teeth_spec`,
+ * `Teeth_normal` or `Teeth_roughness`, all three of which sit next to it and
+ * none of which is a colour map. Getting that wrong puts a normal map in the
+ * albedo slot, which reads as a lilac mouth rather than as an error.
+ */
+export async function loadTeethTexture(parser) {
+    const json = parser?.json;
+    if (!json?.images || !json.textures || !parser?.getDependency)
+        return null;
+    const image = json.images.findIndex((im) => /^teeth$/i.test(im?.name ?? ''));
+    if (image < 0)
+        return null;
+    const texture = json.textures.findIndex((t) => t?.source === image);
+    if (texture < 0)
+        return null;
+    try {
+        const tex = (await parser.getDependency('texture', texture));
+        if (!tex)
+            return null;
+        // getDependency hands back the raw texture. Colour space is normally
+        // applied by the material assignment that never happened here, and an
+        // albedo left in linear renders washed out — which would look like a
+        // PARTIAL fix and be much harder to spot than no texture at all.
+        tex.colorSpace = THREE.SRGBColorSpace;
+        tex.needsUpdate = true;
+        return tex;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Mutate in place, and feature-test every property.
+ *
+ * The original's first attempt built a fresh MeshPhysicalMaterial and
+ * `copy()`d the old one into it, to get clearcoat. That threw
+ * "Cannot read properties of undefined" out of the renderer and took the whole
+ * canvas down: on a node-material path these are node materials, and a
+ * hand-rolled swap loses internals the renderer then dereferences. Mutating
+ * the material the loader made keeps its maps, its node graph and its
+ * skinning/morph bindings — and anything unsupported is simply skipped.
+ */
+function tune(mat, teethScale, found) {
+    const m = mat;
+    const name = m.name || '';
+    if (/^Eyes/i.test(name)) {
+        // ONLY on a material that has a lighting model.
+        //
+        // Measured on the live convai-mha-1 body: its eye materials arrive as
+        // MeshBasicMaterial — already unlit. Gloss cannot exist there, so
+        // `roughness`, `clearcoat` and `specularIntensity` would be inert
+        // properties bolted onto a material that never reads them.
+        //
+        // The colour lift is the part that actually does harm. It exists to
+        // compensate for clearcoat darkening the perceived iris; applied to an
+        // unlit material it is simply a 25% brightening of the baked eye map, and
+        // washes out the very colour it was meant to protect. An export that
+        // already decided eyes are unlit has made this choice deliberately.
+        if (!('roughness' in m))
+            return;
+        m.metalness = 0;
+        // Wet, glossy sphere — this is what produces a catchlight at all.
+        m.roughness = 0.12;
+        if ('clearcoat' in m) {
+            // The dome over the iris: a second, sharper highlight that reads as
+            // cornea rather than polished plastic.
+            m.clearcoat = 1;
+            m.clearcoatRoughness = 0.02;
+        }
+        if ('specularIntensity' in m)
+            m.specularIntensity = 1.5;
+        m.envMapIntensity = 1.6;
+        // The baked eye map darkens the iris; lift it so the colour stays readable
+        // under the gloss.
+        m.color?.setRGB(1.25, 1.25, 1.25);
+        found.eyes++;
+    }
+    else if (/^Eyeocclusion/i.test(name)) {
+        m.metalness = 0;
+        m.roughness = 1;
+        // Ships white at full lighting response, which flattens the socket.
+        // Darkened and unlit it beds the eyeball into the face.
+        m.color?.setRGB(0.5, 0.45, 0.45);
+        m.envMapIntensity = 0;
+        found.occlusion++;
+    }
+    else if (/^Teeth/i.test(name)) {
+        m.metalness = 0;
+        // Enamel, slightly worn — not white, and emphatically not black.
+        //
+        // The original drove this to 0.045 trying to stop the teeth reading as a
+        // bright slab. It worked, and then read as a hole in the face. Albedo was
+        // the wrong tool: the teeth looked wrong because they did not MOVE with
+        // the jaw, not because they were the wrong colour. That is fixed properly
+        // in the lipsync layer, which is why this can go back to being a colour.
+        m.roughness = 0.55;
+        m.color?.setRGB(0.46 * teethScale, 0.45 * teethScale, 0.43 * teethScale);
+        m.envMapIntensity = 0.1;
+        if ('specularIntensity' in m)
+            m.specularIntensity = 0.2;
+        found.teeth++;
+    }
+    else {
+        return;
+    }
+    m.needsUpdate = true;
+}
+/** How many meshes under `root` reference this exact material instance. */
+function referenceCount(root, target) {
+    let n = 0;
+    root.traverse((obj) => {
+        const mesh = obj;
+        if (!mesh.isMesh || !mesh.material)
+            return;
+        const mats = Array.isArray(mesh.material) ? mesh.material : [mesh.material];
+        for (const m of mats)
+            if (m === target)
+                n++;
+    });
+    return n;
+}
+/**
+ * Apply the face pass. Synchronous for the material work; the teeth texture
+ * resolves on its own and lands a frame or two later.
+ *
+ * Deliberately not awaited by the caller: the teeth are correctly shaped and
+ * correctly coloured in the meantime, they simply have no tooth separation
+ * until the image decodes.
+ */
+export function applyFaceMaterials(root, tuning = {}, parser) {
+    const teethScale = tuning.teethScale ?? 1;
+    const found = { eyes: 0, teeth: 0, occlusion: 0 };
+    /** Every teeth material on THIS model, for the map to land on once loaded. */
+    const mouths = [];
+    root.traverse((obj) => {
+        const mesh = obj;
+        if (!mesh.isMesh || !mesh.material)
+            return;
+        const mats = Array.isArray(mesh.material) ? mesh.material : [mesh.material];
+        for (const m of mats)
+            tune(m, teethScale, found);
+        const teeth = mats.find((mm) => /^Teeth/i.test(mm.name || ''));
+        if (!teeth)
+            return;
+        // A LIT teeth material is the odd one out: nothing in a scene occludes a
+        // mouth, so a character standing in sun gets bright teeth and one in shade
+        // gets dark ones from the same material. Unlit makes every mouth read the
+        // same wherever the character stands. The TEXTURE is what still separates
+        // one tooth from another — which is why the rescue above matters, and why
+        // making them unlit without it produces a flat grey shape.
+        if (!teeth.isMeshBasicMaterial) {
+            const flat = new THREE.MeshBasicMaterial({
+                name: 'Teeth',
+                color: teeth.color.clone(),
+                side: teeth.side,
+                map: teeth.map ?? null,
+            });
+            // Replace only the teeth SLOT. A head mesh routinely carries several
+            // materials — skin, teeth, tongue — and assigning `mesh.material = flat`
+            // wholesale would collapse that array to one entry and leave every other
+            // group rendering with the teeth material. Found by a test written for a
+            // different reason.
+            if (Array.isArray(mesh.material)) {
+                mesh.material = mesh.material.map((m) => (m === teeth ? flat : m));
+            }
+            else {
+                mesh.material = flat;
+            }
+            // The original is now orphaned: dispose() walks the graph and would only
+            // ever find the replacement. Dispose it here — but only if nothing else
+            // under this root still points at it, since one material instance can be
+            // shared across several meshes.
+            if (referenceCount(root, teeth) === 0)
+                teeth.dispose();
+            mouths.push(flat);
+        }
+        else {
+            mouths.push(teeth);
+        }
+    });
+    if (mouths.length && !mouths.every((m) => m.map)) {
+        void loadTeethTexture(parser).then((tex) => {
+            if (!tex)
+                return;
+            for (const m of mouths) {
+                if (!m.map) {
+                    m.map = tex;
+                    m.needsUpdate = true;
+                }
+            }
+        });
+    }
+    return found;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@convai/embody",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Give a Convai character a body: resolve 3D characters by characterId and drive them with lipsync and animation. Layered on @convai/web-sdk.",
   "license": "Apache-2.0",
   "author": "Convai <support@convai.com>",