@convai/embody 1.0.1 → 1.1.0-beta.0

package/dist/react/PortraitLights.js

@@ -0,0 +1,44 @@
+import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "react/jsx-runtime";
+/**
+ * The light `ConvaiCharacter`'s own canvas puts on a character: a portrait
+ * rig, as the coffee-shop studio lit its films, scaled to the full-body
+ * default framing.
+ *
+ * The old default (a hemisphere at 1.4, an unshadowed key at 2.2, a room
+ * environment at 0.6, ACES) flooded the face from every side: the skin's
+ * wrap and shade, the shadow work and the eyes' catchlights never showed,
+ * and ACES pushed skin orange. This is:
+ *
+ *  · a soft-shadowed key 45° to screen left of the camera and 30° up,
+ *    following the camera round the character (as a photographer moves the
+ *    lights with each shot), never more than 70° off the character's front,
+ *    so a turn to its darker side does not become a black profile;
+ *  · a fill on the other side at about a ninth of the key, a warm rim
+ *    behind, a dim hemisphere and a dim room environment for reflections;
+ *  · Neutral tone mapping at 1.05 (ACES turned skin orange).
+ *
+ * Only for embody's own canvas: an app with its own `<Canvas>` lights it.
+ */
+import { useEffect, useMemo } from 'react';
+import { useFrame, useThree } from '@react-three/fiber';
+import * as THREE from 'three';
+import { createPortraitRig, placePortraitRig, PORTRAIT_TONE } from '../render/portraitRig.js';
+export function PortraitLights({ target }) {
+    const gl = useThree((s) => s.gl);
+    const lights = useMemo(createPortraitRig, []);
+    useEffect(() => {
+        const tone = gl.toneMapping, exposure = gl.toneMappingExposure;
+        gl.toneMapping = PORTRAIT_TONE.toneMapping;
+        gl.toneMappingExposure = PORTRAIT_TONE.exposure;
+        return () => {
+            gl.toneMapping = tone;
+            gl.toneMappingExposure = exposure;
+            lights.key.dispose();
+            lights.fill.dispose();
+            lights.rim.dispose();
+        };
+    }, [gl, lights]);
+    const at = useMemo(() => new THREE.Vector3(...target), [target]);
+    useFrame(({ camera }) => placePortraitRig(lights, camera.position, at));
+    return (_jsxs(_Fragment, { children: [_jsx("primitive", { object: lights.key }), _jsx("primitive", { object: lights.fill }), _jsx("primitive", { object: lights.rim }), _jsx("primitive", { object: lights.sky })] }));
+}

package/dist/render/faceMaterials.d.ts

@@ -36,9 +36,9 @@ import * as THREE from 'three';
  *     character that is still using it, and the failure would appear on an
  *     unrelated character at an unrelated time. Each character gets its own
  *     map or none.
- *  2. Replaced materials are disposed here, not leaked. Swapping a lit teeth
- *     material for an unlit one orphans the original — `dispose()` walks the
- *     scene graph and would only ever see the replacement.
+ *  2. Replaced materials are disposed here, not leaked. Swapping the teeth
+ *     material orphans the original — `dispose()` walks the scene graph and
+ *     would only ever see the replacement.
  */
 /** What this actually needs of `GLTF.parser`, structurally. */
 export type GltfParserLike = {
@@ -65,6 +65,11 @@ export interface FaceMaterialTuning {
      * 1 is the original's tuning and the default, so nothing silently changes.
      */
     teethScale?: number;
+    /**
+     * Brightness of the eye light reflected in each cornea (WebGL), as from a
+     * small panel just above the camera. 0 switches it off. Default 1.2.
+     */
+    catchlight?: number;
 }
 export interface FaceMaterialReport {
     eyes: number;
@@ -75,6 +80,12 @@ export interface FaceMaterialReport {
     /** Hidden `Hide` helper materials. */
     hidden: number;
 }
+/**
+ * An eyeball material: "eye" as a word, camel case included (MetaHuman
+ * `Eyes`, re-exports' `EyeSofia`, Character Creator `Std_Eye_R`, Ready
+ * Player Me `Wolf3D_Eye`), never the films, lashes and shells around it.
+ */
+export declare const isEye: (name: string) => boolean;
 /**
  * Pull this model's teeth albedo out of the file, bound or not.
  *
@@ -96,6 +107,52 @@ export interface FaceMaterialReport {
  * albedo slot, which reads as a lilac mouth rather than as an error.
  */
 export declare function loadTeethTexture(parser: GltfParserLike | undefined): Promise<THREE.Texture | null>;
+/** Iris and pupil radii in an eye texture's UV space, centred at (0.5, 0.5). */
+export interface IrisLayout {
+    iris: number;
+    pupil: number;
+}
+/**
+ * Read the iris and pupil radii off an eye texture, from its radial
+ * brightness: the pupil edge where it rises halfway from the pupil to the
+ * iris body, the iris edge where it rises halfway from the iris body to the
+ * sclera. Measured on the published MetaHumans: 0.047 and 0.155, against the
+ * 0.05 and 0.158 the studio set by eye. Null when the image cannot be read
+ * (no DOM, a tainted canvas) or does not look like a centred eye, in which
+ * case the iris is left as painted.
+ */
+export declare function measureIris(image: CanvasImageSource | undefined | null): IrisLayout | null;
+/**
+ * The eye light (WebGL): a small soft panel just above and to the side of
+ * the camera, as a portrait photographer adds, reflected in the cornea.
+ * Under a dim scene a cornea has nothing bright to reflect and the eyes read
+ * dead; the studio painted a spot on for its films. A reflection of a light
+ * that travels with the view works for any export and any camera, and the
+ * eyes look at the camera, so it lands in the iris.
+ */
+export declare function patchCatchlight(fs: string): string;
+/** The iris treatment on three's map stage. Unchanged without a map stage. */
+export declare function patchIrisShader(fs: string): string;
+/** Pass the mouth's openness (0 inside, 1 at the front teeth) to the fragment. */
+export declare function patchMouthVertex(vs: string): string;
+/** Rough and without specular where the mouth hides a surface. */
+export declare function patchMouthFragment(fs: string): string;
+/**
+ * How much light reaches each vertex of the mouth, from where it sits in the
+ * bind pose, as vertex colours (every renderer) and `embodyMouthOpen` (the
+ * WebGL specular mask). Measured in the character's own frame (glTF: +Z
+ * forward, +Y up), so it holds for any export:
+ *
+ *  · depth: the mouth falls to near black within the first centimetre behind
+ *    the front teeth (to 0.06, not 0, so a small opening is not a black slit);
+ *  · height: the lower teeth and gums sit in the lower lip's shade, at half
+ *    the upper ones;
+ *  · side: the side teeth sit in the cheeks' shade (the buccal corridor);
+ *    lit, they flashed white at the mouth corners.
+ *
+ * Values are the coffee-shop studio's, matched to Convai's Unreal references.
+ */
+export declare function mouthOcclusion(mesh: THREE.Mesh, slot?: number): void;
 /**
  * Apply the face pass. Synchronous for the material work; the teeth texture
  * resolves on its own and lands a frame or two later.

package/dist/render/faceMaterials.js

@@ -1,5 +1,10 @@
 import * as THREE from 'three';
-const isEye = (name) => /^Eyes?(?!occlusion)/i.test(name);
+/**
+ * An eyeball material: "eye" as a word, camel case included (MetaHuman
+ * `Eyes`, re-exports' `EyeSofia`, Character Creator `Std_Eye_R`, Ready
+ * Player Me `Wolf3D_Eye`), never the films, lashes and shells around it.
+ */
+export const isEye = (name) => /(?:^|[^a-z])eyes?(?:[^a-z]|$)/i.test(name.replace(/([a-z])([A-Z])/g, '$1_$2')) && !/occlusion|lash|brow|tear|cornea|shell|socket|lid/i.test(name);
 /**
  * Repair what the convai-mha-1 Blender exports get wrong, matched on the
  * defect rather than the material name wherever the defect is detectable, so
@@ -36,17 +41,7 @@ function repairExport(mat, found) {
         const sameImage = !!m.map && (m.map === emissiveMap || m.map.source === emissiveMap.source);
         const lit = emissive.r + emissive.g + emissive.b > 0;
         if (isEye(name)) {
-            // Unlit eyes are a deliberate export choice (see `tune`); lit ones get a
-            // floor of their own albedo.
-            if ('roughness' in m) {
-                if (!lit)
-                    emissive.setScalar(0.25);
-                else
-                    emissive.multiplyScalar(0.25);
-                if ('emissiveIntensity' in m)
-                    m.emissiveIntensity = 1;
-                changed = true;
-            }
+            // Replaced by a lit eye with no emissive (see eyeMaterial).
         }
         else if (sameImage && lit) {
             m.emissiveMap = null;
@@ -55,11 +50,6 @@ function repairExport(mat, found) {
             changed = true;
         }
     }
-    else if (isEye(name) && emissive && m.map && 'roughness' in m) {
-        m.emissiveMap = m.map;
-        emissive.setScalar(0.25);
-        changed = true;
-    }
     if (typeof m.roughness === 'number' && m.roughness > 1) {
         m.roughness = 1;
         changed = true;
@@ -72,7 +62,7 @@ function repairExport(mat, found) {
     if (/^(Base)?Head/i.test(name) && 'roughness' in m) {
         if (m.roughnessMap) {
             m.roughnessMap = null;
-            m.roughness = 0.62;
+            m.roughness = 0.52;
         }
         if ('specularIntensity' in m)
             m.specularIntensity = 0.35;
@@ -144,67 +134,299 @@ function tune(mat, teethScale, found) {
     const m = mat;
     const name = m.name || '';
     if (isEye(name)) {
-        // ONLY on a material that has a lighting model.
-        //
-        // Measured on the live convai-mha-1 body: its eye materials arrive as
-        // MeshBasicMaterial — already unlit. Gloss cannot exist there, so
-        // `roughness`, `clearcoat` and `specularIntensity` would be inert
-        // properties bolted onto a material that never reads them.
-        //
-        // The colour lift is the part that actually does harm. It exists to
-        // compensate for clearcoat darkening the perceived iris; applied to an
-        // unlit material it is simply a 25% brightening of the baked eye map, and
-        // washes out the very colour it was meant to protect. An export that
-        // already decided eyes are unlit has made this choice deliberately.
-        if (!('roughness' in m))
-            return;
-        m.metalness = 0;
-        // Wet, glossy sphere — this is what produces a catchlight at all.
-        m.roughness = 0.12;
-        if ('clearcoat' in m) {
-            // The dome over the iris: a second, sharper highlight that reads as
-            // cornea rather than polished plastic.
-            m.clearcoat = 1;
-            m.clearcoatRoughness = 0.02;
-        }
-        if ('specularIntensity' in m)
-            m.specularIntensity = 1.5;
-        m.envMapIntensity = 1.6;
-        // The baked eye map darkens the iris; lift it so the colour stays readable
-        // under the gloss.
-        m.color?.setRGB(1.25, 1.25, 1.25);
+        // Replaced wholesale in applyFaceMaterials (lit, with a cornea and a
+        // measured iris).
         found.eyes++;
+        return;
     }
     else if (/^Eyeocclusion/i.test(name)) {
         m.metalness = 0;
         m.roughness = 1;
-        // Ships white at full lighting response, which flattens the socket.
-        // Darkened and unlit it beds the eyeball into the face.
-        m.color?.setRGB(0.5, 0.45, 0.45);
+        // A film that may only ever darken: in MetaHuman it is the socket's
+        // shadow, its texture's alpha saying where. Shipped white at full
+        // lighting response it flattened the socket; at the old 0.5 grey it still
+        // reflected a hard key and drew a pale rim along the upper lash line
+        // (seen under a top spot). Black, with no specular, its alpha only shades.
+        m.color?.setRGB(0, 0, 0);
+        // Strong enough to carry the upper lid's shadow on the iris: at 0.6 the
+        // eyes read flat, pasted on (the studio's).
+        if (m.transparent)
+            m.opacity = 0.8;
         m.envMapIntensity = 0;
+        if ('specularIntensity' in m)
+            m.specularIntensity = 0;
         found.occlusion++;
     }
     else if (/^Teeth/i.test(name)) {
-        m.metalness = 0;
-        // Enamel, slightly worn — not white, and emphatically not black.
-        //
-        // The original drove this to 0.045 trying to stop the teeth reading as a
-        // bright slab. It worked, and then read as a hole in the face. Albedo was
-        // the wrong tool: the teeth looked wrong because they did not MOVE with
-        // the jaw, not because they were the wrong colour. That is fixed properly
-        // in the lipsync layer, which is why this can go back to being a colour.
-        m.roughness = 0.55;
-        m.color?.setRGB(0.46 * teethScale, 0.45 * teethScale, 0.43 * teethScale);
-        m.envMapIntensity = 0.1;
-        if ('specularIntensity' in m)
-            m.specularIntensity = 0.2;
+        // Replaced wholesale in applyFaceMaterials (lit, with mouth occlusion).
         found.teeth++;
+        return;
     }
     else {
         return;
     }
     m.needsUpdate = true;
 }
+/**
+ * Read the iris and pupil radii off an eye texture, from its radial
+ * brightness: the pupil edge where it rises halfway from the pupil to the
+ * iris body, the iris edge where it rises halfway from the iris body to the
+ * sclera. Measured on the published MetaHumans: 0.047 and 0.155, against the
+ * 0.05 and 0.158 the studio set by eye. Null when the image cannot be read
+ * (no DOM, a tainted canvas) or does not look like a centred eye, in which
+ * case the iris is left as painted.
+ */
+export function measureIris(image) {
+    if (!image || typeof document === 'undefined')
+        return null;
+    try {
+        const N = 256;
+        const canvas = document.createElement('canvas');
+        canvas.width = canvas.height = N;
+        const ctx = canvas.getContext('2d', { willReadFrequently: true });
+        if (!ctx)
+            return null;
+        ctx.drawImage(image, 0, 0, N, N);
+        const d = ctx.getImageData(0, 0, N, N).data;
+        const BINS = 64;
+        const sum = new Float32Array(BINS), count = new Float32Array(BINS);
+        for (let j = 0; j < N; j++)
+            for (let i = 0; i < N; i++) {
+                const r = Math.hypot((i + 0.5) / N - 0.5, (j + 0.5) / N - 0.5);
+                const b = Math.floor((r / 0.5) * BINS);
+                if (b >= BINS)
+                    continue;
+                const k = (j * N + i) * 4;
+                sum[b] += (0.2126 * d[k] + 0.7152 * d[k + 1] + 0.0722 * d[k + 2]) / 255;
+                count[b]++;
+            }
+        const L = Array.from(sum, (v, i) => v / (count[i] || 1));
+        const at = (r) => L[Math.min(BINS - 1, Math.floor((r / 0.5) * BINS))];
+        const radius = (b) => ((b + 0.5) / BINS) * 0.5;
+        const sclera = Math.max(...L);
+        const body = at(0.1);
+        const firstAbove = (from, level) => {
+            for (let b = Math.floor((from / 0.5) * BINS); b < BINS; b++)
+                if (L[b] >= level)
+                    return radius(b);
+            return -1;
+        };
+        const iris = firstAbove(0.05, (body + sclera) / 2);
+        const pupil = firstAbove(0, (L[0] + body) / 2);
+        if (!(sclera - body > 0.15 && iris > 0.06 && iris < 0.3 && pupil > 0.01 && pupil < iris * 0.7))
+            return null;
+        return { iris, pupil };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * A living eye under any light (the coffee-shop studio's):
+ *  · lit, with a wet sclera (roughness 0.32) under a cornea coat at 0.1:
+ *    not mirror-glassy, since at viewing distance a sharper cornea shrinks
+ *    every highlight below a pixel and the eyes read dead;
+ *  · faint reflections (specular and environment 0.25): reflected light
+ *    ignores shadows, and at full strength the eye on the shadow side glowed
+ *    as bright as the lit one;
+ *  · no emissive: the old albedo glow lifted the pupil to grey;
+ *  · on WebGL, with a measured iris: its colour enriched (saturation 1.4,
+ *    contrast), a darker limbal ring and a near-black pupil, where exports
+ *    paint a desaturated grey ring with a soft edge.
+ */
+function eyeMaterial(src, catchlight) {
+    const m = new THREE.MeshPhysicalMaterial({
+        name: src.name,
+        side: src.side,
+        map: src.map ?? null,
+        normalMap: src.normalMap ?? null,
+        metalness: 0,
+        roughness: 0.32,
+        clearcoat: 1,
+        clearcoatRoughness: 0.1,
+        specularIntensity: 0.25,
+        envMapIntensity: 0.25,
+    });
+    if (src.normalScale)
+        m.normalScale.copy(src.normalScale);
+    m.userData.embodyEye = true;
+    const layout = m.map ? measureIris(src.map?.image) : null;
+    if (layout)
+        m.userData.embodyIris = layout;
+    const uniforms = {
+        embodyIris: { value: layout?.iris ?? 0 },
+        embodyPupil: { value: layout?.pupil ?? 0 },
+        embodyCatchlight: { value: catchlight },
+    };
+    m.userData.embodyEyeUniforms = uniforms;
+    m.onBeforeCompile = (shader) => {
+        Object.assign(shader.uniforms, uniforms);
+        if (layout)
+            shader.fragmentShader = patchIrisShader(shader.fragmentShader);
+        shader.fragmentShader = patchCatchlight(shader.fragmentShader);
+    };
+    m.customProgramCacheKey = () => `embody-eye-${layout ? 1 : 0}`;
+    return m;
+}
+/**
+ * The eye light (WebGL): a small soft panel just above and to the side of
+ * the camera, as a portrait photographer adds, reflected in the cornea.
+ * Under a dim scene a cornea has nothing bright to reflect and the eyes read
+ * dead; the studio painted a spot on for its films. A reflection of a light
+ * that travels with the view works for any export and any camera, and the
+ * eyes look at the camera, so it lands in the iris.
+ */
+export function patchCatchlight(fs) {
+    if (!fs.includes('#include <opaque_fragment>'))
+        return fs;
+    return fs
+        .replace('void main() {', 'uniform float embodyCatchlight;\nvoid main() {')
+        .replace('#include <opaque_fragment>', `{
+		// A panel ~10° above and to one side of the lens: up and to one side of
+		// the pupil. Further out it fell under the upper lid (the open eye
+		// shows only the front ~25° of the ball).
+		vec3 embodyH = normalize( normalize( vec3( -0.1, 0.13, 1.0 ) ) + vec3( 0.0, 0.0, 1.0 ) );
+		float embodySpot = pow( max( dot( normalize( normal ), embodyH ), 0.0 ), 120.0 );
+		outgoingLight += vec3( 1.0, 0.97, 0.92 ) * smoothstep( 0.3, 0.7, embodySpot ) * embodyCatchlight;
+	}
+	#include <opaque_fragment>`);
+}
+/** The iris treatment on three's map stage. Unchanged without a map stage. */
+export function patchIrisShader(fs) {
+    if (!fs.includes('#include <map_fragment>'))
+        return fs;
+    return fs
+        .replace('void main() {', 'uniform float embodyIris;\nuniform float embodyPupil;\nvoid main() {')
+        .replace('#include <map_fragment>', `#include <map_fragment>
+	#ifdef USE_MAP
+	{
+		float r = length( vMapUv - 0.5 );
+		float iris = 1.0 - smoothstep( embodyIris - 0.008, embodyIris + 0.006, r );
+		float ring = smoothstep( embodyIris * 0.7, embodyIris, r ) * iris;
+		float pupil = 1.0 - smoothstep( embodyPupil - 0.006, embodyPupil + 0.004, r );
+		vec3 c = diffuseColor.rgb;
+		float lum = dot( c, vec3( 0.2126, 0.7152, 0.0722 ) );
+		vec3 rich = pow( max( mix( vec3( lum ), c, 1.4 ), vec3( 0.0 ) ), vec3( 1.2 ) ) * 1.1;
+		c = mix( c, rich, iris );
+		c *= 1.0 - ring * 0.45;
+		c *= 1.0 - pupil * 0.92;
+		diffuseColor.rgb = c;
+	}
+	#endif`);
+}
+/** Enamel under the mouth's light: worn off-white, well under skin brightness. */
+const ENAMEL = new THREE.Color(0.93, 0.9, 0.86).multiplyScalar(0.7);
+/**
+ * The lit teeth material. Front faces only: double-sided, the backs of the
+ * gums and tongue faced the camera through an open mouth and caught the key
+ * as a grey sheet. No environment reflection (there is no sky in a mouth);
+ * specular and roughness follow the mouth occlusion on WebGL, so only the
+ * front teeth and tongue tip keep a little wet shine and the inner cheeks
+ * stop flashing white at grazing angles.
+ */
+function teethMaterial(src, teethScale) {
+    const m = new THREE.MeshPhysicalMaterial({
+        name: src.name || 'Teeth',
+        side: THREE.FrontSide,
+        map: src.map ?? null,
+        color: ENAMEL.clone().multiplyScalar(teethScale),
+        metalness: 0,
+        roughness: 0.5,
+        specularIntensity: 0.42,
+        specularColor: new THREE.Color(1, 0.85, 0.82),
+        envMapIntensity: 0,
+        vertexColors: true,
+    });
+    m.userData.embodyTeeth = true;
+    m.onBeforeCompile = (shader) => {
+        shader.vertexShader = patchMouthVertex(shader.vertexShader);
+        shader.fragmentShader = patchMouthFragment(shader.fragmentShader);
+    };
+    m.customProgramCacheKey = () => 'embody-teeth';
+    return m;
+}
+/** Pass the mouth's openness (0 inside, 1 at the front teeth) to the fragment. */
+export function patchMouthVertex(vs) {
+    if (!vs.includes('#include <begin_vertex>'))
+        return vs;
+    return vs
+        .replace('void main() {', 'attribute float embodyMouthOpen;\nvarying float vEmbodyMouthOpen;\nvoid main() {')
+        .replace('#include <begin_vertex>', '#include <begin_vertex>\n\tvEmbodyMouthOpen = embodyMouthOpen;');
+}
+/** Rough and without specular where the mouth hides a surface. */
+export function patchMouthFragment(fs) {
+    if (!fs.includes('#include <lights_physical_fragment>') || !fs.includes('#include <roughnessmap_fragment>'))
+        return fs;
+    return fs
+        .replace('void main() {', 'varying float vEmbodyMouthOpen;\nvoid main() {')
+        .replace('#include <roughnessmap_fragment>', '#include <roughnessmap_fragment>\n\troughnessFactor = mix( 1.0, roughnessFactor, vEmbodyMouthOpen * vEmbodyMouthOpen );')
+        .replace('#include <lights_physical_fragment>', '#include <lights_physical_fragment>\n\tmaterial.specularColor *= vEmbodyMouthOpen;\n\tmaterial.specularF90 *= vEmbodyMouthOpen;');
+}
+const smooth = THREE.MathUtils.smoothstep;
+/**
+ * How much light reaches each vertex of the mouth, from where it sits in the
+ * bind pose, as vertex colours (every renderer) and `embodyMouthOpen` (the
+ * WebGL specular mask). Measured in the character's own frame (glTF: +Z
+ * forward, +Y up), so it holds for any export:
+ *
+ *  · depth: the mouth falls to near black within the first centimetre behind
+ *    the front teeth (to 0.06, not 0, so a small opening is not a black slit);
+ *  · height: the lower teeth and gums sit in the lower lip's shade, at half
+ *    the upper ones;
+ *  · side: the side teeth sit in the cheeks' shade (the buccal corridor);
+ *    lit, they flashed white at the mouth corners.
+ *
+ * Values are the coffee-shop studio's, matched to Convai's Unreal references.
+ */
+export function mouthOcclusion(mesh, slot = 0) {
+    const g = mesh.geometry;
+    if (g.userData.embodyMouth || !g.attributes.position)
+        return;
+    mesh.updateWorldMatrix(true, false);
+    const n = g.attributes.position.count;
+    // On a multi-material mesh, only the mouth's own triangles set the ranges.
+    const mine = new Uint8Array(n);
+    const groups = Array.isArray(mesh.material) ? g.groups.filter((gr) => gr.materialIndex === slot) : [];
+    if (groups.length) {
+        for (const gr of groups)
+            for (let k = gr.start; k < gr.start + gr.count; k++)
+                mine[g.index ? g.index.getX(k) : k] = 1;
+    }
+    else
+        mine.fill(1);
+    const p = new THREE.Vector3();
+    const xs = new Float32Array(n), ys = new Float32Array(n), zs = new Float32Array(n);
+    for (let i = 0; i < n; i++) {
+        const v = mesh.isSkinnedMesh ? mesh.getVertexPosition(i, p) : p.fromBufferAttribute(g.attributes.position, i);
+        v.applyMatrix4(mesh.matrixWorld);
+        xs[i] = v.x;
+        ys[i] = v.y;
+        zs[i] = v.z;
+    }
+    const unit = (a) => {
+        let lo = Infinity, hi = -Infinity;
+        a.forEach((x, i) => { if (mine[i]) {
+            lo = Math.min(lo, x);
+            hi = Math.max(hi, x);
+        } });
+        return (x) => (x - lo) / (hi - lo || 1);
+    };
+    const ux = unit(xs), uy = unit(ys), uz = unit(zs);
+    const color = new Float32Array(n * 3);
+    const open = new Float32Array(n);
+    for (let i = 0; i < n; i++) {
+        const front = smooth(uz(zs[i]), 0.5, 0.97);
+        const cavity = 0.06 + 0.94 * front * front * front;
+        const upper = 0.5 + 0.5 * smooth(uy(ys[i]), 0.42, 0.6);
+        const corridor = 1 - 0.93 * smooth(Math.abs(ux(xs[i]) * 2 - 1), 0.18, 0.5);
+        const light = cavity * upper * corridor;
+        color[i * 3] = color[i * 3 + 1] = color[i * 3 + 2] = light;
+        open[i] = cavity * corridor;
+    }
+    g.setAttribute('color', new THREE.BufferAttribute(color, 3));
+    g.setAttribute('embodyMouthOpen', new THREE.BufferAttribute(open, 1));
+    g.userData.embodyMouth = true;
+}
 /** How many meshes under `root` reference this exact material instance. */
 function referenceCount(root, target) {
     let n = 0;
@@ -241,44 +463,46 @@ export function applyFaceMaterials(root, tuning = {}, parser) {
             repairExport(m, found);
             tune(m, teethScale, found);
         }
-        const teeth = mats.find((mm) => /^Teeth/i.test(mm.name || ''));
+        for (const eye of mats.filter((mm) => isEye(mm.name || '') && !mm.userData.embodyEye)) {
+            const lit = eyeMaterial(eye, tuning.catchlight ?? 1.2);
+            if (mesh.geometry.attributes.position && !mesh.geometry.attributes.normal)
+                mesh.geometry.computeVertexNormals();
+            mesh.material = Array.isArray(mesh.material) ? mesh.material.map((m) => (m === eye ? lit : m)) : lit;
+            if (referenceCount(root, eye) === 0)
+                eye.dispose();
+        }
+        const teeth = mats.find((mm) => /^Teeth/i.test(mm.name || '') && !mm.userData.embodyTeeth);
         if (!teeth)
             return;
-        // A LIT teeth material is the odd one out: nothing in a scene occludes a
-        // mouth, so a character standing in sun gets bright teeth and one in shade
-        // gets dark ones from the same material. Unlit makes every mouth read the
-        // same wherever the character stands. The TEXTURE is what still separates
-        // one tooth from another — which is why the rescue above matters, and why
-        // making them unlit without it produces a flat grey shape.
-        if (!teeth.isMeshBasicMaterial) {
-            const flat = new THREE.MeshBasicMaterial({
-                name: 'Teeth',
-                color: teeth.color.clone(),
-                side: teeth.side,
-                map: teeth.map ?? null,
-            });
-            // Replace only the teeth SLOT. A head mesh routinely carries several
-            // materials — skin, teeth, tongue — and assigning `mesh.material = flat`
-            // wholesale would collapse that array to one entry and leave every other
-            // group rendering with the teeth material. Found by a test written for a
-            // different reason.
-            if (Array.isArray(mesh.material)) {
-                mesh.material = mesh.material.map((m) => (m === teeth ? flat : m));
-            }
-            else {
-                mesh.material = flat;
-            }
-            // The original is now orphaned: dispose() walks the graph and would only
-            // ever find the replacement. Dispose it here — but only if nothing else
-            // under this root still points at it, since one material instance can be
-            // shared across several meshes.
-            if (referenceCount(root, teeth) === 0)
-                teeth.dispose();
-            mouths.push(flat);
+        // Lit, as the rest of the face, and shaded by the mouth around it (see
+        // `mouthOcclusion`). Unlit teeth (the export's own choice on some bodies,
+        // and this pass's old one) glowed grey in a dark mouth: nothing in a
+        // scene's lighting reaches inside a mouth, so the light has to be taken
+        // away by where each surface sits.
+        const lit = teethMaterial(teeth, teethScale);
+        // Exports that ship the teeth unlit ship them without normals too (the
+        // published MetaHumans do), and lit without normals they render black.
+        if (mesh.geometry.attributes.position && !mesh.geometry.attributes.normal)
+            mesh.geometry.computeVertexNormals();
+        mouthOcclusion(mesh, mats.indexOf(teeth));
+        // Replace only the teeth SLOT. A head mesh routinely carries several
+        // materials — skin, teeth, tongue — and assigning `mesh.material = lit`
+        // wholesale would collapse that array to one entry and leave every other
+        // group rendering with the teeth material. Found by a test written for a
+        // different reason.
+        if (Array.isArray(mesh.material)) {
+            mesh.material = mesh.material.map((m) => (m === teeth ? lit : m));
         }
         else {
-            mouths.push(teeth);
+            mesh.material = lit;
         }
+        // The original is now orphaned: dispose() walks the graph and would only
+        // ever find the replacement. Dispose it here — but only if nothing else
+        // under this root still points at it, since one material instance can be
+        // shared across several meshes.
+        if (referenceCount(root, teeth) === 0)
+            teeth.dispose();
+        mouths.push(lit);
     });
     if (mouths.length && !mouths.every((m) => m.map)) {
         void loadTeethTexture(parser).then((tex) => {

package/dist/render/portraitRig.d.ts

@@ -0,0 +1,28 @@
+/**
+ * A portrait light rig for a character, framework-free: what
+ * `ConvaiCharacter` lights its own canvas with, for apps on plain three.js.
+ * After the coffee-shop studio's film rig: a soft-shadowed key 45° to screen
+ * left of the camera and 30° up, following the camera round the character
+ * and never more than 70° off its front (a turn to its darker side is not a
+ * black profile); a fill at about a ninth of the key, a warm rim, a dim sky.
+ * Pair it with `PORTRAIT_TONE` (ACES pushed skin orange) and a dim
+ * environment, and call `placePortraitRig` every frame.
+ */
+import * as THREE from 'three';
+/** Tone mapping the rig was tuned under. */
+export declare const PORTRAIT_TONE: {
+    readonly toneMapping: 7;
+    readonly exposure: 1.05;
+};
+/** Where the key goes for a camera at `camAz` degrees round the character. */
+export declare function keyAzimuth(camAz: number): number;
+export interface PortraitRig {
+    key: THREE.DirectionalLight;
+    fill: THREE.DirectionalLight;
+    rim: THREE.DirectionalLight;
+    sky: THREE.HemisphereLight;
+}
+/** The rig's lights: a soft-shadowed key, a fill, a warm rim, a dim sky. */
+export declare function createPortraitRig(): PortraitRig;
+/** Place the rig for a camera at `camera`, lighting the point `at`. */
+export declare function placePortraitRig(rig: PortraitRig, camera: THREE.Vector3, at: THREE.Vector3): void;