@convai/embody 1.0.1 → 1.1.0-beta.0

package/dist/render/portraitRig.js

@@ -0,0 +1,55 @@
+/**
+ * A portrait light rig for a character, framework-free: what
+ * `ConvaiCharacter` lights its own canvas with, for apps on plain three.js.
+ * After the coffee-shop studio's film rig: a soft-shadowed key 45° to screen
+ * left of the camera and 30° up, following the camera round the character
+ * and never more than 70° off its front (a turn to its darker side is not a
+ * black profile); a fill at about a ninth of the key, a warm rim, a dim sky.
+ * Pair it with `PORTRAIT_TONE` (ACES pushed skin orange) and a dim
+ * environment, and call `placePortraitRig` every frame.
+ */
+import * as THREE from 'three';
+/** Tone mapping the rig was tuned under. */
+export const PORTRAIT_TONE = { toneMapping: THREE.NeutralToneMapping, exposure: 1.05 };
+/** Where the key sits from the lens, and how far from the character's front it may go, degrees. */
+const KEY_FROM_LENS = -45;
+const KEY_LIMIT = 70;
+/** Unit direction at an azimuth (from +Z, the character's front) and elevation, degrees. */
+const toward = (az, el, out) => {
+    const a = THREE.MathUtils.degToRad(az), e = THREE.MathUtils.degToRad(el);
+    return out.set(Math.sin(a) * Math.cos(e), Math.sin(e), Math.cos(a) * Math.cos(e));
+};
+/** Where the key goes for a camera at `camAz` degrees round the character. */
+export function keyAzimuth(camAz) {
+    return THREE.MathUtils.clamp(camAz + KEY_FROM_LENS, -KEY_LIMIT, KEY_LIMIT);
+}
+/** The rig's lights: a soft-shadowed key, a fill, a warm rim, a dim sky. */
+export function createPortraitRig() {
+    const key = new THREE.DirectionalLight(0xfff4ea, 3.2);
+    key.castShadow = true;
+    key.shadow.mapSize.set(2048, 2048);
+    const cam = key.shadow.camera;
+    cam.left = cam.bottom = -1.1;
+    cam.right = cam.top = 1.1;
+    cam.near = 0.1;
+    cam.far = 10;
+    key.shadow.bias = -0.0002;
+    key.shadow.normalBias = 0.003;
+    key.shadow.radius = 3;
+    const fill = new THREE.DirectionalLight(0xe8ecff, 0.36);
+    const rim = new THREE.DirectionalLight(0xffd8b4, 1.6);
+    const sky = new THREE.HemisphereLight(0xffffff, 0x3a3530, 0.25);
+    return { key, fill, rim, sky };
+}
+const dir = new THREE.Vector3();
+/** Place the rig for a camera at `camera`, lighting the point `at`. */
+export function placePortraitRig(rig, camera, at) {
+    const keyAz = keyAzimuth(THREE.MathUtils.radToDeg(Math.atan2(camera.x - at.x, camera.z - at.z)));
+    rig.key.position.copy(at).addScaledVector(toward(keyAz, 30, dir), 4);
+    rig.fill.position.copy(at).addScaledVector(toward(keyAz - 115, 20, dir), 4);
+    rig.rim.position.copy(at).addScaledVector(toward(keyAz + 150, 35, dir), 4);
+    for (const l of [rig.key, rig.fill, rig.rim]) {
+        l.target.position.copy(at);
+        l.target.updateMatrixWorld();
+    }
+}

package/dist/render/surfaces.d.ts

@@ -0,0 +1,181 @@
+/**
+ * Surface shading for skin and hair — how the body takes whatever light the
+ * app puts it in.
+ *
+ * Lighting is the app's. What embody owns is that the materials respond to
+ * it the way skin and hair do, under a portrait key, noon sun, a neon club or
+ * an overcast sky alike. The convai-mha-1 exports ship neither: the face is a
+ * plain standard material (metalness map and all) whose light/shadow edge cuts
+ * straight from lit to black, and the hair is roughness 1, IOR 1.1 chalk drawn
+ * as a depthless blended shell over hard-cut cards.
+ *
+ * What changes, and on which renderer:
+ *
+ *   EVERY RENDERER (material properties three's WebGL and WebGPU renderers
+ *   both honour):
+ *   · skin → MeshPhysicalMaterial: skin's IOR (1.4) and specular, no metal,
+ *     and a faint warm sheen at grazing angles where light scatters out;
+ *   · hair cards and lashes → alpha-to-coverage in the opaque pass: they write
+ *     depth (so they sort and take shadows) and MSAA softens their edges; the
+ *     export's blended hair layer stays blended, after the cards, as the soft
+ *     edge; hair gets a broad, soft keratin sheen in its own colour;
+ *   · the scalp under the hair is shaded as skin in the hair's shade (vertex
+ *     colours), so partings and the gaps between cards don't show bare skin;
+ *   · every texture is sampled anisotropically.
+ *
+ *   WEBGL ONLY (a shader patch WebGPU ignores, leaving the above):
+ *   · light wraps past the terminator as a warm band, as it does through skin
+ *     (Green, "Real-time approximations to subsurface scattering", GPU Gems 1
+ *     ch.16), so the edge is soft instead of clay;
+ *   · thin tissue (ears, nostril wings) glows when lit from behind
+ *     (Barré-Brisebois & Bouchard, GDC 2011);
+ *   · a fine pore normal over the face's own normal map — the published face
+ *     textures are 1024², too coarse to hold pores under a hard key.
+ *
+ * Values are the ones the coffee-shop studio arrived at by comparing renders
+ * against Unreal MetaHuman references under moving light.
+ */
+import * as THREE from 'three';
+export interface SkinTuning {
+    /** Strength of the light that wraps past the terminator, 0–1. Default 0.45. */
+    subsurface?: number;
+    /** How far past the terminator light reaches (wrap), 0–1. Default 0.65. */
+    wrap?: number;
+    /** Backlit glow through thin tissue, 0–1. Default 0.3. */
+    translucency?: number;
+    /** Pore detail strength, 0 to disable. Default 0.16. */
+    detail?: number;
+    /** Pore detail repeats across the face texture. Default 30. */
+    detailTile?: number;
+    /**
+     * Light skin keeps in shadow, linear RGB: warm, as skin in shade glows
+     * from light carried under it, and no more saturated than that
+     * (on darker skin a strong red shade read as a bruise). Default [0.12, 0.07, 0.055].
+     */
+    shadowFloor?: [number, number, number];
+    /** Narrowest a shadow edge on skin is filtered, in shadow-map texels (PCF). Default 3. */
+    shadowSoftness?: number;
+    /** Roughness, where the skin has no roughness map. Default 0.52; 0.48 on a male record. */
+    roughness?: number;
+    /** Sheen strength at grazing angles. Default 0.06; 0.04 on a male record. */
+    sheen?: number;
+    /** Sheen colour, linear RGB. Default a warm red; a less pink tone on a male record. */
+    sheenColor?: [number, number, number];
+    /** Specular intensity. Default 0.6; 0.5 on a male record. */
+    specular?: number;
+    /**
+     * Multiplier on the skin's base colour, linear RGB. Default none; a slight
+     * cool one on a male record (the published male albedo runs redder than a
+     * Reallusion reference head: R/B 2.0 against 1.87, measured on skin).
+     */
+    tint?: [number, number, number];
+}
+export interface SurfaceTuning {
+    /** `false` leaves skin materials as shipped. */
+    skin?: boolean | SkinTuning;
+    /** `false` leaves hair, brow and lash materials as shipped. */
+    hair?: boolean;
+    /** `false` leaves the scalp under the hair unshaded. */
+    scalp?: boolean;
+    /** Texture anisotropy. Clamped by the GPU. Default 16. */
+    anisotropy?: number;
+    /**
+     * The avatar record's gender, when the resolver gave one. The published
+     * bodies ship lashes with an alpha multiplier of ~3.5 (full lashes, which
+     * read as mascara); on a male record it is capped at `LASH_MALE`, and
+     * without a record the lashes stay as shipped.
+     */
+    gender?: string;
+    /** Cap on the lashes' export alpha multiplier; set by `gender`, or directly. */
+    lashOpacity?: number;
+}
+/**
+ * The lash alpha multiplier a male record is capped to: the texture's own
+ * alpha, no multiplier. At 1.6 the lashes were still the darkest line on a
+ * male face whose brows and stubble are faint, and read as the feature.
+ */
+export declare const LASH_MALE = 1;
+export interface SurfaceReport {
+    skin: number;
+    hair: number;
+    textures: number;
+    /** Head vertices shaded as scalp. */
+    scalp: number;
+}
+/** Mean linear colour of a texture's opaque texels, or null if unreadable. */
+export declare function meanTextureColor(tex: THREE.Texture | null | undefined): THREE.Color | null;
+/**
+ * Mean linear colour of a texture where a mesh actually uses it: sampled at
+ * every third vertex's UV. A head atlas is mostly empty or not skin (mouth,
+ * ears, unused space), so its own mean said nothing about the face (luma
+ * 0.013 measured against 0.026 sampled at the mesh). Null if unreadable.
+ */
+export declare function meanSkinColor(mesh: THREE.Mesh, tex: THREE.Texture | null | undefined): THREE.Color | null;
+/** three's physical lighting setup, with hair specular faded at the silhouette. */
+export declare function patchHairSpecular(fs: string): string;
+/**
+ * Coverage that follows the strand's own alpha. three's WebGL
+ * alpha-to-coverage applies the cut as a step (every texel fully in or
+ * out), so a fringe read as sparse separate strings with skin between them;
+ * coverage proportional to alpha (as WebGPU's is, and as the studio films
+ * were rendered) draws faint strand texels faintly and the hair reads full
+ * and soft, with alpha scaled back up on the smaller mips. Only
+ * near-empty texels are discarded.
+ */
+export declare function patchHairCoverage(fs: string): string;
+/**
+ * Seamless pore and fine-crease detail as a tangent-space normal map,
+ * deterministic (seeded) so every load is identical. Shared by every
+ * character: it is never stored on a material, so `dispose()` does not free
+ * it out from under another body.
+ */
+export declare function poreDetailTexture(): THREE.DataTexture;
+/** three's shadowmap_vertex, with the normal offset clamped to face scale. */
+export declare function skinShadowVertex(chunk: string): string;
+/**
+ * A lit fragment shader receiving shadow as described above. Unchanged when
+ * it has no lights (or is already patched).
+ */
+export declare function patchShadowReceiver(fragmentShader: string): string;
+/** three's lights_fragment_begin, with skin's shadow receiver. */
+export declare function skinLightsBegin(chunk: string): string;
+/**
+ * Patch three's WebGL physical fragment shader for skin. Exported for the
+ * test, which runs it on three's real shader source. A shader without the
+ * anchors (a future three, a custom material) is returned unchanged.
+ */
+export declare function patchSkinShader(fragmentShader: string, withDetail: boolean): string;
+/**
+ * Darken the head's vertices that sit under hair: within `near` of a hair
+ * vertex lying over them (within 60° of the skin's outward normal), above
+ * the eyes. Hair beside the skin does not count: at the hairline the fringe
+ * lies along the forehead, and counting it tinted a hard brown strip below
+ * the hairline (the published heads have ~2.6k vertices, so a tinted row
+ * reads as a band). The shade also scales with how much hair is over the
+ * vertex, so it fades out through the thinning hairline. Reads skinned positions in WORLD space, so call it
+ * after the body is scaled to metres and before anything animates it.
+ */
+export declare function shadeScalp(root: THREE.Object3D, opts?: {
+    near?: number;
+    far?: number;
+}): number;
+/**
+ * Depth in the hair: strands near the scalp sit under the ones above them and
+ * get less light, so a real head of hair is darker at the roots and inside
+ * than on its outer shell. Card hair carries none of that unless its texture
+ * bakes it in (the long cuts do, a short light cut did not and read as a flat
+ * cap). Each hair vertex is darkened by how close it lies to the head,
+ * against this hair's own thickness (its 90th-percentile distance from the
+ * head), as a vertex colour that multiplies the diffuse: roots to ~45% on
+ * light hair, less on dark hair, where the texture already carries the depth
+ * and the same darkening would have read as a different cut. Reads skinned
+ * positions in WORLD space: call it after the body is scaled to metres and
+ * before anything animates it. Returns the number of hair vertices shaded.
+ */
+export declare function shadeHairDepth(root: THREE.Object3D): number;
+/**
+ * Upgrade skin and hair materials in place (skin materials are replaced on
+ * their meshes; the originals are disposed, their textures kept). Run once
+ * after load. `shadeScalp` is separate because it needs the body in metres.
+ */
+export declare function applySurfaces(root: THREE.Object3D, tuning?: SurfaceTuning): SurfaceReport;