@convai/embody 0.1.0

package/dist/resolve/resolver.d.ts

@@ -0,0 +1,56 @@
+/**
+ * CharacterResolver — the seam between @convai/embody and the asset backend.
+ *
+ * characterId → CharacterBundle. Backed today by middleman's
+ * GET /character/{id}/embodiment (phase-1 avatar_config storage); the release
+ * tables replace that storage later without this contract moving — which is
+ * the entire reason the seam exists.
+ */
+export interface ModelVariant {
+    lod?: number;
+    url: string;
+    size?: number;
+    encodings?: Record<string, string>;
+}
+export interface Representation {
+    rig: string;
+    model: ModelVariant[];
+    clips?: Record<string, {
+        url: string;
+    }>;
+}
+export interface CharacterBundle {
+    characterId: string;
+    avatarId: string;
+    gender?: 'male' | 'female' | string;
+    release?: number;
+    representations: Representation[];
+    thumbnail?: string;
+}
+export interface CharacterResolver {
+    resolve(characterId: string): Promise<CharacterBundle>;
+}
+/**
+ * Default backend host.
+ *
+ * Release branches carry different values and `scripts/check-hosts.mjs`
+ * enforces which, mirroring how @convai/web-sdk pins its connect host:
+ *
+ *   stable-release -> production   (what ships to `latest`)
+ *   beta-release   -> production   (first line of testing, against real infra)
+ *   stg-release    -> staging      (staging has its own character database)
+ *
+ * A branch shipping the wrong default silently sends every consumer who does
+ * not pass `baseUrl` to the wrong backend, which is why it is checked rather
+ * than trusted.
+ */
+export declare const DEFAULT_BASE_URL = "https://api-stg.convai.com";
+export interface HttpResolverOptions {
+    /** Base url of the embodiment backend. Defaults to the release branch's host. */
+    baseUrl?: string;
+    /** Optional CONVAI-API-KEY; public characters resolve as guest without it. */
+    apiKey?: string;
+}
+export declare function httpResolver(opts?: HttpResolverOptions): CharacterResolver;
+/** Pick the representation for a rig profile this build of embody understands. */
+export declare function pickRepresentation(bundle: CharacterBundle, supported: string[]): Representation;

package/dist/resolve/resolver.js

@@ -0,0 +1,45 @@
+/**
+ * CharacterResolver — the seam between @convai/embody and the asset backend.
+ *
+ * characterId → CharacterBundle. Backed today by middleman's
+ * GET /character/{id}/embodiment (phase-1 avatar_config storage); the release
+ * tables replace that storage later without this contract moving — which is
+ * the entire reason the seam exists.
+ */
+/**
+ * Default backend host.
+ *
+ * Release branches carry different values and `scripts/check-hosts.mjs`
+ * enforces which, mirroring how @convai/web-sdk pins its connect host:
+ *
+ *   stable-release -> production   (what ships to `latest`)
+ *   beta-release   -> production   (first line of testing, against real infra)
+ *   stg-release    -> staging      (staging has its own character database)
+ *
+ * A branch shipping the wrong default silently sends every consumer who does
+ * not pass `baseUrl` to the wrong backend, which is why it is checked rather
+ * than trusted.
+ */
+export const DEFAULT_BASE_URL = 'https://api-stg.convai.com';
+export function httpResolver(opts = {}) {
+    const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
+    return {
+        async resolve(characterId) {
+            const res = await fetch(`${baseUrl}/character/${characterId}/embodiment`, {
+                headers: opts.apiKey ? { 'CONVAI-API-KEY': opts.apiKey } : undefined,
+            });
+            if (!res.ok)
+                throw new Error(`embodiment resolve failed: HTTP ${res.status}`);
+            return (await res.json());
+        },
+    };
+}
+/** Pick the representation for a rig profile this build of embody understands. */
+export function pickRepresentation(bundle, supported) {
+    for (const rig of supported) {
+        const rep = bundle.representations.find((r) => r.rig === rig);
+        if (rep)
+            return rep;
+    }
+    throw new Error(`no supported representation: character offers [${bundle.representations.map((r) => r.rig).join(', ')}], embody supports [${supported.join(', ')}]`);
+}

package/dist/rig/profile.d.ts

@@ -0,0 +1,64 @@
+import * as THREE from 'three';
+/**
+ * Bone roles every motion system addresses by name, resolved through a
+ * `RigProfile`'s alias lists rather than hard-coded bone names. Ported from
+ * the CC4/UE dual-rig hooks in convai-web-sdk's rigBones.ts, generalised so
+ * each rig's aliases and axis basis live together as data on one profile
+ * instead of being picked by name-sniffing (`rigBasisFor`).
+ */
+export type BoneRole = 'head' | 'neck1' | 'neck2' | 'spineChest' | 'spineWaist';
+export type MorphRole = 'blinkL' | 'blinkR' | 'blinkCorrectiveL' | 'blinkCorrectiveR' | 'relax' | 'lowerLidUp' | 'eyeLookUpL' | 'eyeLookDownL' | 'eyeLookInL' | 'eyeLookOutL' | 'eyeLookUpR' | 'eyeLookDownR' | 'eyeLookInR' | 'eyeLookOutR';
+/** Which bone-LOCAL axes point right / up / forward in the rest pose. */
+export interface RigBasis {
+    right: THREE.Vector3;
+    up: THREE.Vector3;
+    fwd: THREE.Vector3;
+}
+export interface RigProfile {
+    id: string;
+    bones: Partial<Record<BoneRole, readonly string[]>>;
+    basis: RigBasis;
+    morphs: Partial<Record<MorphRole, readonly string[]>>;
+}
+/** CC4 (Character Creator) bones: +X right, +Y up, +Z forward. */
+export declare const CC4_BASIS: RigBasis;
+/** Measured live on the Sofia v4 GLBs: head/neck_01/neck_02/spine_04 share it. */
+export declare const UE_BASIS: RigBasis;
+/**
+ * Yaw about `up`, then pitch about `right`, then roll about `fwd`.
+ * Bit-identical to Euler(pitch, yaw, roll, 'YXZ') on the CC4 basis.
+ * Positive yaw turns toward the character's right; positive pitch looks down.
+ *
+ * Writes into `out` and returns it — never allocates on the steady-state path.
+ */
+export declare function setLookQuat(out: THREE.Quaternion, basis: RigBasis, yaw: number, pitch: number, roll: number): THREE.Quaternion;
+/**
+ * The mandated Convai rig (character spec 2.1/2.2). UE MetaHuman names,
+ * lowercase and exact. No eye bones, no jaw bones — both are morph-driven.
+ */
+export declare const CONVAI_MHA_1: RigProfile;
+/**
+ * Legacy CC4 (the old Sofia). Present so the path is not lost; NOT registered.
+ * The character spec is explicit that the CC4 alias table "is not a licence
+ * for new characters to vary."
+ *
+ * RULING (2026-09-07 fix wave, must-fix triage item 2): this profile was
+ * publicly exported but broken — `relax`/`lowerLidUp` used the CC4 ARKit
+ * singular names (`Eye_Relax`, `Eye_LowerLid_Up`) instead of the CC4
+ * ExpressionPlus per-eye names this rig's morph set actually carries
+ * (`Eye_Relax_L`/`_R`, `Eye_LowerLid_Up_L`/`_R` — see useBlink.ts:45-56),
+ * and it had no eye-look table at all, so `EyeGazeSystem` resolved zero
+ * bindings against it. Fixed here (rather than unexported) by porting the
+ * full CC4 ExpressionPlus alias set from useEyeTracking.ts's
+ * `MORPH_NAME_TO_CHANNEL` (lines 139-146). The eye-look in/out cross-naming
+ * mirrors physically between eyes (left eye's nasal/"in" side is named
+ * `_Right_L`; right eye's is named `_Left_R`) — derived from, and consistent
+ * with, `CONVAI_MHA_1`'s own CTRL_expressions cross-naming above, and
+ * verified in rig-resolve.test.ts against a synthetic CC4-named mesh.
+ */
+export declare const CC4_LEGACY: RigProfile;
+/**
+ * Profiles embody will actually resolve against. CC4 is deliberately absent —
+ * the registered rig is `convai-mha-1` only.
+ */
+export declare const REGISTERED_PROFILES: readonly RigProfile[];

package/dist/rig/profile.js

@@ -0,0 +1,110 @@
+import * as THREE from 'three';
+/** CC4 (Character Creator) bones: +X right, +Y up, +Z forward. */
+export const CC4_BASIS = {
+    right: new THREE.Vector3(1, 0, 0),
+    up: new THREE.Vector3(0, 1, 0),
+    fwd: new THREE.Vector3(0, 0, 1),
+};
+/** Measured live on the Sofia v4 GLBs: head/neck_01/neck_02/spine_04 share it. */
+export const UE_BASIS = {
+    right: new THREE.Vector3(0, 0, -1),
+    up: new THREE.Vector3(1, 0, 0),
+    fwd: new THREE.Vector3(0, -1, 0),
+};
+const qYaw = new THREE.Quaternion();
+const qPitch = new THREE.Quaternion();
+const qRoll = new THREE.Quaternion();
+/**
+ * Yaw about `up`, then pitch about `right`, then roll about `fwd`.
+ * Bit-identical to Euler(pitch, yaw, roll, 'YXZ') on the CC4 basis.
+ * Positive yaw turns toward the character's right; positive pitch looks down.
+ *
+ * Writes into `out` and returns it — never allocates on the steady-state path.
+ */
+export function setLookQuat(out, basis, yaw, pitch, roll) {
+    qYaw.setFromAxisAngle(basis.up, yaw);
+    qPitch.setFromAxisAngle(basis.right, pitch);
+    qRoll.setFromAxisAngle(basis.fwd, roll);
+    return out.copy(qYaw).multiply(qPitch).multiply(qRoll);
+}
+/**
+ * The mandated Convai rig (character spec 2.1/2.2). UE MetaHuman names,
+ * lowercase and exact. No eye bones, no jaw bones — both are morph-driven.
+ */
+export const CONVAI_MHA_1 = {
+    id: 'convai-mha-1',
+    basis: UE_BASIS,
+    bones: {
+        head: ['head'],
+        neck1: ['neck_01'],
+        neck2: ['neck_02'],
+        spineChest: ['spine_04'],
+        spineWaist: ['spine_02'],
+    },
+    morphs: {
+        blinkL: ['CTRL_expressions_eyeBlinkL', 'EyeBlinkLeft'],
+        blinkR: ['CTRL_expressions_eyeBlinkR', 'EyeBlinkRight'],
+        relax: ['CTRL_expressions_eyeRelaxL', 'CTRL_expressions_eyeRelaxR'],
+        lowerLidUp: ['CTRL_expressions_eyeLowerLidUpL', 'CTRL_expressions_eyeLowerLidUpR'],
+        // Cross-named on purpose — verified in useEyeTracking. eyeLookLeft*
+        // rotates toward the character's RIGHT, so it is the "in" channel for the
+        // LEFT eye and the "out" channel for the right.
+        eyeLookUpL: ['CTRL_expressions_eyeLookUpL', 'EyeLookUpLeft'],
+        eyeLookDownL: ['CTRL_expressions_eyeLookDownL', 'EyeLookDownLeft'],
+        eyeLookInL: ['CTRL_expressions_eyeLookLeftL', 'EyeLookInLeft'],
+        eyeLookOutL: ['CTRL_expressions_eyeLookRightL', 'EyeLookOutLeft'],
+        eyeLookUpR: ['CTRL_expressions_eyeLookUpR', 'EyeLookUpRight'],
+        eyeLookDownR: ['CTRL_expressions_eyeLookDownR', 'EyeLookDownRight'],
+        eyeLookInR: ['CTRL_expressions_eyeLookRightR', 'EyeLookInRight'],
+        eyeLookOutR: ['CTRL_expressions_eyeLookLeftR', 'EyeLookOutRight'],
+    },
+};
+/**
+ * Legacy CC4 (the old Sofia). Present so the path is not lost; NOT registered.
+ * The character spec is explicit that the CC4 alias table "is not a licence
+ * for new characters to vary."
+ *
+ * RULING (2026-09-07 fix wave, must-fix triage item 2): this profile was
+ * publicly exported but broken — `relax`/`lowerLidUp` used the CC4 ARKit
+ * singular names (`Eye_Relax`, `Eye_LowerLid_Up`) instead of the CC4
+ * ExpressionPlus per-eye names this rig's morph set actually carries
+ * (`Eye_Relax_L`/`_R`, `Eye_LowerLid_Up_L`/`_R` — see useBlink.ts:45-56),
+ * and it had no eye-look table at all, so `EyeGazeSystem` resolved zero
+ * bindings against it. Fixed here (rather than unexported) by porting the
+ * full CC4 ExpressionPlus alias set from useEyeTracking.ts's
+ * `MORPH_NAME_TO_CHANNEL` (lines 139-146). The eye-look in/out cross-naming
+ * mirrors physically between eyes (left eye's nasal/"in" side is named
+ * `_Right_L`; right eye's is named `_Left_R`) — derived from, and consistent
+ * with, `CONVAI_MHA_1`'s own CTRL_expressions cross-naming above, and
+ * verified in rig-resolve.test.ts against a synthetic CC4-named mesh.
+ */
+export const CC4_LEGACY = {
+    id: 'cc4-legacy',
+    basis: CC4_BASIS,
+    bones: {
+        head: ['CC_Base_Head'],
+        neck1: ['CC_Base_NeckTwist01'],
+        neck2: ['CC_Base_NeckTwist02'],
+        spineChest: ['CC_Base_Spine02'],
+        spineWaist: ['CC_Base_Spine01'],
+    },
+    morphs: {
+        blinkL: ['Eye_Blink_L'], blinkR: ['Eye_Blink_R'],
+        blinkCorrectiveL: ['C_BlinkL'], blinkCorrectiveR: ['C_BlinkR'],
+        relax: ['Eye_Relax_L', 'Eye_Relax_R'],
+        lowerLidUp: ['Eye_LowerLid_Up_L', 'Eye_LowerLid_Up_R'],
+        eyeLookUpL: ['Eye_Look_Up_L'],
+        eyeLookDownL: ['Eye_Look_Down_L'],
+        eyeLookInL: ['Eye_Look_Right_L'],
+        eyeLookOutL: ['Eye_Look_Left_L'],
+        eyeLookUpR: ['Eye_Look_Up_R'],
+        eyeLookDownR: ['Eye_Look_Down_R'],
+        eyeLookInR: ['Eye_Look_Left_R'],
+        eyeLookOutR: ['Eye_Look_Right_R'],
+    },
+};
+/**
+ * Profiles embody will actually resolve against. CC4 is deliberately absent —
+ * the registered rig is `convai-mha-1` only.
+ */
+export const REGISTERED_PROFILES = [CONVAI_MHA_1];

package/dist/rig/resolve.d.ts

@@ -0,0 +1,33 @@
+import * as THREE from 'three';
+import type { BoneRole, MorphRole, RigProfile } from './profile.js';
+export interface MorphBinding {
+    mesh: THREE.Mesh;
+    index: number;
+}
+export interface ResolvedRig {
+    profile: RigProfile;
+    bones: Partial<Record<BoneRole, THREE.Object3D>>;
+    /**
+     * Each resolved bone's LOCAL quaternion at resolve time — captured once,
+     * before any pipeline system (mixer, gaze, breathing) has ever touched it,
+     * so it is the true bind/rest pose regardless of whether a clip happens to
+     * be active on frame 1. Motion systems that OVERRIDE a bone (gaze) or
+     * additively perturb it (breathing) compose onto this, never onto identity
+     * — see GazeSystem/BreathingSystem's `update()` for why (C1/C3 in the
+     * 2026-09-07 fix wave: the prior art always composes the look/breath
+     * offset onto the bone's own rest orientation, not world/parent identity).
+     */
+    restPose: Partial<Record<BoneRole, THREE.Quaternion>>;
+    morphs: Map<MorphRole, MorphBinding[]>;
+    missing: BoneRole[];
+}
+/**
+ * Bones the motion systems in this slice actually drive. The character spec
+ * also requires spine_01/03/05 for CLIP compatibility, but nothing here reads
+ * them — validating those is the upload pipeline's job, not embody's.
+ */
+export declare const REQUIRED_BONES: readonly BoneRole[];
+/** Apply a profile to a loaded root. Call once, at load — never per frame. */
+export declare function resolveRig(root: THREE.Object3D, profile: RigProfile): ResolvedRig;
+/** Write a value to every mesh carrying a role. Absent roles are a no-op. */
+export declare function writeMorph(rig: ResolvedRig, role: MorphRole, value: number): void;

package/dist/rig/resolve.js

@@ -0,0 +1,76 @@
+/**
+ * Bones the motion systems in this slice actually drive. The character spec
+ * also requires spine_01/03/05 for CLIP compatibility, but nothing here reads
+ * them — validating those is the upload pipeline's job, not embody's.
+ */
+export const REQUIRED_BONES = [
+    'head', 'neck1', 'neck2', 'spineChest', 'spineWaist',
+];
+function findFirst(root, aliases) {
+    for (const name of aliases) {
+        const hit = root.getObjectByName(name);
+        if (hit)
+            return hit;
+    }
+    return null;
+}
+/** Apply a profile to a loaded root. Call once, at load — never per frame. */
+export function resolveRig(root, profile) {
+    const bones = {};
+    const restPose = {};
+    const missing = [];
+    for (const role of Object.keys(profile.bones)) {
+        const hit = findFirst(root, profile.bones[role] ?? []);
+        if (hit) {
+            bones[role] = hit;
+            // Cloned NOW, before any system's first update() — this is the only
+            // point in the character's lifecycle guaranteed to run before stage 1
+            // (the mixer) has ever written to the bone.
+            restPose[role] = hit.quaternion.clone();
+        }
+    }
+    for (const role of REQUIRED_BONES) {
+        if (!bones[role])
+            missing.push(role);
+    }
+    if (missing.length) {
+        const detail = missing.map((r) => {
+            const aliases = profile.bones[r] ?? [];
+            const desc = aliases.length > 0 ? aliases.join('|') : 'no aliases declared in profile';
+            return `${r} (${desc})`;
+        }).join(', ');
+        console.warn(`[embody] rig "${profile.id}": missing required bone(s) — ${detail}. ` +
+            `Motion systems driving them will no-op.`);
+    }
+    const morphs = new Map();
+    const meshes = [];
+    root.traverse((o) => {
+        const m = o;
+        if (m.isMesh && m.morphTargetDictionary)
+            meshes.push(m);
+    });
+    for (const role of Object.keys(profile.morphs)) {
+        const bindings = [];
+        for (const name of profile.morphs[role] ?? []) {
+            for (const mesh of meshes) {
+                const index = mesh.morphTargetDictionary?.[name];
+                if (index !== undefined)
+                    bindings.push({ mesh, index });
+            }
+        }
+        if (bindings.length)
+            morphs.set(role, bindings);
+    }
+    return { profile, bones, restPose, morphs, missing };
+}
+/** Write a value to every mesh carrying a role. Absent roles are a no-op. */
+export function writeMorph(rig, role, value) {
+    const bindings = rig.morphs.get(role);
+    if (!bindings)
+        return;
+    for (const b of bindings) {
+        const infl = b.mesh.morphTargetInfluences;
+        if (infl)
+            infl[b.index] = value;
+    }
+}

package/dist/rig/units.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Unit normalisation for character models.
+ *
+ * Assets disagree about export scale (sofia.glb: raw centimetres;
+ * japanese-f.glb: centimetre geometry under a baked 0.01 root), and every
+ * static measure of a SkinnedMesh lies about rendered size — Box3 and
+ * computeBoundingBox read bind-pose geometry, bone spans miss bind scaling.
+ *
+ * The assumption-free answer (proven in the playhouse's CharacterPreview):
+ * measure the head→ankle bone span actually in world space every frame and
+ * multiply the scale by the error until it converges. Costs two
+ * getWorldPosition calls; settles in a frame or two; cannot be wrong about
+ * units because it never assumes any.
+ *
+ * This is consumer-side insurance. The character spec gains a units mandate
+ * and the processing pipeline will normalise export scale; this stays as the
+ * belt to that suspenders.
+ */
+import * as THREE from 'three';
+export interface UnitNormalizer {
+    /** Run every frame until it reports true (converged). Cheap either way. */
+    update(): boolean;
+}
+export declare function createUnitNormalizer(root: THREE.Object3D, opts?: {
+    targetHeight?: number;
+    groundY?: number;
+}): UnitNormalizer;

package/dist/rig/units.js

@@ -0,0 +1,61 @@
+/**
+ * Unit normalisation for character models.
+ *
+ * Assets disagree about export scale (sofia.glb: raw centimetres;
+ * japanese-f.glb: centimetre geometry under a baked 0.01 root), and every
+ * static measure of a SkinnedMesh lies about rendered size — Box3 and
+ * computeBoundingBox read bind-pose geometry, bone spans miss bind scaling.
+ *
+ * The assumption-free answer (proven in the playhouse's CharacterPreview):
+ * measure the head→ankle bone span actually in world space every frame and
+ * multiply the scale by the error until it converges. Costs two
+ * getWorldPosition calls; settles in a frame or two; cannot be wrong about
+ * units because it never assumes any.
+ *
+ * This is consumer-side insurance. The character spec gains a units mandate
+ * and the processing pipeline will normalise export scale; this stays as the
+ * belt to that suspenders.
+ */
+import * as THREE from 'three';
+/** Head→ankle joint span as a fraction of full stature (calibrated on Sofia). */
+const BONE_SPAN_RATIO = 0.837;
+export function createUnitNormalizer(root, opts = {}) {
+    const target = (opts.targetHeight ?? 1.7) * BONE_SPAN_RATIO;
+    const groundY = opts.groundY ?? 0;
+    let head = null;
+    let foot = null;
+    root.traverse((o) => {
+        const n = o.name.trim().toLowerCase();
+        if (!head && n === 'head')
+            head = o;
+        if (!foot && n === 'foot_l')
+            foot = o;
+    });
+    const a = new THREE.Vector3();
+    const b = new THREE.Vector3();
+    let settled = 0;
+    return {
+        update() {
+            if (!head || !foot)
+                return true;
+            head.updateWorldMatrix(true, false);
+            foot.updateWorldMatrix(true, false);
+            const span = head.getWorldPosition(a).y - foot.getWorldPosition(b).y;
+            if (!(span > 1e-9))
+                return false;
+            const err = target / span;
+            if (Math.abs(err - 1) > 0.005) {
+                root.scale.multiplyScalar(err);
+                root.updateWorldMatrix(true, true);
+                settled = 0;
+            }
+            else
+                settled++;
+            // feet on the floor, every pass, so scale correction never leaves the
+            // character hovering or sunk
+            foot.updateWorldMatrix(true, false);
+            root.position.y += groundY - foot.getWorldPosition(b).y;
+            return settled > 2;
+        },
+    };
+}

package/dist/speech/lipsync.d.ts

@@ -0,0 +1,56 @@
+/**
+ * LipsyncSystem — MHA lipsync, framework-free.
+ *
+ * A faithful port of pagoda-act's useMhaLipsync. The shaping is the SDK's
+ * (createLipsyncProcessor("mha"): symmetrize → limits → gain table →
+ * bloom/settle envelope → morph write, repairing MetaHuman export name
+ * mangling at bind). What this layer owns:
+ *
+ *  · driving the queue by ASKING it (real fps, bot-speaking flag, the one-shot
+ *    normalization signal) instead of inferring any of it
+ *  · the mouth character layer — jaw cap with proportional lower lip, the
+ *    bilabial lip-seal floor, teeth follow rules — exposed as MouthTuning so
+ *    per-character adjustment is a typed parameter, not a fork of this file
+ *  · speaking/energy signals for higher layers (body clip transitions, gaze)
+ */
+import type * as THREE from 'three';
+import type { BlendshapeQueue } from '@convai/web-sdk/core';
+/** Tier-1 customization: the mouth knobs, per character. Defaults = pagoda-act's tuned values. */
+export interface MouthTuning {
+    /** Hard cap on jawOpen; peak excursions read exaggerated on camera. */
+    jawCap?: number;
+    /** Lip-seal floor ceiling and onset for bilabials (m/b/p). */
+    sealCeiling?: number;
+    sealOnset?: number;
+    /** Teeth follow rules. */
+    teethRetract?: number;
+    teethUpperLift?: number;
+    teethLowerTuck?: number;
+    /** Gain multipliers by MHA channel name, forwarded to the processor. */
+    gainOverrides?: Record<string, number>;
+    /** Seconds of visual lead granted at speech start. */
+    visualLeadSeconds?: number;
+}
+declare const DEFAULTS: Required<Omit<MouthTuning, 'gainOverrides'>> & {
+    gainOverrides: Record<string, number>;
+};
+export declare class LipsyncSystem {
+    readonly t: typeof DEFAULTS;
+    private readonly processor;
+    private bindings;
+    private accum;
+    private lastFrame;
+    private wasSpeaking;
+    private env;
+    /** 0–1 speech energy; gaze layers read it. */
+    energy: number;
+    /** True while the bot owns the mouth; body layers read it. */
+    speaking: boolean;
+    private readonly ch;
+    constructor(tuning?: MouthTuning);
+    /** Bind every morph-bearing mesh under root. Returns matched channel count. */
+    bind(root: THREE.Object3D): number;
+    /** Tick each frame. Pass the queue only if this character owns the turn. */
+    update(delta: number, q: BlendshapeQueue | undefined): void;
+}
+export {};

package/dist/speech/lipsync.js

@@ -0,0 +1,139 @@
+import { createLipsyncProcessor, mhaChannelIndex, } from '@convai/web-sdk/lipsync-helpers';
+const DEFAULTS = {
+    jawCap: 0.55,
+    sealCeiling: 0.5,
+    sealOnset: 0.45,
+    teethRetract: 0.55,
+    teethUpperLift: 0.3,
+    teethLowerTuck: 0.34,
+    visualLeadSeconds: 0.1,
+    gainOverrides: { mouthLowerLipDepressL: 0.28, mouthLowerLipDepressR: 0.28 },
+};
+export class LipsyncSystem {
+    constructor(tuning = {}) {
+        this.bindings = [];
+        this.accum = 0;
+        this.lastFrame = null;
+        this.wasSpeaking = false;
+        this.env = 0;
+        /** 0–1 speech energy; gaze layers read it. */
+        this.energy = 0;
+        /** True while the bot owns the mouth; body layers read it. */
+        this.speaking = false;
+        this.t = { ...DEFAULTS, ...tuning, gainOverrides: { ...DEFAULTS.gainOverrides, ...tuning.gainOverrides } };
+        this.processor = createLipsyncProcessor('mha', {
+            skipEyeChannels: true,
+            gainOverrides: this.t.gainOverrides,
+        });
+        const idx = (n) => mhaChannelIndex(n) ?? -1;
+        const idxs = (...ns) => ns.map((n) => mhaChannelIndex(n)).filter((i) => i !== undefined);
+        this.ch = {
+            jawOpen: idx('jawOpen'),
+            lipsTogether: idxs('mouthLipsTogetherUL', 'mouthLipsTogetherUR', 'mouthLipsTogetherDL', 'mouthLipsTogetherDR'),
+            lowerLipDepress: idxs('mouthLowerLipDepressL', 'mouthLowerLipDepressR'),
+            funnel: idxs('mouthFunnelUL', 'mouthFunnelUR'),
+            teethBackUpper: idx('teethBackU'),
+            teethBackLower: idx('teethBackD'),
+            teethUpUpper: idx('teethUpU'),
+            teethDownLower: idx('teethDownD'),
+        };
+    }
+    /** Bind every morph-bearing mesh under root. Returns matched channel count. */
+    bind(root) {
+        this.bindings = this.processor.bindAll(root);
+        return this.bindings.reduce((s, b) => s + b.matched, 0);
+    }
+    /** Tick each frame. Pass the queue only if this character owns the turn. */
+    update(delta, q) {
+        if (!this.bindings.length)
+            return;
+        if (!q) {
+            // Not holding the queue means not speaking — and that must be SAID,
+            // or the body layer holds "talking" forever.
+            this.speaking = false;
+            this.energy += (0 - this.energy) * (1 - Math.exp(-delta / 0.22));
+            this.processor.apply(this.processor.process(null, delta), this.bindings);
+            return;
+        }
+        const fps = q.getPlaybackFps?.() || 60;
+        const frameTime = 1 / fps;
+        const speaking = q.isBotSpeaking?.() ?? true;
+        if (speaking && !this.wasSpeaking)
+            this.accum = this.t.visualLeadSeconds;
+        this.wasSpeaking = speaking;
+        let fresh = null;
+        if (speaking) {
+            this.accum += delta;
+            while (this.accum >= frameTime && q.hasFrames()) {
+                const frame = q.getFrameWithAlpha(0);
+                q.consumeFrames(1);
+                this.accum -= frameTime;
+                if (frame)
+                    fresh = frame; // draw only the newest; skip, never slow-replay
+            }
+            if (this.accum > 0.25)
+                this.accum = 0.25; // a starved queue banks no debt
+            if (fresh)
+                this.lastFrame = fresh;
+        }
+        else {
+            this.accum = 0;
+        }
+        const raw = speaking ? this.lastFrame : null;
+        const out = this.processor.process(raw, delta);
+        // mirror the processor's envelope so the seal fades with speech
+        const target = raw ? 1 : 0;
+        const tau = target > this.env ? 0.09 : 0.18;
+        this.env += (target - this.env) * (1 - Math.exp(-delta / tau));
+        if (out) {
+            const { jawCap, sealCeiling, sealOnset, teethRetract, teethUpperLift, teethLowerTuck } = this.t;
+            const ch = this.ch;
+            // hard jaw cap, lower lip scaled by the same ratio so the cap changes
+            // the size of the opening without changing its shape
+            let capRatio = 1;
+            if (ch.jawOpen >= 0 && out[ch.jawOpen] > jawCap) {
+                capRatio = jawCap / out[ch.jawOpen];
+                out[ch.jawOpen] = jawCap;
+            }
+            if (capRatio < 1)
+                for (const i of ch.lowerLipDepress)
+                    out[i] *= capRatio;
+            // bilabial closure floor, driven from the RAW jaw
+            if (raw && ch.jawOpen >= 0 && ch.lipsTogether.length) {
+                const jawRaw = raw[ch.jawOpen];
+                const bilabial = 1 - Math.min(1, Math.max(0, (jawRaw - 0.02) / 0.16));
+                if (bilabial > sealOnset) {
+                    const seal = ((bilabial - sealOnset) / (1 - sealOnset)) * sealCeiling * this.env;
+                    for (const i of ch.lipsTogether)
+                        if (out[i] < seal)
+                            out[i] = seal;
+                }
+            }
+            // teeth follow the SHAPED jaw
+            const jaw = ch.jawOpen >= 0 ? out[ch.jawOpen] : 0;
+            if (ch.teethBackUpper >= 0)
+                out[ch.teethBackUpper] = jaw * teethRetract;
+            if (ch.teethBackLower >= 0)
+                out[ch.teethBackLower] = jaw * teethRetract;
+            if (ch.teethUpUpper >= 0)
+                out[ch.teethUpUpper] = jaw * teethUpperLift;
+            if (ch.teethDownLower >= 0) {
+                // normalised against the CAP, not 1 — lands on exactly 0 at widest
+                const open = jawCap > 0 ? Math.min(1, jaw / jawCap) : 0;
+                out[ch.teethDownLower] = teethLowerTuck * (1 - open);
+            }
+            // speech energy: fast attack, slow release
+            let mag = jaw;
+            for (const i of ch.funnel)
+                mag = Math.max(mag, out[i]);
+            const e = Math.min(1, mag / 0.35);
+            const eTau = e > this.energy ? 0.06 : 0.22;
+            this.energy += (e - this.energy) * (1 - Math.exp(-delta / eTau));
+        }
+        else {
+            this.energy += (0 - this.energy) * (1 - Math.exp(-delta / 0.22));
+        }
+        this.processor.apply(out, this.bindings);
+        this.speaking = speaking;
+    }
+}