@bitmagic/cli 0.1.66-dev.1 → 0.1.66-dev.10

package/assets/metahuman/src/work/metahuman/MetaHumanSpeech.ts

@@ -0,0 +1,242 @@
+/**
+ * Lip sync for the MetaHuman: plays a spoken line and drives the face's ARKit
+ * blendshapes from Rhubarb Lip Sync mouth cues (https://github.com/DanielSWolf/rhubarb-lip-sync),
+ * falling back to an amplitude-driven jaw when a line ships without cues.
+ *
+ * Rhubarb's cues are the Preston Blair mouth shapes: A (closed, M/B/P), B (slightly open,
+ * most consonants + EE), C (open, EH), D (wide open, AA), E (rounded, AO/ER), F (puckered,
+ * UW/OW/W), G (F/V, teeth on lip), H (L, tongue up) and X (rest). Each is expressed here
+ * as a small set of ARKit weights; consecutive cues are cross-faded for co-articulation.
+ */
+import * as THREE from 'three';
+import type { MetaHumanActor } from './MetaHumanActor.js';
+
+export interface MouthCue { start: number; end: number; value: string }
+export interface LipSyncCues { metadata?: { soundFile?: string; duration?: number }; mouthCues: MouthCue[] }
+export interface SpeechLine { id: string; text: string; audioUrl: string; cuesUrl?: string; duration?: number }
+
+/** ARKit blendshape weights per Rhubarb mouth shape. */
+export const VISEMES: Record<string, Record<string, number>> = {
+    X: {},
+    A: { mouthPressLeft: 0.35, mouthPressRight: 0.35, mouthRollLower: 0.15, mouthRollUpper: 0.1 },
+    B: { jawOpen: 0.12, mouthStretchLeft: 0.25, mouthStretchRight: 0.25, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, mouthLowerDownLeft: 0.15, mouthLowerDownRight: 0.15 },
+    C: { jawOpen: 0.32, mouthLowerDownLeft: 0.2, mouthLowerDownRight: 0.2, mouthUpperUpLeft: 0.15, mouthUpperUpRight: 0.15, mouthStretchLeft: 0.1, mouthStretchRight: 0.1 },
+    D: { jawOpen: 0.62, mouthLowerDownLeft: 0.25, mouthLowerDownRight: 0.25, mouthUpperUpLeft: 0.2, mouthUpperUpRight: 0.2 },
+    E: { jawOpen: 0.26, mouthFunnel: 0.45, mouthPucker: 0.15 },
+    F: { jawOpen: 0.1, mouthPucker: 0.75, mouthFunnel: 0.3 },
+    G: { jawOpen: 0.08, mouthRollLower: 0.55, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, mouthShrugUpper: 0.1 },
+    H: { jawOpen: 0.28, tongueOut: 0.15, mouthLowerDownLeft: 0.1, mouthLowerDownRight: 0.1 },
+};
+
+/** Every blendshape any viseme touches — these are zeroed when a shape does not mention them. */
+const MOUTH_SHAPES: string[] = Array.from(new Set(Object.values(VISEMES).flatMap((v) => Object.keys(v))));
+
+/** How far ahead of a cue boundary the mouth starts moving toward the next shape (s). */
+const COARTICULATION = 0.06;
+
+export class MetaHumanSpeech {
+    private audio: HTMLAudioElement | null = null;
+    private ctx: AudioContext | null = null;
+    private analyser: AnalyserNode | null = null;
+    private readonly samples = new Uint8Array(512);
+    private cues: MouthCue[] = [];
+    private cueIdx = 0;
+    private speaking = false;
+    private line: SpeechLine | null = null;
+    private readonly current = new Map<string, number>();
+    private energy = 0;
+    private rms = 0;
+    private hold: string | null = null;
+    /**
+     * Clock-driven mode (the F9 recorder): cue time advances by the engine's frame delta instead of
+     * the audio element's clock, so the visemes land on the same frames whatever the capture speed,
+     * and the line runs even where audio playback is refused. The audio track is muxed in afterwards.
+     */
+    private clockDriven = false;
+    private clockT = 0;
+    private clockEnd = 0;
+    private peakJaw = 0;
+    onEnd: ((line: SpeechLine) => void) | null = null;
+
+    constructor(private readonly actor: MetaHumanActor) {}
+
+    static async loadLines(url: string): Promise<SpeechLine[]> {
+        try {
+            const res = await fetch(url, { cache: 'no-store' });
+            if (!res.ok) return [];
+            const lines = (await res.json()) as SpeechLine[];
+            return Array.isArray(lines) ? lines : [];
+        } catch {
+            return [];
+        }
+    }
+
+    isSpeaking(): boolean { return this.speaking; }
+    setClockDriven(on: boolean): void { this.clockDriven = on; }
+    isClockDriven(): boolean { return this.clockDriven; }
+    getEnergy(): number { return this.energy; }
+    getLine(): SpeechLine | null { return this.line; }
+
+    /** Start a line. Must be called from a user gesture the first time (browser audio policy). */
+    async speak(line: SpeechLine): Promise<boolean> {
+        this.stop();
+        this.line = line;
+        let cues: LipSyncCues | null = null;
+        if (line.cuesUrl) {
+            try {
+                const res = await fetch(line.cuesUrl, { cache: 'no-store' });
+                if (res.ok) cues = (await res.json()) as LipSyncCues;
+            } catch { /* no cues → amplitude fallback */ }
+        }
+        this.cues = cues?.mouthCues ?? [];
+        this.cueIdx = 0;
+
+        const audio = new Audio(line.audioUrl);
+        audio.preload = 'auto';
+        audio.crossOrigin = 'anonymous';
+        this.audio = audio;
+        this.analyser = null;
+        try {
+            if (!this.ctx) this.ctx = new AudioContext();
+            if (this.ctx.state === 'suspended') await this.ctx.resume();
+            const source = this.ctx.createMediaElementSource(audio);
+            const analyser = this.ctx.createAnalyser();
+            analyser.fftSize = 512;
+            analyser.smoothingTimeConstant = 0.4;
+            source.connect(analyser);
+            analyser.connect(this.ctx.destination);
+            this.analyser = analyser;
+        } catch (err) {
+            console.warn('[MetaHumanSpeech] no AudioContext analyser (audio still plays):', err);
+        }
+        audio.addEventListener('ended', () => { if (!this.clockDriven) this.finish(); });
+        this.clockT = 0;
+        this.clockEnd = line.duration ?? (this.cues.length ? this.cues[this.cues.length - 1]!.end : 0);
+        try {
+            await audio.play();
+        } catch (err) {
+            console.warn('[MetaHumanSpeech] audio.play() rejected (needs a user gesture?):', err);
+            this.audio = null;
+            this.analyser = null;
+            if (!this.clockDriven) return false; // clock-driven: the face still runs, the track is muxed later
+        }
+        this.speaking = true;
+        this.hold = null;
+        this.peakJaw = 0;
+        console.log(`[MetaHumanSpeech] speaking "${line.id}" (${this.cues.length} cues${this.analyser ? ', analyser' : ''})`);
+        return true;
+    }
+
+    stop(): void {
+        if (this.audio) {
+            this.audio.pause();
+            this.audio.src = '';
+            this.audio = null;
+        }
+        this.speaking = false;
+    }
+
+    private finish(): void {
+        if (!this.speaking) return;
+        this.speaking = false;
+        const line = this.line;
+        this.audio = null;
+        if (line) this.onEnd?.(line);
+    }
+
+    /** Debug: snap and HOLD the mouth on one Rhubarb shape (screenshots, tuning); `null`/'X' releases. */
+    scrub(shape: string | null): void {
+        this.hold = shape && shape !== 'X' ? shape : null;
+        const target = VISEMES[shape ?? 'X'] ?? {};
+        for (const name of MOUTH_SHAPES) {
+            const v = target[name] ?? 0;
+            this.current.set(name, v);
+            this.actor.setMorph(name, v);
+        }
+    }
+
+    debug(): Record<string, unknown> {
+        const t = this.audio?.currentTime ?? 0;
+        const cue = this.cues[this.cueIdx];
+        return { speaking: this.speaking, t, cueIdx: this.cueIdx, cues: this.cues.length, shape: cue?.value ?? null, energy: this.energy, rms: this.rms, analyser: !!this.analyser, line: this.line?.id ?? null, peakJaw: this.peakJaw, hold: this.hold };
+    }
+
+    update(deltaTime: number): void {
+        const dt = Math.min(deltaTime, 0.1);
+        if (this.speaking && this.clockDriven) {
+            this.clockT += dt;
+            if (this.clockEnd > 0 && this.clockT >= this.clockEnd) this.finish();
+        }
+        if (!this.speaking || (!this.audio && !this.clockDriven)) {
+            this.energy += (0 - this.energy) * (1 - Math.exp(-dt * 8));
+            if (this.hold) return; // a scrubbed shape stays put
+            if (this.current.size) this.blendTo({}, dt, 14);
+            return;
+        }
+        // Loudness (for brow/head accents, and the no-cues fallback).
+        this.rms = 0;
+        if (this.analyser) {
+            this.analyser.getByteTimeDomainData(this.samples);
+            let sum = 0;
+            for (let i = 0; i < this.samples.length; i++) {
+                const v = (this.samples[i]! - 128) / 128;
+                sum += v * v;
+            }
+            this.rms = Math.sqrt(sum / this.samples.length);
+        }
+        const t = this.clockDriven ? this.clockT : (this.audio?.currentTime ?? 0);
+        // Loudness: from the analyser in real time; from the cue shape (any open mouth) on the clock.
+        const cueNow = this.cues.length ? this.cues.find((c) => t >= c.start && t < c.end) : undefined;
+        const loud = this.clockDriven ? (cueNow && cueNow.value !== 'X' ? 0.6 : 0) : THREE.MathUtils.clamp(this.rms * 6, 0, 1);
+        this.energy += (loud - this.energy) * (1 - Math.exp(-dt * 12));
+
+        let target: Record<string, number>;
+        if (this.cues.length > 0) {
+            while (this.cueIdx < this.cues.length - 1 && t >= this.cues[this.cueIdx]!.end) this.cueIdx++;
+            const cue = this.cues[this.cueIdx]!;
+            const shape = t >= cue.start && t < cue.end ? cue.value : 'X';
+            target = { ...(VISEMES[shape] ?? {}) };
+            const next = this.cues[this.cueIdx + 1];
+            const remaining = cue.end - t;
+            if (next && remaining < COARTICULATION) {
+                const w = (1 - remaining / COARTICULATION) * 0.6;
+                const nextTarget = VISEMES[next.value] ?? {};
+                for (const name of MOUTH_SHAPES) {
+                    const a = target[name] ?? 0;
+                    const b = nextTarget[name] ?? 0;
+                    target[name] = a + (b - a) * w;
+                }
+            }
+            // Louder syllables open the jaw a little more.
+            if (this.analyser && target.jawOpen) target.jawOpen *= 0.85 + 0.3 * this.energy;
+        } else {
+            // No cues: amplitude-driven jaw flap.
+            const open = THREE.MathUtils.clamp(this.rms * 5, 0, 0.7);
+            target = { jawOpen: open, mouthFunnel: open * 0.3, mouthLowerDownLeft: open * 0.3, mouthLowerDownRight: open * 0.3 };
+        }
+        this.blendTo(target, dt, 24);
+        this.peakJaw = Math.max(this.peakJaw, this.current.get('jawOpen') ?? 0);
+    }
+
+    private blendTo(target: Record<string, number>, dt: number, rate: number): void {
+        const k = 1 - Math.exp(-dt * rate);
+        for (const name of MOUTH_SHAPES) {
+            const cur = this.current.get(name) ?? 0;
+            const tgt = target[name] ?? 0;
+            const v = cur + (tgt - cur) * k;
+            if (tgt === 0 && Math.abs(v) < 1e-3) {
+                this.current.delete(name);
+                this.actor.setMorph(name, 0);
+            } else {
+                this.current.set(name, v);
+                this.actor.setMorph(name, v);
+            }
+        }
+    }
+
+    dispose(): void {
+        this.stop();
+        this.ctx?.close().catch(() => { /* ignore */ });
+        this.ctx = null;
+    }
+}

package/assets/metahuman/src/work/metahuman/StudioEnvironment.ts

@@ -0,0 +1,142 @@
+/**
+ * Image-based studio lighting for the MetaHuman lab: a real HDRI (Poly Haven,
+ * CC0) prefiltered into `scene.environment`, plus one fixed rim light.
+ *
+ * The engine's AmbientLighting builds scene.environment from the skybox (or a
+ * neutral grey fallback) and may re-apply it on skybox events, so this class
+ * re-asserts its own map right before every render.
+ */
+import * as THREE from 'three';
+import { RGBELoader } from 'three/addons/loaders/RGBELoader.js';
+import { PMREMGenerator as PMREMGeneratorWebGPU } from 'three/webgpu';
+import { PMREMGenerator as PMREMGeneratorWebGL } from 'three/src/extras/PMREMGenerator.js';
+import { isWebGpuActive } from 'engine/RendererType.js';
+import type { EngineLike } from 'types/game.js';
+
+export interface StudioEnvironmentOptions {
+    /** Equirectangular .hdr (Radiance RGBE). */
+    hdrUrl: string;
+    /** Yaw of the environment in radians — turns the softboxes around the subject. */
+    rotationY: number;
+    /** Rim light colour / intensity / direction (world-space, pointing FROM the light). */
+    rim: { color: string; intensity: number; direction: { x: number; y: number; z: number } } | null;
+    /** Also show the HDRI as the visible sky (scene.background), rotated like the environment. */
+    background?: boolean;
+    /** A different equirect for the visible sky (`.hdr`, or an LDR .png/.jpg); the HDRI still lights. Falls back to the HDRI if it fails to load. */
+    backgroundUrl?: string;
+    backgroundIntensity?: number;
+    backgroundBlurriness?: number;
+}
+
+export const DEFAULT_STUDIO_ENVIRONMENT: StudioEnvironmentOptions = {
+    hdrUrl: '/assets/env/brown_photostudio_02_1k.hdr',
+    rotationY: 0,
+    rim: { color: '#dfe9ff', intensity: 1.6, direction: { x: -0.6, y: 0.9, z: -1.0 } },
+};
+
+type PmremLike = { fromEquirectangular(tex: THREE.Texture): THREE.RenderTarget; dispose(): void };
+
+export class StudioEnvironment {
+    private target: THREE.RenderTarget | null = null;
+    private envTexture: THREE.Texture | null = null;
+    private backgroundTexture: THREE.Texture | null = null;
+    private rimLight: THREE.DirectionalLight | null = null;
+    private unregister: (() => void) | null = null;
+
+    constructor(private readonly engine: EngineLike, private readonly options: StudioEnvironmentOptions) {}
+
+    async load(): Promise<void> {
+        const scene = this.engine.scene;
+        const renderer = this.engine.renderer;
+        if (!scene || !renderer) throw new Error('[StudioEnvironment] scene/renderer not ready');
+
+        // WebGPURenderer.init() is fired without await by the engine; PMREM needs it done.
+        const r = renderer as unknown as { hasInitialized?: () => boolean; init?: () => Promise<unknown> };
+        if (typeof r.hasInitialized === 'function' && !r.hasInitialized()) {
+            await r.init?.();
+        }
+
+        const hdr = await new RGBELoader().loadAsync(this.options.hdrUrl);
+        hdr.mapping = THREE.EquirectangularReflectionMapping;
+
+        const pmrem: PmremLike = isWebGpuActive()
+            ? new PMREMGeneratorWebGPU(renderer as unknown as ConstructorParameters<typeof PMREMGeneratorWebGPU>[0])
+            : new PMREMGeneratorWebGL(renderer);
+        this.target = pmrem.fromEquirectangular(hdr);
+        pmrem.dispose();
+        hdr.dispose();
+
+        this.envTexture = this.target.texture;
+        scene.environment = this.envTexture;
+        scene.environmentRotation.set(0, this.options.rotationY, 0);
+        if (this.options.background) {
+            // Keep an equirect for the sky: the PMREM only holds prefiltered mips. A separate sky map
+            // (the street's composite with the districts' styles on their sides) takes precedence.
+            let sky: THREE.Texture | null = null;
+            const url = this.options.backgroundUrl;
+            if (url) {
+                try {
+                    if (/\.hdr$/i.test(url)) sky = await new RGBELoader().loadAsync(url);
+                    else { sky = await new THREE.TextureLoader().loadAsync(url); sky.colorSpace = THREE.SRGBColorSpace; }
+                    console.log('[StudioEnvironment] sky map:', url);
+                } catch (err) { console.warn('[StudioEnvironment] sky map failed, using the HDRI:', url, err); sky = null; }
+            }
+            if (!sky) sky = await new RGBELoader().loadAsync(this.options.hdrUrl);
+            sky.mapping = THREE.EquirectangularReflectionMapping;
+            this.backgroundTexture = sky;
+            scene.background = sky;
+            scene.backgroundRotation.set(0, this.options.rotationY, 0);
+            scene.backgroundIntensity = this.options.backgroundIntensity ?? 1;
+            scene.backgroundBlurriness = this.options.backgroundBlurriness ?? 0;
+        }
+
+        if (this.options.rim && !this.rimLight) {
+            const d = this.options.rim.direction;
+            this.rimLight = new THREE.DirectionalLight(new THREE.Color(this.options.rim.color), this.options.rim.intensity);
+            this.rimLight.position.set(d.x, d.y, d.z).normalize().multiplyScalar(20);
+            this.rimLight.target.position.set(0, 1, 0);
+            this.rimLight.castShadow = false;
+            scene.add(this.rimLight);
+            scene.add(this.rimLight.target);
+        }
+
+        // Keep our map in place even if AmbientLighting swaps scene.environment later.
+        this.unregister = this.engine.registerBeforeRender?.(() => {
+            if (this.envTexture && scene.environment !== this.envTexture) scene.environment = this.envTexture;
+            if (this.backgroundTexture && scene.background !== this.backgroundTexture) scene.background = this.backgroundTexture;
+        }) ?? null;
+        console.log('[StudioEnvironment] HDRI environment ready:', this.options.hdrUrl);
+    }
+
+    /** Aim the rim light so it always sits behind-above the subject relative to the camera. */
+    updateRim(subject: THREE.Vector3, cameraPos: THREE.Vector3): void {
+        if (!this.rimLight) return;
+        const toCam = new THREE.Vector3().subVectors(cameraPos, subject); toCam.y = 0;
+        if (toCam.lengthSq() < 1e-6) return;
+        toCam.normalize();
+        // Opposite side of the camera, lifted, and swung 35° to one side.
+        const behind = toCam.clone().multiplyScalar(-1).applyAxisAngle(new THREE.Vector3(0, 1, 0), 0.6);
+        this.rimLight.position.copy(subject).addScaledVector(behind, 6).add(new THREE.Vector3(0, 5, 0));
+        this.rimLight.target.position.copy(subject).add(new THREE.Vector3(0, 1.2, 0));
+        this.rimLight.target.updateMatrixWorld();
+    }
+
+    setRotation(rotationY: number): void {
+        this.engine.scene?.environmentRotation.set(0, rotationY, 0);
+        this.engine.scene?.backgroundRotation.set(0, rotationY, 0);
+    }
+
+    dispose(): void {
+        this.unregister?.();
+        this.unregister = null;
+        if (this.rimLight && this.engine.scene) {
+            this.engine.scene.remove(this.rimLight);
+            this.engine.scene.remove(this.rimLight.target);
+        }
+        this.target?.dispose();
+        this.target = null;
+        this.envTexture = null;
+        this.backgroundTexture?.dispose();
+        this.backgroundTexture = null;
+    }
+}

package/assets/metahuman/tools/lipsync/make_line.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Make one spoken line for the MetaHuman lab.
+
+  ElevenLabs text-to-speech (--elevenlabs <voice_id>, key in ELEVENLABS_API_KEY), or --audio <file> you
+  generated elsewhere, or — placeholder only — macOS `say`
+    -> assets/speech/<id>.m4a          web audio
+  PocketSphinx forced alignment of the transcript -> phones -> Rhubarb-style mouth cues
+    -> assets/speech/<id>.json         {"metadata": {...}, "mouthCues": [{start, end, value}]}
+  and registers the line in assets/speech/lines.json (loaded by Game.ts; press T in the game).
+
+Usage (from the project root, with the venv in tools/lipsync/.venv):
+  tools/lipsync/.venv/bin/python tools/lipsync/make_line.py intro "Hi, I'm Bo." [--voice Samantha]
+  tools/lipsync/.venv/bin/python tools/lipsync/make_line.py greet "<transcript>" --audio path/to/tts.mp3
+  ELEVENLABS_API_KEY=... tools/lipsync/.venv/bin/python tools/lipsync/make_line.py pitch "<text>" --elevenlabs <voice_id> [--model eleven_multilingual_v2]
+"""
+import argparse, base64, json, os, re, subprocess, sys, urllib.request, wave
+
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+OUT = os.path.join(ROOT, 'assets', 'speech')
+
+# ARPAbet phone -> Rhubarb mouth shape (A closed MBP, B teeth/most consonants, C open EH,
+# D wide AA, E rounded AO/ER, F puckered UW/OW/W, G F/V, H L, X rest).
+PHONE_SHAPE = {
+    'AA': 'D', 'AE': 'C', 'AH': 'C', 'AO': 'E', 'AW': 'D', 'AY': 'D', 'EH': 'C', 'ER': 'E', 'EY': 'C',
+    'IH': 'B', 'IY': 'B', 'OW': 'E', 'OY': 'E', 'UH': 'F', 'UW': 'F',
+    'B': 'A', 'M': 'A', 'P': 'A', 'F': 'G', 'V': 'G', 'L': 'H', 'W': 'F', 'R': 'E',
+    'SIL': 'X', '<sil>': 'X', '<s>': 'X', '</s>': 'X', '+SPN+': 'X', '+NSN+': 'X', '+BREATH+': 'X',
+}
+DEFAULT_SHAPE = 'B'
+MIN_CUE = 0.03  # seconds; shorter cues are merged into their predecessor
+
+# Pronunciations for words the CMU dictionary lacks. Extend as your lines need it.
+EXTRA_WORDS = {
+    'metahuman': 'M EH T AH HH Y UW M AH N', 'metahumans': 'M EH T AH HH Y UW M AH N Z',
+    'bitmagic': 'B IH T M AE JH IH K', 'arkit': 'AA R K IH T',
+    'blendshape': 'B L EH N D SH EY P', 'blendshapes': 'B L EH N D SH EY P S',
+    'voxel': 'V AA K S AH L', 'voxels': 'V AA K S AH L Z',
+}
+ONES = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve',
+        'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen']
+TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety']
+
+def num_words(n):
+    if n < 20: return ONES[n]
+    if n < 100: return TENS[n // 10] + ('' if n % 10 == 0 else ' ' + ONES[n % 10])
+    if n < 1000: return ONES[n // 100] + ' hundred' + ('' if n % 100 == 0 else ' ' + num_words(n % 100))
+    return ' '.join(ONES[int(c)] for c in str(n))
+
+def normalize(text):
+    t = text.lower().replace('-', ' ').replace('—', ' ').replace('…', ' ')
+    t = re.sub(r'\d+', lambda m: ' ' + num_words(int(m.group())) + ' ', t)
+    t = re.sub(r"[^a-z' ]+", ' ', t)
+    return [w for w in t.split() if w]
+
+def run(cmd, **kw):
+    subprocess.run(cmd, check=True, **kw)
+
+def elevenlabs_tts(args, base):
+    """ElevenLabs `with-timestamps`: the audio plus character-level alignment (kept next to the line for
+    tooling; the mouth cues still come from the forced alignment below, which yields phones)."""
+    key = os.environ.get('ELEVENLABS_API_KEY')
+    if not key: sys.exit('[make_line] --elevenlabs needs ELEVENLABS_API_KEY in the environment')
+    api = (os.environ.get('ELEVENLABS_API_URL') or 'https://api.elevenlabs.io').rstrip('/')
+    body = json.dumps({'text': args.text, 'model_id': args.model, 'output_format': 'mp3_44100_128'}).encode()
+    req = urllib.request.Request(f'{api}/v1/text-to-speech/{args.elevenlabs}/with-timestamps', data=body,
+                                 headers={'xi-api-key': key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
+    with urllib.request.urlopen(req, timeout=120) as res:
+        payload = json.load(res)
+    mp3 = base + '.source.mp3'
+    with open(mp3, 'wb') as f: f.write(base64.b64decode(payload['audio_base64']))
+    with open(base + '.alignment.json', 'w') as f: json.dump(payload.get('alignment') or payload.get('normalized_alignment') or {}, f)
+    print(f'[make_line] ElevenLabs voice {args.elevenlabs} / {args.model}: {os.path.getsize(mp3)} bytes', file=sys.stderr)
+    return mp3
+
+def make_audio(args, base):
+    aiff = base + '.aiff'
+    if args.elevenlabs:
+        src = elevenlabs_tts(args, base)
+    elif args.audio:
+        src = args.audio
+    else:
+        print('[make_line] WARNING: macOS `say` is a PLACEHOLDER voice — use --elevenlabs or --audio for anything shipped', file=sys.stderr)
+        say = ['say', '-o', aiff, args.text]
+        if args.voice: say[1:1] = ['-v', args.voice]
+        run(say)
+        src = aiff
+    run(['ffmpeg', '-y', '-loglevel', 'error', '-i', src, '-ac', '1', '-ar', '16000', '-sample_fmt', 's16', base + '.wav'])
+    run(['ffmpeg', '-y', '-loglevel', 'error', '-i', src, '-vn', '-c:a', 'aac', '-b:a', '96k', base + '.m4a'])
+    if os.path.exists(aiff): os.remove(aiff)
+
+def align(wav_path, words):
+    from pocketsphinx import Decoder
+    with wave.open(wav_path, 'rb') as w:
+        assert w.getframerate() == 16000 and w.getnchannels() == 1 and w.getsampwidth() == 2, 'need 16 kHz mono 16-bit'
+        pcm = w.readframes(w.getnframes())
+        duration = w.getnframes() / 16000
+    dec = Decoder(samprate=16000, loglevel='ERROR')
+    try:
+        frate = int(dec.config['frate'])
+    except Exception:
+        frate = 100
+    for word, phones in EXTRA_WORDS.items():
+        dec.add_word(word, phones, True)
+    oov = [w for w in words if dec.lookup_word(w) is None]
+    if oov:
+        raise SystemExit(f'words missing from the dictionary: {oov} - add pronunciations to EXTRA_WORDS in {__file__}')
+    dec.set_align_text(' '.join(words))
+    dec.start_utt(); dec.process_raw(pcm, False, True); dec.end_utt()
+    dec.set_alignment()            # second pass: phone-level timing
+    dec.start_utt(); dec.process_raw(pcm, False, True); dec.end_utt()
+    phones = []
+    for word in dec.get_alignment():
+        for ph in word:
+            phones.append((ph.name, ph.start / frate, (ph.start + ph.duration) / frate))
+    return phones, duration
+
+def to_cues(phones, duration):
+    cues = []
+    for name, start, end in phones:
+        base = re.sub(r'\d', '', name).upper()
+        shape = PHONE_SHAPE.get(base, PHONE_SHAPE.get(name, DEFAULT_SHAPE))
+        if cues and cues[-1]['value'] == shape:
+            cues[-1]['end'] = end
+        else:
+            cues.append({'start': start, 'end': end, 'value': shape})
+    merged = []
+    for c in cues:  # merge blips into their predecessor so the mouth doesn't flicker
+        if merged and (c['end'] - c['start']) < MIN_CUE:
+            merged[-1]['end'] = c['end']
+        else:
+            merged.append(c)
+    if merged:
+        merged[-1]['end'] = max(merged[-1]['end'], duration)
+        if merged[-1]['value'] != 'X':
+            merged.append({'start': merged[-1]['end'], 'end': duration, 'value': 'X'})
+    for c in merged:
+        c['start'] = round(c['start'], 3); c['end'] = round(c['end'], 3)
+    return merged
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('id'); ap.add_argument('text')
+    ap.add_argument('--voice', default=None, help='macOS voice for `say` (default: system voice)')
+    ap.add_argument('--audio', default=None, help='use this audio file instead of `say` (any ffmpeg format)')
+    ap.add_argument('--elevenlabs', default=None, metavar='VOICE_ID', help='ElevenLabs text-to-speech with this voice id (ELEVENLABS_API_KEY in the env)')
+    ap.add_argument('--model', default='eleven_multilingual_v2', help='ElevenLabs model id (with --elevenlabs)')
+    args = ap.parse_args()
+    os.makedirs(OUT, exist_ok=True)
+    base = os.path.join(OUT, args.id)
+    with open(base + '.txt', 'w') as f: f.write(args.text)
+    make_audio(args, base)
+    words = normalize(args.text)
+    cues, duration = None, None
+    try:
+        phones, duration = align(base + '.wav', words)
+        cues = to_cues(phones, duration)
+        with open(base + '.json', 'w') as f:
+            json.dump({'metadata': {'soundFile': f'{args.id}.m4a', 'duration': round(duration, 3)}, 'mouthCues': cues}, f, indent=1)
+    except SystemExit:
+        raise
+    except Exception as e:  # keep the audio; the game falls back to amplitude lip-flap
+        print(f'[make_line] alignment failed ({e}); line registered WITHOUT cues', file=sys.stderr)
+    if duration is None:
+        duration = float(subprocess.check_output(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', base + '.m4a']))
+    os.remove(base + '.wav')
+    lines_path = os.path.join(OUT, 'lines.json')
+    lines = json.load(open(lines_path)) if os.path.exists(lines_path) else []
+    lines = [l for l in lines if l.get('id') != args.id]
+    entry = {'id': args.id, 'text': args.text, 'audioUrl': f'/assets/speech/{args.id}.m4a', 'duration': round(duration, 2)}
+    if cues: entry['cuesUrl'] = f'/assets/speech/{args.id}.json'
+    lines.append(entry)
+    json.dump(lines, open(lines_path, 'w'), indent=2)
+    if cues:
+        counts = {}
+        for c in cues: counts[c['value']] = counts.get(c['value'], 0) + 1
+        print(f"{args.id}: {duration:.2f} s, {len(cues)} cues {counts}; first: " + ' '.join(f"{c['value']}@{c['start']}" for c in cues[:14]))
+    else:
+        print(f'{args.id}: {duration:.2f} s, no cues')
+
+if __name__ == '__main__':
+    main()

package/assets/metahuman/tools/metahuman/README.md

@@ -0,0 +1,21 @@
+# MetaHuman tools
+
+Installed by `bitmagic tools install metahuman`, used by the `importing-metahumans` skill. Everything
+here runs with Node ≥ 18; `npm install` in this directory fetches gltf-transform (GLB editing) and
+playwright-core (the rigs drive the system Chrome).
+
+| File | Does |
+|---|---|
+| `prepare.sh <export.glb> <out.glb>` | the five preparation steps below, in order |
+| `strip-draco.mjs` | rewrite without Draco (the engine's loader has no decoder) + print an inventory |
+| `merge-skeletons.mjs` | graft the body hierarchy's limbs onto the face hierarchy: one skeleton |
+| `reorder-body-first.mjs` | body mesh first in the scene (the engine reads the bind pose from the first skin) |
+| `smooth-outfit-weights.mjs` | Laplacian-smooth a garment's skin weights |
+| `transfer-skin-weights.mjs` | give the garment the skin's weights wherever skin lies under it |
+| `e2e/shot.mjs [port]` | headless stills of the character + hair-follows-head and locomotion checks → `e2e/shots/` |
+| `e2e/eval.mjs "<expr>"\|probe.js [port] [shot.png]` | evaluate an expression or a probe file inside the running game |
+| `e2e/walk-probe.mjs [port]` | walk for a few seconds and print idle-vs-walking bone / garment statistics |
+| `e2e/probes/*.js` | the probes (arm pose, torn edges, helper-follow sweeps, clip shoulders, …) |
+
+The rigs need `window.__lab` in the game: `installMetaHumanLabHooks(...)` from
+`src/work/metahuman/MetaHumanLabHooks.ts` installs it. The lip-sync tool lives in `tools/lipsync/`.

package/assets/metahuman/tools/metahuman/e2e/eval.mjs

@@ -0,0 +1,34 @@
+// Evaluate a JS expression (or a .js file) inside the running game (after Play) in headless Chrome and print it.
+//   node tools/metahuman/e2e/eval.mjs "window.__lab.retargetInfo()" [port] [screenshot.png]
+//   node tools/metahuman/e2e/eval.mjs tools/metahuman/e2e/probes/arm-pose-check.js [port]
+// Env: VIEW=640x360 (viewport), DPR=2 (device scale factor), URL_SUFFIX="?debug=1" (appended to the page URL),
+//      TIMEOUT_MS=150000 (watchdog: the browser is force-closed and the process exits 2).
+import { chromium } from 'playwright-core';
+import fs from 'node:fs';
+const expr = /\.js$/.test(process.argv[2]) && fs.existsSync(process.argv[2]) ? fs.readFileSync(process.argv[2], 'utf8') : process.argv[2];
+const PORT = process.argv[3] || '3010'; const SHOT = process.argv[4] || null;
+const [VW, VH] = (process.env.VIEW || '1280x720').split('x').map(Number);
+const SUFFIX = process.env.URL_SUFFIX || '';
+const TIMEOUT_MS = Number(process.env.TIMEOUT_MS || 150000);
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const browser = await chromium.launch({ channel: 'chrome', headless: true,
+  args: ['--use-angle=metal', '--enable-unsafe-webgpu', `--window-size=${VW},${VH}`, '--autoplay-policy=no-user-gesture-required'] });
+const watchdog = setTimeout(async () => { console.log(`[eval] watchdog: ${TIMEOUT_MS} ms elapsed, closing browser`); try { await browser.close(); } catch {} process.exit(2); }, TIMEOUT_MS);
+try {
+  const page = await browser.newPage({ viewport: { width: VW, height: VH }, deviceScaleFactor: Number(process.env.DPR || 1) });
+  const MATCH = process.env.CONSOLE_MATCH ? new RegExp(process.env.CONSOLE_MATCH, 'i') : null; let errCount = 0;
+  page.on('console', (m) => { const t = m.text(); if (m.type() === 'error') { errCount++; if (errCount <= 3) console.log('PAGE[error]:', t.slice(0, 200)); } else if (MATCH && MATCH.test(t)) console.log('PAGE[' + m.type() + ']:', t.slice(0, 260)); });
+  await page.goto(`http://localhost:${PORT}/${SUFFIX}`, { waitUntil: 'load' });
+  const start = Date.now();
+  while (Date.now() - start < 60000 && !/PLAY/i.test(await page.evaluate(() => document.body.innerText || ''))) await sleep(400);
+  await sleep(600);
+  const btn = page.getByText(/^\s*PLAY\s*$/i).first(); if (await btn.count()) await btn.click({ force: true }); else await page.mouse.click(VW / 2, VH * 0.72);
+  while (Date.now() - start < 60000 && !(await page.evaluate(() => !!(window.__lab && window.__lab.actor && window.__lab.actor.getModel())))) await sleep(400);
+  await sleep(1500);
+  const result = await page.evaluate(expr);
+  console.log(JSON.stringify(result, null, 1));
+  if (SHOT) { await sleep(700); await page.screenshot({ path: SHOT }); console.log('[shot]', SHOT); }
+} finally {
+  clearTimeout(watchdog);
+  await browser.close();
+}

package/assets/metahuman/tools/metahuman/e2e/probes/arm-pose-check.js

@@ -0,0 +1,32 @@
+(() => {
+  const a = window.__lab.actor; const model = a.getModel(); window.__lab.setLife(false);
+  const V = model.position.constructor;
+  const wp = (o) => o.getWorldPosition(new V());
+  const dir = (from, to) => wp(to).sub(wp(from)).normalize();
+  const deg = (u, v) => Math.round(Math.acos(Math.max(-1, Math.min(1, u.dot(v)))) * 180 / Math.PI);
+  const b = (n) => a.getBone(n);
+  const out = {};
+  // the character's arm chain directions (world)
+  const upL = dir(b('upperarm_l'), b('lowerarm_l')), foreL = dir(b('lowerarm_l'), b('hand_l'));
+  const upR = dir(b('upperarm_r'), b('lowerarm_r')), foreR = dir(b('lowerarm_r'), b('hand_r'));
+  out.bo = { upperarm_l: upL.toArray().map((x) => +x.toFixed(2)), forearm_l: foreL.toArray().map((x) => +x.toFixed(2)), upperarm_r: upR.toArray().map((x) => +x.toFixed(2)) };
+  // Mixamo clip skeleton (the engine's hidden animation rig)
+  const pl = window.__lab.playerLoader ? window.__lab.playerLoader() : null;
+  let mix = null;
+  try {
+    const ac = pl.getAnimationController(); const mp = ac.getTrackAMixamoPlayer ? ac.getTrackAMixamoPlayer() : null;
+    const bm = mp && mp.getBoneMap(); const g = (n) => bm.get(n);
+    if (bm) { const mUp = dir(g('mixamorigLeftArm'), g('mixamorigLeftForeArm')), mFore = dir(g('mixamorigLeftForeArm'), g('mixamorigLeftHand'));
+      mix = { upperarm_l: mUp.toArray().map((x) => +x.toFixed(2)), forearm_l: mFore.toArray().map((x) => +x.toFixed(2)), angleUpper: deg(mUp, upL), angleFore: deg(mFore, foreL) }; }
+  } catch (e) { mix = 'err ' + e.message; }
+  out.mixamo = mix;
+  // Hand and elbow relative to pelvis, in the model's frame
+  const pelvis = wp(b('pelvis')); const rel = (n) => wp(b(n)).sub(pelvis).toArray().map((x) => +x.toFixed(2));
+  out.relPelvis = { hand_l: rel('hand_l'), lowerarm_l: rel('lowerarm_l'), upperarm_l: rel('upperarm_l'), hand_r: rel('hand_r'), head: rel('head') };
+  // hair vs head
+  let hair = null; model.traverse((o) => { if (!hair && o.isMesh && /^Hair_/.test(o.name)) hair = o; });
+  out.hairParent = hair && hair.parent && hair.parent.name;
+  hair.geometry.computeBoundingBox(); const c = hair.localToWorld(hair.geometry.boundingBox.getCenter(new V()));
+  out.hairCenterRelHead = c.sub(wp(b('head'))).toArray().map((x) => +x.toFixed(3));
+  return out;
+})()