@convai/embody 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/LICENSE +191 -0
- package/README.md +165 -0
- package/dist/character/createCharacter.d.ts +101 -0
- package/dist/character/createCharacter.js +163 -0
- package/dist/character/lifecycle.d.ts +55 -0
- package/dist/character/lifecycle.js +28 -0
- package/dist/character/pipeline.d.ts +46 -0
- package/dist/character/pipeline.js +29 -0
- package/dist/clips/animations.d.ts +39 -0
- package/dist/clips/animations.js +77 -0
- package/dist/decorate/types.d.ts +29 -0
- package/dist/decorate/types.js +13 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.js +14 -0
- package/dist/motion/blink.d.ts +56 -0
- package/dist/motion/blink.js +127 -0
- package/dist/motion/breathe.d.ts +72 -0
- package/dist/motion/breathe.js +93 -0
- package/dist/motion/eyeGaze.d.ts +65 -0
- package/dist/motion/eyeGaze.js +66 -0
- package/dist/motion/gaze.d.ts +212 -0
- package/dist/motion/gaze.js +467 -0
- package/dist/motion/reducedMotion.d.ts +13 -0
- package/dist/motion/reducedMotion.js +13 -0
- package/dist/motion/types.d.ts +16 -0
- package/dist/motion/types.js +1 -0
- package/dist/react/ConvaiCharacter.d.ts +7 -0
- package/dist/react/ConvaiCharacter.js +25 -0
- package/dist/react/index.d.ts +4 -0
- package/dist/react/index.js +2 -0
- package/dist/react/useCharacter.d.ts +11 -0
- package/dist/react/useCharacter.js +45 -0
- package/dist/resolve/resolver.d.ts +56 -0
- package/dist/resolve/resolver.js +45 -0
- package/dist/rig/profile.d.ts +64 -0
- package/dist/rig/profile.js +110 -0
- package/dist/rig/resolve.d.ts +33 -0
- package/dist/rig/resolve.js +76 -0
- package/dist/rig/units.d.ts +27 -0
- package/dist/rig/units.js +61 -0
- package/dist/speech/lipsync.d.ts +56 -0
- package/dist/speech/lipsync.js +139 -0
- package/package.json +109 -0
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Owns the create/cancel/dispose lifecycle of a single `createCharacter(...)`
|
|
3
|
+
* call. Framework-free — no React, no R3F, nothing that needs a renderer —
|
|
4
|
+
* so it is testable with a plain deferred promise and a fake handle.
|
|
5
|
+
*
|
|
6
|
+
* This exists because "cancel a pending async creation and dispose whatever
|
|
7
|
+
* shows up late" is behaviour, not wiring (R56): `useCharacter` would
|
|
8
|
+
* otherwise have to reimplement this cancellation guard itself, which is
|
|
9
|
+
* exactly the kind of logic the spec forbids from living in `react/`.
|
|
10
|
+
*
|
|
11
|
+
* One instance is one attempt: call `start` once, `cancel` any number of
|
|
12
|
+
* times (idempotent). To retry — e.g. a new `characterId` — create a fresh
|
|
13
|
+
* instance rather than reusing this one.
|
|
14
|
+
*/
|
|
15
|
+
export interface CharacterLifecycle<T extends {
|
|
16
|
+
dispose(): void;
|
|
17
|
+
}> {
|
|
18
|
+
/**
|
|
19
|
+
* Begin creating. `onStart` fires synchronously, before `create()` is even
|
|
20
|
+
* invoked — a caller wires it to whatever "hide the previous result" reset
|
|
21
|
+
* needs to happen the instant a new attempt begins (R57: this used to be
|
|
22
|
+
* two raw `setState(null)` calls sitting in `useCharacter`, undeclared
|
|
23
|
+
* behaviour outside the lifecycle contract; it is now owned and tested
|
|
24
|
+
* here instead). `onStart` fires exactly once, at the top of `start()` —
|
|
25
|
+
* `cancel()` never triggers it again.
|
|
26
|
+
*
|
|
27
|
+
* Resolves asynchronously into exactly one of:
|
|
28
|
+
* - `onReady(handle)`, if still active when `create()` resolves
|
|
29
|
+
* - a silent call to `handle.dispose()`, if `cancel()` ran first
|
|
30
|
+
* - `onError(error)`, if `create()` rejects and `cancel()` did not run first
|
|
31
|
+
*
|
|
32
|
+
* "Disposed" above means exactly this: `T.dispose()` is invoked. Whether
|
|
33
|
+
* that actually releases every resource the handle holds is entirely
|
|
34
|
+
* `T.dispose()`'s own contract, not something this lifecycle can verify or
|
|
35
|
+
* guarantee — it does not inspect what `dispose()` does. For
|
|
36
|
+
* `CharacterHandle` specifically, see `disposeCharacter` in
|
|
37
|
+
* `createCharacter.ts`, which is what makes that promise true in
|
|
38
|
+
* practice (until 2026-09-07 it released only a media-query listener and
|
|
39
|
+
* leaked the entire loaded GLB).
|
|
40
|
+
*/
|
|
41
|
+
start(create: () => Promise<T>, onStart: () => void, onReady: (handle: T) => void, onError: (error: Error) => void): void;
|
|
42
|
+
/**
|
|
43
|
+
* Idempotent — a second call is a harmless no-op (idempotency falls out of
|
|
44
|
+
* nulling `handle` after disposing it, not from a separate re-entrancy
|
|
45
|
+
* guard). If the handle already arrived, calls its `dispose()` now. If it
|
|
46
|
+
* is still in flight, marks this attempt cancelled so `dispose()` is
|
|
47
|
+
* called on it (never handed to `onReady`) the moment it arrives — see the
|
|
48
|
+
* "Disposed" note on `start()` above for what that promise does and does
|
|
49
|
+
* not cover.
|
|
50
|
+
*/
|
|
51
|
+
cancel(): void;
|
|
52
|
+
}
|
|
53
|
+
export declare function createCharacterLifecycle<T extends {
|
|
54
|
+
dispose(): void;
|
|
55
|
+
}>(): CharacterLifecycle<T>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export function createCharacterLifecycle() {
|
|
2
|
+
let cancelled = false;
|
|
3
|
+
let handle = null;
|
|
4
|
+
return {
|
|
5
|
+
start(create, onStart, onReady, onError) {
|
|
6
|
+
onStart();
|
|
7
|
+
create()
|
|
8
|
+
.then((created) => {
|
|
9
|
+
if (cancelled) {
|
|
10
|
+
created.dispose();
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
handle = created;
|
|
14
|
+
onReady(created);
|
|
15
|
+
})
|
|
16
|
+
.catch((error) => {
|
|
17
|
+
if (cancelled)
|
|
18
|
+
return;
|
|
19
|
+
onError(error instanceof Error ? error : new Error(String(error)));
|
|
20
|
+
});
|
|
21
|
+
},
|
|
22
|
+
cancel() {
|
|
23
|
+
cancelled = true;
|
|
24
|
+
handle?.dispose();
|
|
25
|
+
handle = null;
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The explicit frame pipeline.
|
|
3
|
+
*
|
|
4
|
+
* Order is load-bearing and each constraint below is inherited from the
|
|
5
|
+
* prior art, not invented here:
|
|
6
|
+
* 0 normalize — root scale must settle before anything reads world space
|
|
7
|
+
* 1 mixer — clips drive the skeleton
|
|
8
|
+
* 2 gaze — OVERRIDES the mixer's head rotation, never blends with it
|
|
9
|
+
* 3 additive — breathe/lean/gestures, added AFTER the mixer so nothing
|
|
10
|
+
* accumulates (the mixer rewrites every frame)
|
|
11
|
+
* 4 ik — foot lock (reserved; no system in this slice)
|
|
12
|
+
* 5 attach — hair follows the final transform (reserved)
|
|
13
|
+
* 6 morph — lipsync, eye-look, blink. Blink must run after lipsync.
|
|
14
|
+
*/
|
|
15
|
+
import type * as THREE from 'three';
|
|
16
|
+
import type { BlendshapeQueue } from '@convai/web-sdk/core';
|
|
17
|
+
import type { ResolvedRig } from '../rig/resolve.js';
|
|
18
|
+
export type Stage = 0 | 1 | 2 | 3 | 4 | 5 | 6;
|
|
19
|
+
export interface FrameContext {
|
|
20
|
+
root: THREE.Object3D;
|
|
21
|
+
rig: ResolvedRig;
|
|
22
|
+
camera?: THREE.Camera;
|
|
23
|
+
/** Present only while this character owns the conversational turn. */
|
|
24
|
+
queue?: BlendshapeQueue;
|
|
25
|
+
/**
|
|
26
|
+
* Produced by LipsyncSystem in stage 6, so stage-2 gaze reads the PREVIOUS
|
|
27
|
+
* frame's value. 16ms of lag on a gaze cue is imperceptible and far cheaper
|
|
28
|
+
* than a pre-pass that would exist only to remove it. Deliberate.
|
|
29
|
+
*/
|
|
30
|
+
speaking: boolean;
|
|
31
|
+
energy: number;
|
|
32
|
+
reducedMotion: boolean;
|
|
33
|
+
}
|
|
34
|
+
export interface System {
|
|
35
|
+
readonly stage: Stage;
|
|
36
|
+
update(dt: number, ctx: FrameContext): void;
|
|
37
|
+
}
|
|
38
|
+
export declare class Pipeline {
|
|
39
|
+
private readonly systems;
|
|
40
|
+
private dirty;
|
|
41
|
+
add(system: System): void;
|
|
42
|
+
private sortIfDirty;
|
|
43
|
+
/** Ordered view, for tests and debugging. */
|
|
44
|
+
get ordered(): readonly System[];
|
|
45
|
+
update(dt: number, ctx: FrameContext): void;
|
|
46
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export class Pipeline {
|
|
2
|
+
constructor() {
|
|
3
|
+
this.systems = [];
|
|
4
|
+
this.dirty = false;
|
|
5
|
+
}
|
|
6
|
+
add(system) {
|
|
7
|
+
this.systems.push(system);
|
|
8
|
+
this.dirty = true;
|
|
9
|
+
}
|
|
10
|
+
sortIfDirty() {
|
|
11
|
+
if (!this.dirty)
|
|
12
|
+
return;
|
|
13
|
+
// Stable sort: Array.prototype.sort is stable in every engine we target,
|
|
14
|
+
// so equal stages keep registration order — that IS the intra-stage
|
|
15
|
+
// contract, not an accident.
|
|
16
|
+
this.systems.sort((a, b) => a.stage - b.stage);
|
|
17
|
+
this.dirty = false;
|
|
18
|
+
}
|
|
19
|
+
/** Ordered view, for tests and debugging. */
|
|
20
|
+
get ordered() {
|
|
21
|
+
this.sortIfDirty();
|
|
22
|
+
return this.systems;
|
|
23
|
+
}
|
|
24
|
+
update(dt, ctx) {
|
|
25
|
+
this.sortIfDirty();
|
|
26
|
+
for (const s of this.systems)
|
|
27
|
+
s.update(dt, ctx);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AnimationController — generic clip playback and transitions.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately NOT an idle/talk switch: the API is animation → animation.
|
|
5
|
+
* `crossfadeTo(name)` moves from whatever is playing to any named clip, and
|
|
6
|
+
* higher layers (the SDK's isSpeaking, an action event, a UI button) decide
|
|
7
|
+
* which names those are via their own mapping. Loaded clips are track-filtered
|
|
8
|
+
* against the bones this model actually has — embody's dev-time validation:
|
|
9
|
+
* the clip GLBs carry richer rigs (fingers, hair) than some models.
|
|
10
|
+
*/
|
|
11
|
+
import * as THREE from 'three';
|
|
12
|
+
export interface ClipTransition {
|
|
13
|
+
/** Seconds for the crossfade. Default 0.35. */
|
|
14
|
+
duration?: number;
|
|
15
|
+
/** Play once and freeze on the last frame instead of looping. */
|
|
16
|
+
once?: boolean;
|
|
17
|
+
/** For `once` clips: crossfade back to this clip when finished. */
|
|
18
|
+
thenBackTo?: string;
|
|
19
|
+
}
|
|
20
|
+
export declare class AnimationController {
|
|
21
|
+
private readonly root;
|
|
22
|
+
readonly mixer: THREE.AnimationMixer;
|
|
23
|
+
private readonly actions;
|
|
24
|
+
private current;
|
|
25
|
+
private pendingReturn;
|
|
26
|
+
constructor(root: THREE.Object3D);
|
|
27
|
+
/** Register clips, filtering tracks to bones present on this model. */
|
|
28
|
+
register(clips: THREE.AnimationClip[]): {
|
|
29
|
+
name: string;
|
|
30
|
+
kept: number;
|
|
31
|
+
dropped: number;
|
|
32
|
+
}[];
|
|
33
|
+
get names(): string[];
|
|
34
|
+
get playing(): string | null;
|
|
35
|
+
play(name: string): void;
|
|
36
|
+
/** Generic transition: whatever is playing → `name`. Idempotent. */
|
|
37
|
+
crossfadeTo(name: string, opts?: ClipTransition): void;
|
|
38
|
+
update(delta: number): void;
|
|
39
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AnimationController — generic clip playback and transitions.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately NOT an idle/talk switch: the API is animation → animation.
|
|
5
|
+
* `crossfadeTo(name)` moves from whatever is playing to any named clip, and
|
|
6
|
+
* higher layers (the SDK's isSpeaking, an action event, a UI button) decide
|
|
7
|
+
* which names those are via their own mapping. Loaded clips are track-filtered
|
|
8
|
+
* against the bones this model actually has — embody's dev-time validation:
|
|
9
|
+
* the clip GLBs carry richer rigs (fingers, hair) than some models.
|
|
10
|
+
*/
|
|
11
|
+
import * as THREE from 'three';
|
|
12
|
+
export class AnimationController {
|
|
13
|
+
constructor(root) {
|
|
14
|
+
this.root = root;
|
|
15
|
+
this.actions = new Map();
|
|
16
|
+
this.current = null;
|
|
17
|
+
this.pendingReturn = null;
|
|
18
|
+
this.mixer = new THREE.AnimationMixer(root);
|
|
19
|
+
this.mixer.addEventListener('finished', (e) => {
|
|
20
|
+
const p = this.pendingReturn;
|
|
21
|
+
if (p && e.action === p.action) {
|
|
22
|
+
this.pendingReturn = null;
|
|
23
|
+
this.crossfadeTo(p.to);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
/** Register clips, filtering tracks to bones present on this model. */
|
|
28
|
+
register(clips) {
|
|
29
|
+
const bones = new Set();
|
|
30
|
+
this.root.traverse((o) => bones.add(o.name));
|
|
31
|
+
const report = [];
|
|
32
|
+
for (const src of clips) {
|
|
33
|
+
const kept = src.tracks.filter((t) => bones.has(t.name.split('.')[0]));
|
|
34
|
+
const dropped = src.tracks.length - kept.length;
|
|
35
|
+
if (dropped > 0)
|
|
36
|
+
console.warn(`[embody] clip "${src.name}": ${dropped}/${src.tracks.length} tracks target absent bones — dropped`);
|
|
37
|
+
const clip = new THREE.AnimationClip(src.name, src.duration, kept);
|
|
38
|
+
this.actions.set(src.name, this.mixer.clipAction(clip));
|
|
39
|
+
report.push({ name: src.name, kept: kept.length, dropped });
|
|
40
|
+
}
|
|
41
|
+
return report;
|
|
42
|
+
}
|
|
43
|
+
get names() { return [...this.actions.keys()]; }
|
|
44
|
+
get playing() { return this.current; }
|
|
45
|
+
play(name) {
|
|
46
|
+
const a = this.actions.get(name);
|
|
47
|
+
if (!a)
|
|
48
|
+
throw new Error(`[embody] unknown clip "${name}" (have: ${this.names.join(', ')})`);
|
|
49
|
+
a.reset().setLoop(THREE.LoopRepeat, Infinity).play();
|
|
50
|
+
this.current = name;
|
|
51
|
+
}
|
|
52
|
+
/** Generic transition: whatever is playing → `name`. Idempotent. */
|
|
53
|
+
crossfadeTo(name, opts = {}) {
|
|
54
|
+
if (name === this.current)
|
|
55
|
+
return;
|
|
56
|
+
const next = this.actions.get(name);
|
|
57
|
+
if (!next)
|
|
58
|
+
throw new Error(`[embody] unknown clip "${name}" (have: ${this.names.join(', ')})`);
|
|
59
|
+
const prev = this.current ? this.actions.get(this.current) : undefined;
|
|
60
|
+
const dur = opts.duration ?? 0.35;
|
|
61
|
+
next.reset();
|
|
62
|
+
if (opts.once) {
|
|
63
|
+
next.setLoop(THREE.LoopOnce, 1);
|
|
64
|
+
next.clampWhenFinished = true;
|
|
65
|
+
if (opts.thenBackTo)
|
|
66
|
+
this.pendingReturn = { to: opts.thenBackTo, action: next };
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
next.setLoop(THREE.LoopRepeat, Infinity);
|
|
70
|
+
}
|
|
71
|
+
next.play();
|
|
72
|
+
if (prev)
|
|
73
|
+
next.crossFadeFrom(prev, dur, false);
|
|
74
|
+
this.current = name;
|
|
75
|
+
}
|
|
76
|
+
update(delta) { this.mixer.update(delta); }
|
|
77
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CharacterDecorator — the sanctioned escape hatch for per-character code.
|
|
3
|
+
*
|
|
4
|
+
* The question this answers: "what if we want to change the character the way
|
|
5
|
+
* pagoda-act did — mouth constants, material tweaks?" Three tiers, in order of
|
|
6
|
+
* preference:
|
|
7
|
+
*
|
|
8
|
+
* 1. TUNING PARAMS on built-in systems (e.g. MouthTuning on the lipsync
|
|
9
|
+
* system): the common knobs, typed, documented, no scene-graph spelunking.
|
|
10
|
+
* 2. DECORATORS (this file): arbitrary code run once after the model loads,
|
|
11
|
+
* with the loaded graph and the manifest in hand — material overrides,
|
|
12
|
+
* mesh hiding, custom shaders. Client-side, versioned with the app.
|
|
13
|
+
* 3. THE PIPELINE (long-term home): fixes every consumer needs — a wrong
|
|
14
|
+
* roughness, a broken alpha mode — belong baked into the processed GLB or
|
|
15
|
+
* expressed as manifest metadata, not repeated in every app's decorator.
|
|
16
|
+
*
|
|
17
|
+
* A decorator that every consumer copies is a pipeline bug report.
|
|
18
|
+
*/
|
|
19
|
+
import type * as THREE from 'three';
|
|
20
|
+
import type { CharacterBundle, Representation } from '../resolve/resolver.js';
|
|
21
|
+
export interface DecoratorContext {
|
|
22
|
+
root: THREE.Object3D;
|
|
23
|
+
bundle: CharacterBundle;
|
|
24
|
+
representation: Representation;
|
|
25
|
+
/** Every unique material in the model, for tweak-by-name. */
|
|
26
|
+
materials: Map<string, THREE.Material>;
|
|
27
|
+
}
|
|
28
|
+
export type CharacterDecorator = (ctx: DecoratorContext) => void;
|
|
29
|
+
export declare function collectMaterials(root: THREE.Object3D): Map<string, THREE.Material>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export function collectMaterials(root) {
|
|
2
|
+
const out = new Map();
|
|
3
|
+
root.traverse((o) => {
|
|
4
|
+
const mesh = o;
|
|
5
|
+
if (!mesh.isMesh)
|
|
6
|
+
return;
|
|
7
|
+
const mats = Array.isArray(mesh.material) ? mesh.material : [mesh.material];
|
|
8
|
+
for (const m of mats)
|
|
9
|
+
if (m && m.name)
|
|
10
|
+
out.set(m.name, m);
|
|
11
|
+
});
|
|
12
|
+
return out;
|
|
13
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export { httpResolver, pickRepresentation, DEFAULT_BASE_URL } from './resolve/resolver.js';
|
|
2
|
+
export type { HttpResolverOptions } from './resolve/resolver.js';
|
|
3
|
+
export type { CharacterResolver, CharacterBundle, Representation, ModelVariant } from './resolve/resolver.js';
|
|
4
|
+
export { createCharacter } from './character/createCharacter.js';
|
|
5
|
+
export type { CharacterHandle, CreateCharacterOptions } from './character/createCharacter.js';
|
|
6
|
+
export { AnimationController } from './clips/animations.js';
|
|
7
|
+
export type { ClipTransition } from './clips/animations.js';
|
|
8
|
+
export { LipsyncSystem } from './speech/lipsync.js';
|
|
9
|
+
export type { MouthTuning } from './speech/lipsync.js';
|
|
10
|
+
export { createUnitNormalizer } from './rig/units.js';
|
|
11
|
+
export { collectMaterials } from './decorate/types.js';
|
|
12
|
+
export type { CharacterDecorator, DecoratorContext } from './decorate/types.js';
|
|
13
|
+
export { Pipeline } from './character/pipeline.js';
|
|
14
|
+
export type { Stage, System, FrameContext } from './character/pipeline.js';
|
|
15
|
+
export { setLookQuat, CONVAI_MHA_1, CC4_LEGACY, CC4_BASIS, UE_BASIS, REGISTERED_PROFILES } from './rig/profile.js';
|
|
16
|
+
export type { RigProfile, RigBasis, BoneRole, MorphRole } from './rig/profile.js';
|
|
17
|
+
export { resolveRig, writeMorph, REQUIRED_BONES } from './rig/resolve.js';
|
|
18
|
+
export type { ResolvedRig, MorphBinding } from './rig/resolve.js';
|
|
19
|
+
export { GazeSystem } from './motion/gaze.js';
|
|
20
|
+
export type { GazeConfig } from './motion/gaze.js';
|
|
21
|
+
export { EyeGazeSystem } from './motion/eyeGaze.js';
|
|
22
|
+
export type { EyeGazeConfig } from './motion/eyeGaze.js';
|
|
23
|
+
export { BlinkSystem } from './motion/blink.js';
|
|
24
|
+
export type { BlinkConfig } from './motion/blink.js';
|
|
25
|
+
export { BreathingSystem } from './motion/breathe.js';
|
|
26
|
+
export type { BreathingConfig } from './motion/breathe.js';
|
|
27
|
+
export { createReducedMotion } from './motion/reducedMotion.js';
|
|
28
|
+
export type { ReducedMotion } from './motion/reducedMotion.js';
|
|
29
|
+
export type { GazeDirection, EyeGazeDirection, GazeSource } from './motion/types.js';
|
|
30
|
+
export type { UnitNormalizer } from './rig/units.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export { httpResolver, pickRepresentation, DEFAULT_BASE_URL } from './resolve/resolver.js';
|
|
2
|
+
export { createCharacter } from './character/createCharacter.js';
|
|
3
|
+
export { AnimationController } from './clips/animations.js';
|
|
4
|
+
export { LipsyncSystem } from './speech/lipsync.js';
|
|
5
|
+
export { createUnitNormalizer } from './rig/units.js';
|
|
6
|
+
export { collectMaterials } from './decorate/types.js';
|
|
7
|
+
export { Pipeline } from './character/pipeline.js';
|
|
8
|
+
export { setLookQuat, CONVAI_MHA_1, CC4_LEGACY, CC4_BASIS, UE_BASIS, REGISTERED_PROFILES } from './rig/profile.js';
|
|
9
|
+
export { resolveRig, writeMorph, REQUIRED_BONES } from './rig/resolve.js';
|
|
10
|
+
export { GazeSystem } from './motion/gaze.js';
|
|
11
|
+
export { EyeGazeSystem } from './motion/eyeGaze.js';
|
|
12
|
+
export { BlinkSystem } from './motion/blink.js';
|
|
13
|
+
export { BreathingSystem } from './motion/breathe.js';
|
|
14
|
+
export { createReducedMotion } from './motion/reducedMotion.js';
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { FrameContext, System } from '../character/pipeline.js';
|
|
2
|
+
import type { GazeSource } from './types.js';
|
|
3
|
+
export interface BlinkConfig {
|
|
4
|
+
/** Floor on the gap between blinks (seconds). */
|
|
5
|
+
minInterval?: number;
|
|
6
|
+
/** MEDIAN of the log-normal blink gap — the long right tail gives the
|
|
7
|
+
* occasional 20-60s attentive hold seen in the reference footage. */
|
|
8
|
+
maxInterval?: number;
|
|
9
|
+
/** Phase durations (seconds), each jittered +/-15% per blink. */
|
|
10
|
+
closeDuration?: number;
|
|
11
|
+
holdDuration?: number;
|
|
12
|
+
openDuration?: number;
|
|
13
|
+
doubleBlinkChance?: number;
|
|
14
|
+
/** Reference blinks are mostly soft: 5 of 6 measured events were under 55%
|
|
15
|
+
* amplitude, so full snaps are the exception. */
|
|
16
|
+
partialBlinkChance?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Resting upper-lid droop (0 = wide stare, 1 = closed). RULING (2026-09-07
|
|
19
|
+
* fix wave): default is the rig-tuned value from the actual convai-mha-1
|
|
20
|
+
* call site (`examples/react-three-fiber/src/characters/SofiaV4.tsx:338`),
|
|
21
|
+
* not the brief's untuned 0.35 — that call site's own comment names the
|
|
22
|
+
* exact failure: "stacked on the Eye_Look_Down lid travel[,] the old 0.35
|
|
23
|
+
* default read half-asleep whenever the gaze dropped" (Sofia.tsx:238-241).
|
|
24
|
+
* Compounds with C2 (the resting droop no longer also floors the blink
|
|
25
|
+
* channel) and C4 (`DOWN_LID_SCALE` on the down-gaze morph) — all three
|
|
26
|
+
* touch this same eyelid pathway together.
|
|
27
|
+
*/
|
|
28
|
+
restingLidDroop?: number;
|
|
29
|
+
restingLowerLid?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Extra droop at full downward gaze. Modest, because the eyeLookDown
|
|
32
|
+
* morphs already carry most of the lid travel. Default is the same
|
|
33
|
+
* rig-tuned call site as `restingLidDroop` above (SofiaV4.tsx:339).
|
|
34
|
+
*/
|
|
35
|
+
gazeFollowAmount?: number;
|
|
36
|
+
enabled?: boolean;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Procedural eyelid behaviour, matched frame-by-frame against Unreal
|
|
40
|
+
* MetaHuman reference footage. Owns the WHOLE lid state, not just the blink.
|
|
41
|
+
* Registered AFTER lipsync in stage 6 so it wins over NeuroSync's own damped
|
|
42
|
+
* blinks — the pipeline enforces that by registration order, not by comment.
|
|
43
|
+
*/
|
|
44
|
+
export declare class BlinkSystem implements System {
|
|
45
|
+
private readonly gaze?;
|
|
46
|
+
readonly stage: 6;
|
|
47
|
+
private readonly cfg;
|
|
48
|
+
private clock;
|
|
49
|
+
private nextBlink;
|
|
50
|
+
/** Current blink; phases are pre-jittered when it is scheduled. */
|
|
51
|
+
private b;
|
|
52
|
+
/** Resting droop wander: a level gliding toward a re-targeted value. */
|
|
53
|
+
private droop;
|
|
54
|
+
constructor(config?: BlinkConfig, gaze?: GazeSource | undefined);
|
|
55
|
+
update(dt: number, ctx: FrameContext): void;
|
|
56
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
2
|
+
import { writeMorph } from '../rig/resolve.js';
|
|
3
|
+
const rand = (a, b) => a + Math.random() * (b - a);
|
|
4
|
+
/** +/-15% multiplicative jitter. */
|
|
5
|
+
const jitter = (v) => v * rand(0.85, 1.15);
|
|
6
|
+
/**
|
|
7
|
+
* Log-normal-ish blink gap matched to the reference footage: median ~=
|
|
8
|
+
* `median` seconds, long right tail, never below `min`. Uniform intervals are
|
|
9
|
+
* what read robotic. Sigma 0.55 in log space — 0.7 produced long dead
|
|
10
|
+
* stretches that read as "barely blinks".
|
|
11
|
+
*/
|
|
12
|
+
function blinkGap(median, min) {
|
|
13
|
+
const u = Math.random() || 1e-6;
|
|
14
|
+
const v = Math.random();
|
|
15
|
+
const g = Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
|
|
16
|
+
return Math.max(min, median * Math.exp(0.55 * g));
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Procedural eyelid behaviour, matched frame-by-frame against Unreal
|
|
20
|
+
* MetaHuman reference footage. Owns the WHOLE lid state, not just the blink.
|
|
21
|
+
* Registered AFTER lipsync in stage 6 so it wins over NeuroSync's own damped
|
|
22
|
+
* blinks — the pipeline enforces that by registration order, not by comment.
|
|
23
|
+
*/
|
|
24
|
+
export class BlinkSystem {
|
|
25
|
+
constructor(config = {}, gaze) {
|
|
26
|
+
this.gaze = gaze;
|
|
27
|
+
this.stage = 6;
|
|
28
|
+
this.clock = 0;
|
|
29
|
+
/** Current blink; phases are pre-jittered when it is scheduled. */
|
|
30
|
+
this.b = { start: -1, peak: 1, close: 0.09, hold: 0.055, open: 0.3 };
|
|
31
|
+
/** Resting droop wander: a level gliding toward a re-targeted value. */
|
|
32
|
+
this.droop = { level: 0, target: 0, next: 0 };
|
|
33
|
+
this.cfg = {
|
|
34
|
+
minInterval: 1.2, maxInterval: 3.5,
|
|
35
|
+
closeDuration: 0.09, holdDuration: 0.055, openDuration: 0.3,
|
|
36
|
+
doubleBlinkChance: 0.12, partialBlinkChance: 0.3,
|
|
37
|
+
restingLidDroop: 0.22, restingLowerLid: 0.12,
|
|
38
|
+
gazeFollowAmount: 0.08, enabled: true,
|
|
39
|
+
...config,
|
|
40
|
+
};
|
|
41
|
+
this.nextBlink = rand(1.5, this.cfg.maxInterval);
|
|
42
|
+
this.droop.level = this.cfg.restingLidDroop;
|
|
43
|
+
this.droop.target = this.cfg.restingLidDroop;
|
|
44
|
+
}
|
|
45
|
+
update(dt, ctx) {
|
|
46
|
+
if (!this.cfg.enabled)
|
|
47
|
+
return;
|
|
48
|
+
this.clock += dt;
|
|
49
|
+
const t = this.clock;
|
|
50
|
+
const b = this.b;
|
|
51
|
+
// --- blink envelope -----------------------------------------------
|
|
52
|
+
let v = 0;
|
|
53
|
+
if (!ctx.reducedMotion) {
|
|
54
|
+
// Kick off a blink when the timer elapses; roll its character. Partials
|
|
55
|
+
// are soft AND slow-opening; full blinks snap shut but still reopen
|
|
56
|
+
// lazily. Every phase is re-rolled so no two blinks match.
|
|
57
|
+
if (b.start < 0 && t >= this.nextBlink) {
|
|
58
|
+
b.start = t;
|
|
59
|
+
const partial = Math.random() < this.cfg.partialBlinkChance;
|
|
60
|
+
b.peak = partial ? rand(0.5, 0.85) : rand(0.92, 1);
|
|
61
|
+
b.close = jitter(this.cfg.closeDuration) * (partial ? rand(1.1, 1.6) : 1);
|
|
62
|
+
b.hold = jitter(this.cfg.holdDuration);
|
|
63
|
+
b.open = rand(0.8, 1.5) * this.cfg.openDuration;
|
|
64
|
+
}
|
|
65
|
+
if (b.start >= 0) {
|
|
66
|
+
const el = t - b.start;
|
|
67
|
+
if (el < b.close) {
|
|
68
|
+
// Fast close, decelerating into contact.
|
|
69
|
+
v = Math.sin((el / b.close) * (Math.PI / 2));
|
|
70
|
+
}
|
|
71
|
+
else if (el < b.close + b.hold) {
|
|
72
|
+
v = 1;
|
|
73
|
+
}
|
|
74
|
+
else if (el < b.close + b.hold + b.open) {
|
|
75
|
+
// Slow reopen with a long ease-out tail.
|
|
76
|
+
const u = (el - b.close - b.hold) / b.open;
|
|
77
|
+
v = 0.5 * (1 + Math.cos(Math.PI * u));
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
b.start = -1;
|
|
81
|
+
this.nextBlink = t + (Math.random() < this.cfg.doubleBlinkChance
|
|
82
|
+
? rand(0.35, 0.9) // quick second blink
|
|
83
|
+
: Math.min(blinkGap(this.cfg.maxInterval, this.cfg.minInterval), this.cfg.maxInterval * 2.2));
|
|
84
|
+
}
|
|
85
|
+
v *= b.peak;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
b.start = -1;
|
|
90
|
+
}
|
|
91
|
+
// --- resting droop -------------------------------------------------
|
|
92
|
+
// A slow random WANDER, not a sine: a loop, however slow, eventually
|
|
93
|
+
// reads repetitive. Every few seconds the lid tension re-targets within
|
|
94
|
+
// +/-35% of the configured droop and glides there.
|
|
95
|
+
const d = this.droop;
|
|
96
|
+
if (t >= d.next) {
|
|
97
|
+
d.target = this.cfg.restingLidDroop * rand(0.65, 1.35);
|
|
98
|
+
d.next = t + rand(3, 9);
|
|
99
|
+
}
|
|
100
|
+
d.level += (d.target - d.level) * (1 - Math.exp(-1.2 * dt));
|
|
101
|
+
// The lid follows the eye vertically: extra droop as the gaze drops, and
|
|
102
|
+
// the relaxed hood lifts away as it rises. The lift is MULTIPLICATIVE
|
|
103
|
+
// (damping the resting droop toward zero as gazeUp -> 1), not a flat
|
|
104
|
+
// subtraction — a constant offset diverges from the droop level itself
|
|
105
|
+
// and can flip sign or floor incorrectly across the resting band.
|
|
106
|
+
const gazeV = this.gaze?.direction.vertical ?? 0;
|
|
107
|
+
const gazeDown = Math.max(0, -gazeV);
|
|
108
|
+
const gazeUp = Math.max(0, gazeV);
|
|
109
|
+
const relax = THREE.MathUtils.clamp(d.level * (1 - 0.6 * gazeUp) + gazeDown * this.cfg.gazeFollowAmount, 0, 1);
|
|
110
|
+
// RULING (2026-09-07 fix wave, C2): `blink`/`blinkCorrective` get the
|
|
111
|
+
// PURE blink envelope (`v`, zero between blinks); `relax` is its own
|
|
112
|
+
// channel. Folding them together with `Math.max(relax, v)` (the old
|
|
113
|
+
// behaviour) stacked two independent morphs onto one another
|
|
114
|
+
// anatomically — the resting droop kept the lids permanently ~1/3
|
|
115
|
+
// closed AND clipped the bottom third of every blink's envelope (30% of
|
|
116
|
+
// blinks are partial, per `partialBlinkChance`, and peak there is
|
|
117
|
+
// already below 0.85). See useBlink.ts:247-250: the prior art writes
|
|
118
|
+
// `v` to `blink`/`blinkCorr` and `relax`/`restingLowerLid` to their own
|
|
119
|
+
// slots — never combined.
|
|
120
|
+
writeMorph(ctx.rig, 'blinkL', v);
|
|
121
|
+
writeMorph(ctx.rig, 'blinkR', v);
|
|
122
|
+
writeMorph(ctx.rig, 'blinkCorrectiveL', v); // CC4 only; absent = no-op
|
|
123
|
+
writeMorph(ctx.rig, 'blinkCorrectiveR', v);
|
|
124
|
+
writeMorph(ctx.rig, 'relax', relax);
|
|
125
|
+
writeMorph(ctx.rig, 'lowerLidUp', this.cfg.restingLowerLid);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { FrameContext, System } from '../character/pipeline.js';
|
|
2
|
+
export interface BreathingConfig {
|
|
3
|
+
/** Breaths per second. Default 0.22 (~13/min). */
|
|
4
|
+
rate?: number;
|
|
5
|
+
/** Peak pitch amplitude (radians) on the upper spine. Default 0.012 (~0.7 deg). */
|
|
6
|
+
amount?: number;
|
|
7
|
+
enabled?: boolean;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Subtle additive chest/spine motion so the body never reads frozen.
|
|
11
|
+
*
|
|
12
|
+
* RULING (2026-09-07 fix wave, C3, superseding an earlier fix in this same
|
|
13
|
+
* wave — see the coordinator correction in final-fix-report.md): the
|
|
14
|
+
* ORIGINAL version of this class post-multiplied the breath delta directly
|
|
15
|
+
* onto `chest.quaternion`/`waist.quaternion`, on the claim that "the mixer
|
|
16
|
+
* rewrites every frame, so multiplying in never accumulates." That's only
|
|
17
|
+
* true while an active AnimationMixer action is bound to those bones, and
|
|
18
|
+
* `createCharacter` registers clips without ever calling `.play()` — on
|
|
19
|
+
* that (default) path nothing resets the bone between frames, so the
|
|
20
|
+
* original code compounded without bound (measured ±30°+ at 30fps).
|
|
21
|
+
*
|
|
22
|
+
* A FIRST fix composed `rest * breath` absolutely every frame, using the
|
|
23
|
+
* bone's bind pose captured once at `resolveRig` time. That bounded the
|
|
24
|
+
* no-clip case but silently broke the WITH-clip case: every clip in
|
|
25
|
+
* `demo/fixtures/japanese-f-animations.glb` (Idle included, which the demo
|
|
26
|
+
* actually plays) animates `spine_02`/`spine_04` — the exact two bones this
|
|
27
|
+
* system drives — so composing onto a STATIC captured rest pose discarded
|
|
28
|
+
* the clip's own spine motion outright, every frame, on the normal
|
|
29
|
+
* (with-clip) path. That is worse than the bug it fixed: a spinning torso
|
|
30
|
+
* with no clip is unmissable on day one; silently-dropped clip spine
|
|
31
|
+
* animation on the normal path is not.
|
|
32
|
+
*
|
|
33
|
+
* Fixed properly here by having each bone remember what THIS system itself
|
|
34
|
+
* last wrote to it (`BoneAnchor.lastWritten`). Each frame:
|
|
35
|
+
* - if the bone's LIVE quaternion differs from `lastWritten`, something
|
|
36
|
+
* else (the mixer) rewrote it since — adopt the bone's current value as
|
|
37
|
+
* this frame's `base` and remember it.
|
|
38
|
+
* - if it is identical, this system's own previous write is still
|
|
39
|
+
* standing (nothing else touched the bone) — reuse the stored `base`
|
|
40
|
+
* rather than compounding on top of what was already composed.
|
|
41
|
+
* Then write `base * breath`. With an active clip continuously rewriting
|
|
42
|
+
* the bone (the normal path), `base` re-adopts the clip's live pose every
|
|
43
|
+
* frame and breath composes on top of it, exactly as the pipeline's stage-3
|
|
44
|
+
* "additive, after the mixer" contract intends. With no clip at all (the
|
|
45
|
+
* path the original bug was about), nothing ever differs from
|
|
46
|
+
* `lastWritten`, so `base` stays anchored to whatever the bone held on the
|
|
47
|
+
* first frame this system ever touched it (its bind pose, in practice) —
|
|
48
|
+
* bounded, exactly as the earlier fix intended, without needing a
|
|
49
|
+
* separately-captured rest pose at all.
|
|
50
|
+
*
|
|
51
|
+
* The comparison is EXACT (`Quaternion.equals`, not an epsilon): a
|
|
52
|
+
* `.copy()` followed by a later read of that SAME untouched object is
|
|
53
|
+
* bit-identical, so exact equality has no false-negative risk from float
|
|
54
|
+
* drift on the "nothing touched it" path. An epsilon would also risk a
|
|
55
|
+
* false negative on the "something touched it" path — a real clip whose
|
|
56
|
+
* frame-to-frame delta happens to be smaller than some chosen tolerance
|
|
57
|
+
* (e.g. a slow sway lingering near an extremum) would be missed and
|
|
58
|
+
* wrongly treated as "unchanged."
|
|
59
|
+
*/
|
|
60
|
+
export declare class BreathingSystem implements System {
|
|
61
|
+
readonly stage: 3;
|
|
62
|
+
private readonly rate;
|
|
63
|
+
private readonly amount;
|
|
64
|
+
private readonly enabled;
|
|
65
|
+
private clock;
|
|
66
|
+
private readonly q;
|
|
67
|
+
private readonly chestAnchor;
|
|
68
|
+
private readonly waistAnchor;
|
|
69
|
+
constructor(config?: BreathingConfig);
|
|
70
|
+
update(dt: number, ctx: FrameContext): void;
|
|
71
|
+
private applyBreath;
|
|
72
|
+
}
|