@convai/embody 1.0.1-beta.0 → 1.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +12 -8
- package/dist/character/createCharacter.d.ts +66 -5
- package/dist/character/createCharacter.js +93 -22
- package/dist/character/pipeline.d.ts +9 -5
- package/dist/clips/animations.d.ts +13 -1
- package/dist/clips/animations.js +28 -6
- package/dist/clips/retarget.d.ts +168 -0
- package/dist/clips/retarget.js +856 -0
- package/dist/clips/sources.d.ts +70 -0
- package/dist/clips/sources.js +119 -0
- package/dist/connect/session.d.ts +11 -0
- package/dist/connect/session.js +22 -0
- package/dist/index.d.ts +22 -1
- package/dist/index.js +12 -0
- package/dist/inspect/inspectModel.d.ts +14 -0
- package/dist/inspect/inspectModel.js +26 -0
- package/dist/inspect/morphMap.d.ts +54 -0
- package/dist/inspect/morphMap.js +140 -0
- package/dist/inspect/report.d.ts +88 -0
- package/dist/inspect/report.js +193 -0
- package/dist/motion/eyeAim.d.ts +94 -0
- package/dist/motion/eyeAim.js +276 -0
- package/dist/motion/eyeGaze.d.ts +17 -0
- package/dist/motion/eyeGaze.js +42 -0
- package/dist/motion/footLock.d.ts +67 -0
- package/dist/motion/footLock.js +176 -0
- package/dist/motion/gaze.d.ts +135 -180
- package/dist/motion/gaze.js +257 -491
- package/dist/motion/hair.d.ts +27 -0
- package/dist/motion/hair.js +127 -0
- package/dist/motion/hairClearance.d.ts +39 -0
- package/dist/motion/hairClearance.js +125 -0
- package/dist/motion/morphNormals.d.ts +38 -0
- package/dist/motion/morphNormals.js +237 -0
- package/dist/motion/types.d.ts +15 -0
- package/dist/react/CharacterModel.d.ts +3 -1
- package/dist/react/CharacterModel.js +9 -1
- package/dist/react/ConvaiCharacter.js +4 -1
- package/dist/react/PortraitLights.d.ts +3 -0
- package/dist/react/PortraitLights.js +44 -0
- package/dist/render/faceMaterials.d.ts +60 -3
- package/dist/render/faceMaterials.js +320 -96
- package/dist/render/portraitRig.d.ts +28 -0
- package/dist/render/portraitRig.js +55 -0
- package/dist/render/surfaces.d.ts +181 -0
- package/dist/render/surfaces.js +1020 -0
- package/dist/rig/resolve.d.ts +8 -2
- package/dist/rig/resolve.js +7 -3
- package/dist/rig/units.d.ts +35 -16
- package/dist/rig/units.js +143 -42
- package/dist/speech/expression.d.ts +20 -0
- package/dist/speech/expression.js +118 -0
- package/dist/speech/lipsync.d.ts +68 -2
- package/dist/speech/lipsync.js +205 -5
- package/dist/speech/mouthFinish.d.ts +46 -0
- package/dist/speech/mouthFinish.js +235 -0
- package/dist/speech/voiceClock.d.ts +94 -0
- package/dist/speech/voiceClock.js +176 -0
- package/package.json +1 -1
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Foot locking: a planted foot stays where it was put while the body sways
|
|
3
|
+
* above it.
|
|
4
|
+
*
|
|
5
|
+
* Idle and talking clips sway the hips, and with the body held in place the
|
|
6
|
+
* planted feet skate along the floor (~19 cm/s measured with jointQA on a
|
|
7
|
+
* convai-mha-1 body). Each planted foot is held at its lock point horizontally
|
|
8
|
+
* by a two-bone leg IK (thigh, calf); the clip keeps the foot's height and
|
|
9
|
+
* orientation. A foot that lifts or moves on purpose — a step — is released
|
|
10
|
+
* and blended back to the clip. Engaging and releasing run on minimum-jerk
|
|
11
|
+
* curves, so neither is a pop.
|
|
12
|
+
*
|
|
13
|
+
* A straight leg cannot reach a lock the hips have swayed away from; the hips
|
|
14
|
+
* are lowered by the shortfall (a few millimetres, capped) so the knee takes
|
|
15
|
+
* it instead of the foot dragging.
|
|
16
|
+
*
|
|
17
|
+
* Stage 4, after gaze and breathing. Units are world metres (the body has been
|
|
18
|
+
* scaled to metres by then). The values are the coffee-shop studio's, tuned
|
|
19
|
+
* with its joint QA.
|
|
20
|
+
*/
|
|
21
|
+
import * as THREE from 'three';
|
|
22
|
+
import type { FrameContext, System } from '../character/pipeline.js';
|
|
23
|
+
export interface FootLockConfig {
|
|
24
|
+
/** Static gate. Default `true`. */
|
|
25
|
+
enabled?: boolean;
|
|
26
|
+
/** A foot this far (m) above its resting height counts as lifted. Default 0.03. */
|
|
27
|
+
liftHeight?: number;
|
|
28
|
+
/** A foot the clip moves faster than this (m/s) is stepping. Default 0.3. */
|
|
29
|
+
stepSpeed?: number;
|
|
30
|
+
/** Seconds to engage or release a lock. Default 0.4. */
|
|
31
|
+
blendTime?: number;
|
|
32
|
+
/** Most the hips are lowered to let a straight leg reach its lock, m. Default 0.015. */
|
|
33
|
+
maxDrop?: number;
|
|
34
|
+
}
|
|
35
|
+
export declare class FootLockSystem implements System {
|
|
36
|
+
private readonly cfg;
|
|
37
|
+
readonly stage: 4;
|
|
38
|
+
private legs;
|
|
39
|
+
private drop;
|
|
40
|
+
private readonly clipPose;
|
|
41
|
+
private readonly written;
|
|
42
|
+
private readonly a;
|
|
43
|
+
private readonly b;
|
|
44
|
+
private readonly c;
|
|
45
|
+
private readonly p;
|
|
46
|
+
private readonly q;
|
|
47
|
+
private readonly q2;
|
|
48
|
+
constructor(cfg?: FootLockConfig);
|
|
49
|
+
/** Planted legs, for tests and debugging. */
|
|
50
|
+
get locks(): {
|
|
51
|
+
lock: THREE.Vector3 | null;
|
|
52
|
+
weight: number;
|
|
53
|
+
}[];
|
|
54
|
+
/**
|
|
55
|
+
* Put back the clip pose on leg bones this system wrote and the mixer is not
|
|
56
|
+
* about to rewrite (three's mixer skips a bone whose blended value did not
|
|
57
|
+
* change, and the IK would then be solved again on top of itself).
|
|
58
|
+
*/
|
|
59
|
+
beforeMixer(): void;
|
|
60
|
+
update(dt: number, ctx: FrameContext): void;
|
|
61
|
+
private findLegs;
|
|
62
|
+
private copyInto;
|
|
63
|
+
/** Two-bone IK: thigh and calf turn so the foot reaches `goal`, the knee bending forward. */
|
|
64
|
+
private solve;
|
|
65
|
+
/** Apply a world-space rotation to a bone, expressed in its parent's frame. */
|
|
66
|
+
private rotateWorld;
|
|
67
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Foot locking: a planted foot stays where it was put while the body sways
|
|
3
|
+
* above it.
|
|
4
|
+
*
|
|
5
|
+
* Idle and talking clips sway the hips, and with the body held in place the
|
|
6
|
+
* planted feet skate along the floor (~19 cm/s measured with jointQA on a
|
|
7
|
+
* convai-mha-1 body). Each planted foot is held at its lock point horizontally
|
|
8
|
+
* by a two-bone leg IK (thigh, calf); the clip keeps the foot's height and
|
|
9
|
+
* orientation. A foot that lifts or moves on purpose — a step — is released
|
|
10
|
+
* and blended back to the clip. Engaging and releasing run on minimum-jerk
|
|
11
|
+
* curves, so neither is a pop.
|
|
12
|
+
*
|
|
13
|
+
* A straight leg cannot reach a lock the hips have swayed away from; the hips
|
|
14
|
+
* are lowered by the shortfall (a few millimetres, capped) so the knee takes
|
|
15
|
+
* it instead of the foot dragging.
|
|
16
|
+
*
|
|
17
|
+
* Stage 4, after gaze and breathing. Units are world metres (the body has been
|
|
18
|
+
* scaled to metres by then). The values are the coffee-shop studio's, tuned
|
|
19
|
+
* with its joint QA.
|
|
20
|
+
*/
|
|
21
|
+
import * as THREE from 'three';
|
|
22
|
+
import { minJerk } from './gaze.js';
|
|
23
|
+
/** A forward knee bends negatively about (hip→foot × body forward). */
|
|
24
|
+
const KNEE = -1;
|
|
25
|
+
/** A held lock follows the clip's own slow foot drift at this speed, m/s (below any visible slide). */
|
|
26
|
+
const FOLLOW = 0.01;
|
|
27
|
+
export class FootLockSystem {
|
|
28
|
+
constructor(cfg = {}) {
|
|
29
|
+
this.cfg = cfg;
|
|
30
|
+
this.stage = 4;
|
|
31
|
+
this.legs = null;
|
|
32
|
+
this.drop = 0;
|
|
33
|
+
this.clipPose = new Map();
|
|
34
|
+
this.written = new Map();
|
|
35
|
+
this.a = new THREE.Vector3();
|
|
36
|
+
this.b = new THREE.Vector3();
|
|
37
|
+
this.c = new THREE.Vector3();
|
|
38
|
+
this.p = new THREE.Vector3();
|
|
39
|
+
this.q = new THREE.Quaternion();
|
|
40
|
+
this.q2 = new THREE.Quaternion();
|
|
41
|
+
}
|
|
42
|
+
/** Planted legs, for tests and debugging. */
|
|
43
|
+
get locks() {
|
|
44
|
+
return (this.legs ?? []).map((l) => ({ lock: l.lock, weight: l.w }));
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Put back the clip pose on leg bones this system wrote and the mixer is not
|
|
48
|
+
* about to rewrite (three's mixer skips a bone whose blended value did not
|
|
49
|
+
* change, and the IK would then be solved again on top of itself).
|
|
50
|
+
*/
|
|
51
|
+
beforeMixer() {
|
|
52
|
+
for (const [bone, clip] of this.clipPose)
|
|
53
|
+
if (this.written.get(bone)?.equals(bone.quaternion))
|
|
54
|
+
bone.quaternion.copy(clip);
|
|
55
|
+
}
|
|
56
|
+
update(dt, ctx) {
|
|
57
|
+
if (this.cfg.enabled === false || dt <= 0)
|
|
58
|
+
return;
|
|
59
|
+
this.legs ??= this.findLegs(ctx.rig);
|
|
60
|
+
if (!this.legs.length)
|
|
61
|
+
return;
|
|
62
|
+
dt = Math.min(dt, 0.1);
|
|
63
|
+
const root = ctx.root;
|
|
64
|
+
const liftH = this.cfg.liftHeight ?? 0.03;
|
|
65
|
+
const stepV = this.cfg.stepSpeed ?? 0.3;
|
|
66
|
+
root.updateMatrixWorld(true);
|
|
67
|
+
// Knee slack for a leg too straight to reach its lock.
|
|
68
|
+
let need = 0;
|
|
69
|
+
for (const l of this.legs) {
|
|
70
|
+
if (!l.lock || l.w < 0.05)
|
|
71
|
+
continue;
|
|
72
|
+
const A = l.thigh.getWorldPosition(this.a), B = l.calf.getWorldPosition(this.b), C = l.foot.getWorldPosition(this.c);
|
|
73
|
+
const reach = A.distanceTo(B) + B.distanceTo(C);
|
|
74
|
+
need = Math.max(need, (A.distanceTo(this.p.set(l.lock.x, C.y, l.lock.z)) - reach * 0.995 + 0.003) * l.w);
|
|
75
|
+
}
|
|
76
|
+
const goalDrop = THREE.MathUtils.clamp(need, 0, this.cfg.maxDrop ?? 0.015);
|
|
77
|
+
this.drop += (goalDrop - this.drop) * (1 - Math.exp(-dt / 0.15));
|
|
78
|
+
if (this.drop > 1e-5) {
|
|
79
|
+
const s = root.parent ? root.parent.getWorldScale(this.p).y : 1;
|
|
80
|
+
root.position.y -= this.drop / s;
|
|
81
|
+
root.updateMatrixWorld(true);
|
|
82
|
+
}
|
|
83
|
+
const fwd = this.p.set(0, 0, 1).applyQuaternion(root.getWorldQuaternion(this.q)).clone();
|
|
84
|
+
for (const l of this.legs) {
|
|
85
|
+
for (const bone of [l.thigh, l.calf, l.foot])
|
|
86
|
+
this.copyInto(this.clipPose, bone);
|
|
87
|
+
const p = l.foot.getWorldPosition(new THREE.Vector3());
|
|
88
|
+
const raw = l.prev ? Math.hypot(p.x - l.prev.x, p.z - l.prev.z) / dt : 0;
|
|
89
|
+
l.v += (raw - l.v) * (1 - Math.exp(-dt / 0.04));
|
|
90
|
+
l.prev = p.clone();
|
|
91
|
+
// The lowest the foot has been, slowly forgiving.
|
|
92
|
+
l.restY = l.restY === null ? p.y : Math.min(l.restY + dt * 0.01, p.y);
|
|
93
|
+
// Hysteresis: plant below the thresholds, release only clearly above.
|
|
94
|
+
const planted = l.lock
|
|
95
|
+
? p.y < l.restY + liftH * 1.6 && l.v < stepV * 1.4
|
|
96
|
+
: p.y < l.restY + liftH && l.v < stepV;
|
|
97
|
+
if (planted && !l.lock)
|
|
98
|
+
l.lock = p.clone();
|
|
99
|
+
if (l.lock) {
|
|
100
|
+
// Follow the clip's own slow drift, so the IK never holds a foot far from it.
|
|
101
|
+
const dx = p.x - l.lock.x, dz = p.z - l.lock.z, dist = Math.hypot(dx, dz);
|
|
102
|
+
if (dist > 1e-6) {
|
|
103
|
+
const step = Math.min(dist, FOLLOW * dt);
|
|
104
|
+
l.lock.x += (dx / dist) * step;
|
|
105
|
+
l.lock.z += (dz / dist) * step;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const target = planted ? 1 : 0;
|
|
109
|
+
if (target !== l.wTo) {
|
|
110
|
+
l.wFrom = l.w;
|
|
111
|
+
l.wTo = target;
|
|
112
|
+
l.wT = 0;
|
|
113
|
+
}
|
|
114
|
+
l.wT = Math.min(1, l.wT + dt / (this.cfg.blendTime ?? 0.4));
|
|
115
|
+
l.w = l.wFrom + (l.wTo - l.wFrom) * minJerk(l.wT);
|
|
116
|
+
if (!planted && l.w < 0.01)
|
|
117
|
+
l.lock = null;
|
|
118
|
+
if (!l.lock || l.w < 0.001)
|
|
119
|
+
continue;
|
|
120
|
+
// Goal: the lock horizontally (by its weight), the clip's own height.
|
|
121
|
+
const goal = new THREE.Vector3(l.lock.x, p.y, l.lock.z).lerp(p, 1 - l.w);
|
|
122
|
+
if (goal.distanceToSquared(p) < 1e-10)
|
|
123
|
+
continue;
|
|
124
|
+
const footQ = l.foot.getWorldQuaternion(new THREE.Quaternion());
|
|
125
|
+
this.solve(l, goal, fwd);
|
|
126
|
+
// The foot keeps the world orientation the clip gave it.
|
|
127
|
+
l.foot.quaternion.copy(l.foot.parent.getWorldQuaternion(this.q).invert().multiply(footQ));
|
|
128
|
+
l.foot.updateMatrixWorld(true);
|
|
129
|
+
for (const bone of [l.thigh, l.calf, l.foot])
|
|
130
|
+
this.copyInto(this.written, bone);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
findLegs(rig) {
|
|
134
|
+
const b = rig.bones;
|
|
135
|
+
const legs = [];
|
|
136
|
+
for (const [thigh, calf, foot] of [[b.thighL, b.calfL, b.footL], [b.thighR, b.calfR, b.footR]]) {
|
|
137
|
+
if (thigh && calf && foot)
|
|
138
|
+
legs.push({ thigh, calf, foot, lock: null, w: 0, wFrom: 0, wTo: 0, wT: 1, v: 0, restY: null, prev: null });
|
|
139
|
+
}
|
|
140
|
+
return legs;
|
|
141
|
+
}
|
|
142
|
+
copyInto(map, bone) {
|
|
143
|
+
let q = map.get(bone);
|
|
144
|
+
if (!q)
|
|
145
|
+
map.set(bone, (q = new THREE.Quaternion()));
|
|
146
|
+
q.copy(bone.quaternion);
|
|
147
|
+
}
|
|
148
|
+
/** Two-bone IK: thigh and calf turn so the foot reaches `goal`, the knee bending forward. */
|
|
149
|
+
solve(l, goal, fwd) {
|
|
150
|
+
const A = l.thigh.getWorldPosition(this.a), B = l.calf.getWorldPosition(this.b), C = l.foot.getWorldPosition(this.c);
|
|
151
|
+
const l1 = A.distanceTo(B), l2 = B.distanceTo(C);
|
|
152
|
+
const d = THREE.MathUtils.clamp(A.distanceTo(goal), Math.abs(l1 - l2) + 1e-4, l1 + l2 - 1e-4);
|
|
153
|
+
// The knee hinges across the body's forward direction: taken from the
|
|
154
|
+
// leg's own shape, a nearly straight leg's cross product was tiny and
|
|
155
|
+
// noisy, flipped sign, and the knee solved to the wrong side for a frame.
|
|
156
|
+
const n = C.clone().sub(A).normalize().cross(fwd);
|
|
157
|
+
if (n.lengthSq() < 1e-8)
|
|
158
|
+
return;
|
|
159
|
+
n.normalize();
|
|
160
|
+
const u1 = B.clone().sub(A).normalize(), u2 = C.clone().sub(B).normalize();
|
|
161
|
+
const cur = Math.atan2(u1.clone().cross(u2).dot(n), u1.dot(u2));
|
|
162
|
+
const want = KNEE * (Math.PI - Math.acos(THREE.MathUtils.clamp((l1 * l1 + l2 * l2 - d * d) / (2 * l1 * l2), -1, 1)));
|
|
163
|
+
this.rotateWorld(l.calf, this.q2.setFromAxisAngle(n, want - cur));
|
|
164
|
+
l.thigh.updateMatrixWorld(true);
|
|
165
|
+
const from = l.foot.getWorldPosition(new THREE.Vector3()).sub(A).normalize();
|
|
166
|
+
const to = goal.clone().sub(A).normalize();
|
|
167
|
+
this.rotateWorld(l.thigh, this.q2.setFromUnitVectors(from, to));
|
|
168
|
+
}
|
|
169
|
+
/** Apply a world-space rotation to a bone, expressed in its parent's frame. */
|
|
170
|
+
rotateWorld(bone, qWorld) {
|
|
171
|
+
const parentQ = bone.parent.getWorldQuaternion(new THREE.Quaternion());
|
|
172
|
+
const world = parentQ.clone().multiply(bone.quaternion);
|
|
173
|
+
bone.quaternion.copy(parentQ.invert().multiply(qWorld.clone().multiply(world)));
|
|
174
|
+
bone.updateMatrixWorld(true);
|
|
175
|
+
}
|
|
176
|
+
}
|
package/dist/motion/gaze.d.ts
CHANGED
|
@@ -1,220 +1,175 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
1
2
|
import type { FrameContext, System } from '../character/pipeline.js';
|
|
2
3
|
import type { EyeGazeDirection, GazeDirection, GazeSource } from './types.js';
|
|
3
4
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
5
|
+
* Gaze the way people do it: eyes, then neck, then body.
|
|
6
|
+
*
|
|
7
|
+
* · The eyes take every small change on their own (EyeGazeSystem aims each
|
|
8
|
+
* one at `focus`), so the head is not moved by a few degrees of camera.
|
|
9
|
+
* · Beyond that the head follows smoothly, lagging the eyes (smooth pursuit),
|
|
10
|
+
* so a gliding camera is tracked without the head holding and then
|
|
11
|
+
* snapping. A sudden large jump gets a quick eye-led head shift on a
|
|
12
|
+
* minimum-jerk curve, ~80 ms after the eyes, stopping a few degrees short
|
|
13
|
+
* for the eyes to finish.
|
|
14
|
+
* · The torso turns only once the head has had to twist far off the body for
|
|
15
|
+
* a moment; it turns slowly while the neck unwinds by the same amount, so
|
|
16
|
+
* the head stays on target. When the target comes back toward the front,
|
|
17
|
+
* the torso drifts home the same way.
|
|
18
|
+
* · While fixating, most of the clip's fast head sway is cancelled (people
|
|
19
|
+
* hold the head still while looking at something); the slow part of it,
|
|
20
|
+
* and everything below the neck, plays as authored.
|
|
21
|
+
*
|
|
22
|
+
* Every turn is ADDED to the clip's pose, so idle sway and talking gestures
|
|
23
|
+
* stay alive. A bone the clip did not key this frame still holds last
|
|
24
|
+
* frame's output; that is detected and the clip pose restored before adding,
|
|
25
|
+
* so nothing accumulates.
|
|
26
|
+
*
|
|
27
|
+
* Angles are measured from the eyes' own neutral frame (EyeAim, from the
|
|
28
|
+
* body's eye geometry) where the body allows, otherwise from the rig basis.
|
|
29
|
+
* The values are the coffee-shop studio's, tuned on camera.
|
|
12
30
|
*/
|
|
13
31
|
export interface GazeConfig {
|
|
14
|
-
/**
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
* which the implementation itself does not use. Per RULING R20: the
|
|
18
|
-
* DEFAULTS object is ground truth, and its own comment explains 0.44 was
|
|
19
|
-
* a deliberate retune down from a wider zone that put the head near its
|
|
20
|
-
* joint limit for a side viewer. */
|
|
21
|
-
headOnlyAngle?: number;
|
|
22
|
-
/** Beyond this yaw (rad) tracking disengages entirely for the frame
|
|
23
|
-
* (target clamped to zero, not clamped-in-range). */
|
|
32
|
+
/** Static construction-time gate. Live control is `setEnabled()`. */
|
|
33
|
+
enabled?: boolean;
|
|
34
|
+
/** Beyond this angle (rad) off the body's front the character stops attending and turns home. Default PI/2. */
|
|
24
35
|
maxTrackAngle?: number;
|
|
25
|
-
/**
|
|
36
|
+
/** Changes inside this (degrees) are left to the eyes. Default 5 yaw, 4 pitch. */
|
|
37
|
+
eyeZone?: {
|
|
38
|
+
yaw: number;
|
|
39
|
+
pitch: number;
|
|
40
|
+
};
|
|
41
|
+
/** A target this far (degrees) outside the face gets a quick head shift. Default 18 yaw, 12 pitch. */
|
|
42
|
+
shiftZone?: {
|
|
43
|
+
yaw: number;
|
|
44
|
+
pitch: number;
|
|
45
|
+
};
|
|
46
|
+
/** Seconds the head lags the eyes in smooth pursuit. Default 0.45. */
|
|
47
|
+
followTime?: number;
|
|
48
|
+
/** How far the head and neck turn from the body, degrees. Default 45 yaw, 25 pitch. */
|
|
49
|
+
headLimit?: {
|
|
50
|
+
yaw: number;
|
|
51
|
+
pitch: number;
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Torso turn: after the head has been twisted past `trigger` degrees for
|
|
55
|
+
* `hold` seconds the torso turns, at most `limit` degrees. `false` keeps the
|
|
56
|
+
* torso out of it. Default `{ trigger: 32, hold: 0.6, limit: 30 }`.
|
|
57
|
+
*/
|
|
58
|
+
torso?: false | {
|
|
59
|
+
trigger?: number;
|
|
60
|
+
hold?: number;
|
|
61
|
+
limit?: number;
|
|
62
|
+
};
|
|
63
|
+
/** Share of the clip's fast head sway cancelled while fixating, 0–1. Default 0.8. */
|
|
64
|
+
steady?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Occasional short glances away while listening (never while speaking or
|
|
67
|
+
* under reduced motion). Default `true`.
|
|
68
|
+
*/
|
|
69
|
+
lookAways?: boolean;
|
|
70
|
+
/** @deprecated Ignored since 1.1: the head is driven by pursuit and shifts, see `eyeZone`/`followTime`. */
|
|
71
|
+
headOnlyAngle?: number;
|
|
72
|
+
/** @deprecated Ignored since 1.1; see `headLimit.pitch`. */
|
|
26
73
|
maxPitchAngle?: number;
|
|
27
|
-
/**
|
|
28
|
-
* `update()`) toward the target yaw/pitch. Velocity-capped — see
|
|
29
|
-
* `update()`. */
|
|
74
|
+
/** @deprecated Ignored since 1.1; see `followTime`. */
|
|
30
75
|
lerpSpeed?: number;
|
|
31
|
-
/**
|
|
76
|
+
/** @deprecated Ignored since 1.1. */
|
|
32
77
|
headPitchFactor?: number;
|
|
33
|
-
/**
|
|
78
|
+
/** @deprecated Ignored since 1.1. */
|
|
34
79
|
neckPitchFactor?: number;
|
|
35
|
-
/**
|
|
36
|
-
* Verbatim from `DEFAULT_CONFIG.neckYawFactor` (useHeadTracking.ts:113),
|
|
37
|
-
* which is 0.75, not the interface JSDoc's 0.7 (line 30). Per R20. */
|
|
80
|
+
/** @deprecated Ignored since 1.1. */
|
|
38
81
|
neckYawFactor?: number;
|
|
39
|
-
/**
|
|
82
|
+
/** @deprecated Ignored since 1.1: the clip's own motion carries the head's life. */
|
|
40
83
|
idleYawAmp?: number;
|
|
41
|
-
/**
|
|
84
|
+
/** @deprecated Ignored since 1.1. */
|
|
42
85
|
idlePitchAmp?: number;
|
|
43
|
-
/**
|
|
86
|
+
/** @deprecated Ignored since 1.1. */
|
|
44
87
|
idleRollAmp?: number;
|
|
45
|
-
/**
|
|
88
|
+
/** @deprecated Ignored since 1.1. */
|
|
46
89
|
idleSpeed?: number;
|
|
47
|
-
/**
|
|
48
|
-
* turn. */
|
|
90
|
+
/** @deprecated Ignored since 1.1. */
|
|
49
91
|
turnBank?: number;
|
|
50
|
-
/**
|
|
92
|
+
/** @deprecated Ignored since 1.1; see `lookAways`. */
|
|
51
93
|
gazeYawRange?: number;
|
|
52
|
-
/**
|
|
94
|
+
/** @deprecated Ignored since 1.1; see `lookAways`. */
|
|
53
95
|
gazePitchRange?: number;
|
|
54
|
-
/**
|
|
96
|
+
/** @deprecated Ignored since 1.1. */
|
|
55
97
|
gazeRollRange?: number;
|
|
56
|
-
/**
|
|
57
|
-
* gaze-shift offset eases toward its target at. */
|
|
98
|
+
/** @deprecated Ignored since 1.1. */
|
|
58
99
|
gazeEase?: number;
|
|
59
|
-
/**
|
|
100
|
+
/** @deprecated Ignored since 1.1. */
|
|
60
101
|
gazeDwellMin?: number;
|
|
61
|
-
/**
|
|
102
|
+
/** @deprecated Ignored since 1.1. */
|
|
62
103
|
gazeDwellMax?: number;
|
|
63
|
-
/**
|
|
64
|
-
* head rotation is replaced by the look pose. High values mean
|
|
65
|
-
* procedural drift/gaze carry the aliveness and the clip barely
|
|
66
|
-
* reaches the output. */
|
|
104
|
+
/** @deprecated Ignored since 1.1: turns are added to the clip, never locked over it. */
|
|
67
105
|
idleLock?: number;
|
|
68
|
-
/**
|
|
106
|
+
/** @deprecated Ignored since 1.1. */
|
|
69
107
|
talkLock?: number;
|
|
70
|
-
/**
|
|
71
|
-
* look pose) below which the clip flows through untouched. */
|
|
108
|
+
/** @deprecated Ignored since 1.1. */
|
|
72
109
|
deviationLo?: number;
|
|
73
|
-
/**
|
|
74
|
-
* is fully locked back to the look pose regardless of `idleLock`/
|
|
75
|
-
* `talkLock`. */
|
|
110
|
+
/** @deprecated Ignored since 1.1. */
|
|
76
111
|
deviationHi?: number;
|
|
77
|
-
/** Static construction-time gate. Live control is `setEnabled()`. */
|
|
78
|
-
enabled?: boolean;
|
|
79
112
|
}
|
|
80
|
-
/**
|
|
81
|
-
|
|
82
|
-
* (convai-web-sdk-internal `examples/react-three-fiber/src/hooks/useHeadTracking.ts`).
|
|
83
|
-
*
|
|
84
|
-
* Stage 2 — the ONLY system permitted to OVERRIDE the mixer's head/neck
|
|
85
|
-
* rotation outright (`.copy(...)`, never `.multiply(...)`): a head that
|
|
86
|
-
* blends additively with its own animation clip fights itself every frame.
|
|
87
|
-
*
|
|
88
|
-
* Task 6 ported the geometric core: world target -> yaw/pitch about the
|
|
89
|
-
* rig's own basis -> head/neck split -> time-based, velocity-capped ease ->
|
|
90
|
-
* `setLookQuat`. Task 7 layered the aliveness on top of that same solve:
|
|
91
|
-
* always-on idle micro-drift, occasional look-away gaze shifts with dwell
|
|
92
|
-
* timing, and turn-bank roll — all suppressed under `ctx.reducedMotion`.
|
|
93
|
-
* Task 8 (this task) adds the clip-blending layer: a smoothed camera-lock
|
|
94
|
-
* strength (`idleLock`/`talkLock`, eased rather than switched — see
|
|
95
|
-
* `update()`) blended against the mixer's own clip pose, with a deviation
|
|
96
|
-
* pull-back (`deviationLo`/`deviationHi`) that guards large clip excursions
|
|
97
|
-
* regardless of lock strength; and `calm`, which de-emphasises the idle
|
|
98
|
-
* drift/gaze-shift amplitudes while the (smoothed) lock is high, i.e. while
|
|
99
|
-
* speaking — a focused speaker stops glancing around.
|
|
100
|
-
*
|
|
101
|
-
* The HEAD is the only bone that blends with the clip; the neck bones are
|
|
102
|
-
* still overridden outright (Task 6/7 behaviour, unchanged) — see
|
|
103
|
-
* `update()`.
|
|
104
|
-
*
|
|
105
|
-
* See `.superpowers/sdd/2026-09-07-convai-embody-motion-and-staging/task-6-report.md`,
|
|
106
|
-
* `task-7-report.md` and `task-8-report.md` for the line-by-line comparison
|
|
107
|
-
* against the prior art and the places this deliberately differs from it.
|
|
108
|
-
*/
|
|
113
|
+
/** Minimum-jerk ease: 0→1 with zero velocity and acceleration at both ends. */
|
|
114
|
+
export declare const minJerk: (s: number) => number;
|
|
109
115
|
export declare class GazeSystem implements System, GazeSource {
|
|
110
116
|
private readonly cfg;
|
|
111
117
|
readonly stage: 2;
|
|
112
118
|
readonly direction: GazeDirection;
|
|
113
|
-
/** World-space angle still between the head's forward and the target. */
|
|
114
119
|
readonly residual: EyeGazeDirection;
|
|
120
|
+
/** See `GazeSource.attending`. */
|
|
121
|
+
attending: boolean;
|
|
122
|
+
/** The world point the character is looking at this frame; see `GazeSource.focus`. */
|
|
123
|
+
readonly focus: THREE.Vector3;
|
|
115
124
|
private gate;
|
|
116
|
-
private
|
|
117
|
-
private
|
|
118
|
-
|
|
119
|
-
private
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
private
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
private
|
|
127
|
-
private
|
|
128
|
-
private
|
|
129
|
-
/** The
|
|
130
|
-
|
|
131
|
-
private
|
|
132
|
-
private
|
|
133
|
-
private
|
|
134
|
-
|
|
135
|
-
private
|
|
136
|
-
|
|
137
|
-
* each frame (RULING R39) rather than switched instantly — ported
|
|
138
|
-
* from `useHeadTracking.ts:581`'s `lockRef`, but time-based (R40)
|
|
139
|
-
* instead of the prior art's bare per-frame `* 0.04`. Initialised to
|
|
140
|
-
* `idleLock` so the very first frame doesn't jump. */
|
|
141
|
-
private lock;
|
|
142
|
-
private readonly vTarget;
|
|
143
|
-
private readonly vHead;
|
|
144
|
-
private readonly vLocal;
|
|
145
|
-
private readonly mInv;
|
|
146
|
-
private readonly vAim;
|
|
147
|
-
private readonly vFwdW;
|
|
148
|
-
private readonly vRightW;
|
|
149
|
-
private readonly vUpW;
|
|
150
|
-
private readonly qHeadW;
|
|
151
|
-
private readonly qRestInv;
|
|
152
|
-
/** What the mixer wrote to the head this frame, before we touch it. */
|
|
153
|
-
private readonly qClip;
|
|
154
|
-
/** The camera-locked look pose computed this frame. */
|
|
155
|
-
private readonly qLook;
|
|
156
|
-
/** Low-passed (tau ~0.2s) copy of `qClip`. RULING R43 part 2: the
|
|
157
|
-
* deviation boost is measured against THIS, not the raw clip pose, so
|
|
158
|
-
* one noisy/fast clip frame can't make the boost — and therefore the
|
|
159
|
-
* blend — jump within a frame. Ported from useHeadTracking.ts:594-606. */
|
|
160
|
-
private readonly animHead;
|
|
161
|
-
private animHeadHas;
|
|
162
|
-
/** The blended pose, before the output velocity cap below is applied. */
|
|
163
|
-
private readonly qOut;
|
|
164
|
-
/** The actual pose WRITTEN to the head last frame — the anchor the
|
|
165
|
-
* output velocity cap (RULING R43 part 3) rotates from. Ported from
|
|
166
|
-
* useHeadTracking.ts:621-633. */
|
|
167
|
-
private readonly lastOutHead;
|
|
168
|
-
private lastOutHas;
|
|
169
|
-
/** RULING R61: the neck bones blend with their OWN clip pose too — same
|
|
170
|
-
* low-pass, same shared head-derived `blend`, same output cap — ported
|
|
171
|
-
* from useHeadTracking.ts:598-606 (low-pass) and :635-671 (blend +
|
|
172
|
-
* cap). One low-pass/cap pair per neck bone, each independent (the
|
|
173
|
-
* clip poses and, after the cap, the written poses differ per bone). */
|
|
174
|
-
private readonly animNeck1;
|
|
175
|
-
private animNeck1Has;
|
|
176
|
-
private readonly animNeck2;
|
|
177
|
-
private animNeck2Has;
|
|
178
|
-
private readonly qOutNeck;
|
|
179
|
-
private readonly lastOutNeck1;
|
|
180
|
-
private lastOutNeck1Has;
|
|
181
|
-
private readonly lastOutNeck2;
|
|
182
|
-
private lastOutNeck2Has;
|
|
125
|
+
private eyes;
|
|
126
|
+
private headYaw;
|
|
127
|
+
private headPitch;
|
|
128
|
+
private torsoYaw;
|
|
129
|
+
private outside;
|
|
130
|
+
private twisted;
|
|
131
|
+
private readonly sway;
|
|
132
|
+
private glance;
|
|
133
|
+
private readonly base;
|
|
134
|
+
private readonly out;
|
|
135
|
+
private readonly q;
|
|
136
|
+
private readonly pq;
|
|
137
|
+
private readonly up;
|
|
138
|
+
/** The body's lateral axis turned with the head: pitch nods about it. */
|
|
139
|
+
private readonly lateral;
|
|
140
|
+
private readonly fwd;
|
|
141
|
+
private readonly v;
|
|
142
|
+
private readonly w;
|
|
143
|
+
private readonly headPos;
|
|
144
|
+
private readonly m;
|
|
145
|
+
private readonly qHead;
|
|
183
146
|
constructor(cfg?: GazeConfig);
|
|
184
|
-
/**
|
|
185
|
-
* Live on/off gate, read every frame. While false the system writes
|
|
186
|
-
* NOTHING to the rig — the mixer's clips own the head/neck — and eases
|
|
187
|
-
* the internal yaw/pitch/roll state toward neutral so re-enabling does
|
|
188
|
-
* not snap the head back onto the camera.
|
|
189
|
-
*/
|
|
190
147
|
setEnabled(on: boolean): void;
|
|
191
148
|
/**
|
|
192
|
-
*
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
* would track visibly faster than a 60Hz one). Per RULING R22.
|
|
197
|
-
*
|
|
198
|
-
* Hoisted to a method (RULING R25) rather than a closure allocated inside
|
|
199
|
-
* `update()` — a per-frame closure is exactly the steady-state allocation
|
|
200
|
-
* this class's zero-allocation rule forbids. `useHeadTracking.ts` can
|
|
201
|
-
* afford the closure because it's a React hook body re-created rarely and
|
|
202
|
-
* garbage-collected on unmount; embody's `update()` runs every frame for
|
|
203
|
-
* the lifetime of the character.
|
|
149
|
+
* Put back the clip pose on bones this system wrote and the mixer is not
|
|
150
|
+
* about to rewrite. Registered before the mixer (stage 0): at a loop seam
|
|
151
|
+
* or crossfade the mixer can blend from a bone's current value, which
|
|
152
|
+
* would otherwise still hold last frame's turn.
|
|
204
153
|
*/
|
|
205
|
-
|
|
154
|
+
beforeMixer(): void;
|
|
155
|
+
update(dt: number, ctx: FrameContext): void;
|
|
156
|
+
/** Smooth pursuit beyond the eyes' zone, and a quick shift on a large jump. */
|
|
157
|
+
private pursue;
|
|
158
|
+
/** The torso comes round after the neck has been twisted far for a while, and drifts home. */
|
|
159
|
+
private turnTorso;
|
|
160
|
+
/** Occasional short glances away while listening. */
|
|
161
|
+
private planGlances;
|
|
162
|
+
/** Rotate `focus` about the head by yaw (about up) and pitch (about right), degrees. */
|
|
163
|
+
private offsetFocus;
|
|
206
164
|
/**
|
|
207
|
-
*
|
|
208
|
-
*
|
|
209
|
-
* (function `drift`) — NOT the brief's single-sine placeholder
|
|
210
|
-
* (`Math.sin(t * 0.31)`), which is a different, un-sourced waveform.
|
|
211
|
-
* The three fixed frequencies/weights are shared by every axis; only
|
|
212
|
-
* the per-axis `seed` (and its own amplitude) differs, exactly as in
|
|
213
|
-
* the prior art's `drift(t, 1.0|7.3|13.1)` calls at lines 556-559.
|
|
214
|
-
*
|
|
215
|
-
* Hoisted to a method rather than a closure (RULING R25, same as
|
|
216
|
-
* `approach` above) — no steady-state allocation.
|
|
165
|
+
* Where `target` (world) sits from the face now: yaw (+ toward the
|
|
166
|
+
* character's left) and pitch (+ up), degrees.
|
|
217
167
|
*/
|
|
218
|
-
private
|
|
219
|
-
|
|
168
|
+
private angleTo;
|
|
169
|
+
private publish;
|
|
170
|
+
/** Bones the clip did not rewrite this frame still hold our output: put the clip pose back. */
|
|
171
|
+
private restoreClipPose;
|
|
172
|
+
private rememberOutput;
|
|
173
|
+
/** Rotate a chain of bones by `deg` about a world axis, shared by weight. */
|
|
174
|
+
private turn;
|
|
220
175
|
}
|