@convai/embody 1.0.1-beta.0 → 1.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +38 -0
  2. package/README.md +12 -8
  3. package/dist/character/createCharacter.d.ts +66 -5
  4. package/dist/character/createCharacter.js +93 -22
  5. package/dist/character/pipeline.d.ts +9 -5
  6. package/dist/clips/animations.d.ts +13 -1
  7. package/dist/clips/animations.js +28 -6
  8. package/dist/clips/retarget.d.ts +168 -0
  9. package/dist/clips/retarget.js +856 -0
  10. package/dist/clips/sources.d.ts +70 -0
  11. package/dist/clips/sources.js +119 -0
  12. package/dist/connect/session.d.ts +11 -0
  13. package/dist/connect/session.js +22 -0
  14. package/dist/index.d.ts +22 -1
  15. package/dist/index.js +12 -0
  16. package/dist/inspect/inspectModel.d.ts +14 -0
  17. package/dist/inspect/inspectModel.js +26 -0
  18. package/dist/inspect/morphMap.d.ts +54 -0
  19. package/dist/inspect/morphMap.js +140 -0
  20. package/dist/inspect/report.d.ts +88 -0
  21. package/dist/inspect/report.js +193 -0
  22. package/dist/motion/eyeAim.d.ts +94 -0
  23. package/dist/motion/eyeAim.js +276 -0
  24. package/dist/motion/eyeGaze.d.ts +17 -0
  25. package/dist/motion/eyeGaze.js +42 -0
  26. package/dist/motion/footLock.d.ts +67 -0
  27. package/dist/motion/footLock.js +176 -0
  28. package/dist/motion/gaze.d.ts +135 -180
  29. package/dist/motion/gaze.js +257 -491
  30. package/dist/motion/hair.d.ts +27 -0
  31. package/dist/motion/hair.js +127 -0
  32. package/dist/motion/hairClearance.d.ts +39 -0
  33. package/dist/motion/hairClearance.js +125 -0
  34. package/dist/motion/morphNormals.d.ts +38 -0
  35. package/dist/motion/morphNormals.js +237 -0
  36. package/dist/motion/types.d.ts +15 -0
  37. package/dist/react/CharacterModel.d.ts +3 -1
  38. package/dist/react/CharacterModel.js +9 -1
  39. package/dist/react/ConvaiCharacter.js +4 -1
  40. package/dist/react/PortraitLights.d.ts +3 -0
  41. package/dist/react/PortraitLights.js +44 -0
  42. package/dist/render/faceMaterials.d.ts +60 -3
  43. package/dist/render/faceMaterials.js +320 -96
  44. package/dist/render/portraitRig.d.ts +28 -0
  45. package/dist/render/portraitRig.js +55 -0
  46. package/dist/render/surfaces.d.ts +181 -0
  47. package/dist/render/surfaces.js +1020 -0
  48. package/dist/rig/resolve.d.ts +8 -2
  49. package/dist/rig/resolve.js +7 -3
  50. package/dist/rig/units.d.ts +35 -16
  51. package/dist/rig/units.js +143 -42
  52. package/dist/speech/expression.d.ts +20 -0
  53. package/dist/speech/expression.js +118 -0
  54. package/dist/speech/lipsync.d.ts +68 -2
  55. package/dist/speech/lipsync.js +205 -5
  56. package/dist/speech/mouthFinish.d.ts +46 -0
  57. package/dist/speech/mouthFinish.js +235 -0
  58. package/dist/speech/voiceClock.d.ts +94 -0
  59. package/dist/speech/voiceClock.js +176 -0
  60. package/package.json +1 -1
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Foot locking: a planted foot stays where it was put while the body sways
3
+ * above it.
4
+ *
5
+ * Idle and talking clips sway the hips, and with the body held in place the
6
+ * planted feet skate along the floor (~19 cm/s measured with jointQA on a
7
+ * convai-mha-1 body). Each planted foot is held at its lock point horizontally
8
+ * by a two-bone leg IK (thigh, calf); the clip keeps the foot's height and
9
+ * orientation. A foot that lifts or moves on purpose — a step — is released
10
+ * and blended back to the clip. Engaging and releasing run on minimum-jerk
11
+ * curves, so neither is a pop.
12
+ *
13
+ * A straight leg cannot reach a lock the hips have swayed away from; the hips
14
+ * are lowered by the shortfall (a few millimetres, capped) so the knee takes
15
+ * it instead of the foot dragging.
16
+ *
17
+ * Stage 4, after gaze and breathing. Units are world metres (the body has been
18
+ * scaled to metres by then). The values are the coffee-shop studio's, tuned
19
+ * with its joint QA.
20
+ */
21
+ import * as THREE from 'three';
22
+ import type { FrameContext, System } from '../character/pipeline.js';
23
+ export interface FootLockConfig {
24
+ /** Static gate. Default `true`. */
25
+ enabled?: boolean;
26
+ /** A foot this far (m) above its resting height counts as lifted. Default 0.03. */
27
+ liftHeight?: number;
28
+ /** A foot the clip moves faster than this (m/s) is stepping. Default 0.3. */
29
+ stepSpeed?: number;
30
+ /** Seconds to engage or release a lock. Default 0.4. */
31
+ blendTime?: number;
32
+ /** Most the hips are lowered to let a straight leg reach its lock, m. Default 0.015. */
33
+ maxDrop?: number;
34
+ }
35
+ export declare class FootLockSystem implements System {
36
+ private readonly cfg;
37
+ readonly stage: 4;
38
+ private legs;
39
+ private drop;
40
+ private readonly clipPose;
41
+ private readonly written;
42
+ private readonly a;
43
+ private readonly b;
44
+ private readonly c;
45
+ private readonly p;
46
+ private readonly q;
47
+ private readonly q2;
48
+ constructor(cfg?: FootLockConfig);
49
+ /** Planted legs, for tests and debugging. */
50
+ get locks(): {
51
+ lock: THREE.Vector3 | null;
52
+ weight: number;
53
+ }[];
54
+ /**
55
+ * Put back the clip pose on leg bones this system wrote and the mixer is not
56
+ * about to rewrite (three's mixer skips a bone whose blended value did not
57
+ * change, and the IK would then be solved again on top of itself).
58
+ */
59
+ beforeMixer(): void;
60
+ update(dt: number, ctx: FrameContext): void;
61
+ private findLegs;
62
+ private copyInto;
63
+ /** Two-bone IK: thigh and calf turn so the foot reaches `goal`, the knee bending forward. */
64
+ private solve;
65
+ /** Apply a world-space rotation to a bone, expressed in its parent's frame. */
66
+ private rotateWorld;
67
+ }
@@ -0,0 +1,176 @@
1
+ /**
2
+ * Foot locking: a planted foot stays where it was put while the body sways
3
+ * above it.
4
+ *
5
+ * Idle and talking clips sway the hips, and with the body held in place the
6
+ * planted feet skate along the floor (~19 cm/s measured with jointQA on a
7
+ * convai-mha-1 body). Each planted foot is held at its lock point horizontally
8
+ * by a two-bone leg IK (thigh, calf); the clip keeps the foot's height and
9
+ * orientation. A foot that lifts or moves on purpose — a step — is released
10
+ * and blended back to the clip. Engaging and releasing run on minimum-jerk
11
+ * curves, so neither is a pop.
12
+ *
13
+ * A straight leg cannot reach a lock the hips have swayed away from; the hips
14
+ * are lowered by the shortfall (a few millimetres, capped) so the knee takes
15
+ * it instead of the foot dragging.
16
+ *
17
+ * Stage 4, after gaze and breathing. Units are world metres (the body has been
18
+ * scaled to metres by then). The values are the coffee-shop studio's, tuned
19
+ * with its joint QA.
20
+ */
21
+ import * as THREE from 'three';
22
+ import { minJerk } from './gaze.js';
23
+ /** A forward knee bends negatively about (hip→foot × body forward). */
24
+ const KNEE = -1;
25
+ /** A held lock follows the clip's own slow foot drift at this speed, m/s (below any visible slide). */
26
+ const FOLLOW = 0.01;
27
+ export class FootLockSystem {
28
+ constructor(cfg = {}) {
29
+ this.cfg = cfg;
30
+ this.stage = 4;
31
+ this.legs = null;
32
+ this.drop = 0;
33
+ this.clipPose = new Map();
34
+ this.written = new Map();
35
+ this.a = new THREE.Vector3();
36
+ this.b = new THREE.Vector3();
37
+ this.c = new THREE.Vector3();
38
+ this.p = new THREE.Vector3();
39
+ this.q = new THREE.Quaternion();
40
+ this.q2 = new THREE.Quaternion();
41
+ }
42
+ /** Planted legs, for tests and debugging. */
43
+ get locks() {
44
+ return (this.legs ?? []).map((l) => ({ lock: l.lock, weight: l.w }));
45
+ }
46
+ /**
47
+ * Put back the clip pose on leg bones this system wrote and the mixer is not
48
+ * about to rewrite (three's mixer skips a bone whose blended value did not
49
+ * change, and the IK would then be solved again on top of itself).
50
+ */
51
+ beforeMixer() {
52
+ for (const [bone, clip] of this.clipPose)
53
+ if (this.written.get(bone)?.equals(bone.quaternion))
54
+ bone.quaternion.copy(clip);
55
+ }
56
+ update(dt, ctx) {
57
+ if (this.cfg.enabled === false || dt <= 0)
58
+ return;
59
+ this.legs ??= this.findLegs(ctx.rig);
60
+ if (!this.legs.length)
61
+ return;
62
+ dt = Math.min(dt, 0.1);
63
+ const root = ctx.root;
64
+ const liftH = this.cfg.liftHeight ?? 0.03;
65
+ const stepV = this.cfg.stepSpeed ?? 0.3;
66
+ root.updateMatrixWorld(true);
67
+ // Knee slack for a leg too straight to reach its lock.
68
+ let need = 0;
69
+ for (const l of this.legs) {
70
+ if (!l.lock || l.w < 0.05)
71
+ continue;
72
+ const A = l.thigh.getWorldPosition(this.a), B = l.calf.getWorldPosition(this.b), C = l.foot.getWorldPosition(this.c);
73
+ const reach = A.distanceTo(B) + B.distanceTo(C);
74
+ need = Math.max(need, (A.distanceTo(this.p.set(l.lock.x, C.y, l.lock.z)) - reach * 0.995 + 0.003) * l.w);
75
+ }
76
+ const goalDrop = THREE.MathUtils.clamp(need, 0, this.cfg.maxDrop ?? 0.015);
77
+ this.drop += (goalDrop - this.drop) * (1 - Math.exp(-dt / 0.15));
78
+ if (this.drop > 1e-5) {
79
+ const s = root.parent ? root.parent.getWorldScale(this.p).y : 1;
80
+ root.position.y -= this.drop / s;
81
+ root.updateMatrixWorld(true);
82
+ }
83
+ const fwd = this.p.set(0, 0, 1).applyQuaternion(root.getWorldQuaternion(this.q)).clone();
84
+ for (const l of this.legs) {
85
+ for (const bone of [l.thigh, l.calf, l.foot])
86
+ this.copyInto(this.clipPose, bone);
87
+ const p = l.foot.getWorldPosition(new THREE.Vector3());
88
+ const raw = l.prev ? Math.hypot(p.x - l.prev.x, p.z - l.prev.z) / dt : 0;
89
+ l.v += (raw - l.v) * (1 - Math.exp(-dt / 0.04));
90
+ l.prev = p.clone();
91
+ // The lowest the foot has been, slowly forgiving.
92
+ l.restY = l.restY === null ? p.y : Math.min(l.restY + dt * 0.01, p.y);
93
+ // Hysteresis: plant below the thresholds, release only clearly above.
94
+ const planted = l.lock
95
+ ? p.y < l.restY + liftH * 1.6 && l.v < stepV * 1.4
96
+ : p.y < l.restY + liftH && l.v < stepV;
97
+ if (planted && !l.lock)
98
+ l.lock = p.clone();
99
+ if (l.lock) {
100
+ // Follow the clip's own slow drift, so the IK never holds a foot far from it.
101
+ const dx = p.x - l.lock.x, dz = p.z - l.lock.z, dist = Math.hypot(dx, dz);
102
+ if (dist > 1e-6) {
103
+ const step = Math.min(dist, FOLLOW * dt);
104
+ l.lock.x += (dx / dist) * step;
105
+ l.lock.z += (dz / dist) * step;
106
+ }
107
+ }
108
+ const target = planted ? 1 : 0;
109
+ if (target !== l.wTo) {
110
+ l.wFrom = l.w;
111
+ l.wTo = target;
112
+ l.wT = 0;
113
+ }
114
+ l.wT = Math.min(1, l.wT + dt / (this.cfg.blendTime ?? 0.4));
115
+ l.w = l.wFrom + (l.wTo - l.wFrom) * minJerk(l.wT);
116
+ if (!planted && l.w < 0.01)
117
+ l.lock = null;
118
+ if (!l.lock || l.w < 0.001)
119
+ continue;
120
+ // Goal: the lock horizontally (by its weight), the clip's own height.
121
+ const goal = new THREE.Vector3(l.lock.x, p.y, l.lock.z).lerp(p, 1 - l.w);
122
+ if (goal.distanceToSquared(p) < 1e-10)
123
+ continue;
124
+ const footQ = l.foot.getWorldQuaternion(new THREE.Quaternion());
125
+ this.solve(l, goal, fwd);
126
+ // The foot keeps the world orientation the clip gave it.
127
+ l.foot.quaternion.copy(l.foot.parent.getWorldQuaternion(this.q).invert().multiply(footQ));
128
+ l.foot.updateMatrixWorld(true);
129
+ for (const bone of [l.thigh, l.calf, l.foot])
130
+ this.copyInto(this.written, bone);
131
+ }
132
+ }
133
+ findLegs(rig) {
134
+ const b = rig.bones;
135
+ const legs = [];
136
+ for (const [thigh, calf, foot] of [[b.thighL, b.calfL, b.footL], [b.thighR, b.calfR, b.footR]]) {
137
+ if (thigh && calf && foot)
138
+ legs.push({ thigh, calf, foot, lock: null, w: 0, wFrom: 0, wTo: 0, wT: 1, v: 0, restY: null, prev: null });
139
+ }
140
+ return legs;
141
+ }
142
+ copyInto(map, bone) {
143
+ let q = map.get(bone);
144
+ if (!q)
145
+ map.set(bone, (q = new THREE.Quaternion()));
146
+ q.copy(bone.quaternion);
147
+ }
148
+ /** Two-bone IK: thigh and calf turn so the foot reaches `goal`, the knee bending forward. */
149
+ solve(l, goal, fwd) {
150
+ const A = l.thigh.getWorldPosition(this.a), B = l.calf.getWorldPosition(this.b), C = l.foot.getWorldPosition(this.c);
151
+ const l1 = A.distanceTo(B), l2 = B.distanceTo(C);
152
+ const d = THREE.MathUtils.clamp(A.distanceTo(goal), Math.abs(l1 - l2) + 1e-4, l1 + l2 - 1e-4);
153
+ // The knee hinges across the body's forward direction: taken from the
154
+ // leg's own shape, a nearly straight leg's cross product was tiny and
155
+ // noisy, flipped sign, and the knee solved to the wrong side for a frame.
156
+ const n = C.clone().sub(A).normalize().cross(fwd);
157
+ if (n.lengthSq() < 1e-8)
158
+ return;
159
+ n.normalize();
160
+ const u1 = B.clone().sub(A).normalize(), u2 = C.clone().sub(B).normalize();
161
+ const cur = Math.atan2(u1.clone().cross(u2).dot(n), u1.dot(u2));
162
+ const want = KNEE * (Math.PI - Math.acos(THREE.MathUtils.clamp((l1 * l1 + l2 * l2 - d * d) / (2 * l1 * l2), -1, 1)));
163
+ this.rotateWorld(l.calf, this.q2.setFromAxisAngle(n, want - cur));
164
+ l.thigh.updateMatrixWorld(true);
165
+ const from = l.foot.getWorldPosition(new THREE.Vector3()).sub(A).normalize();
166
+ const to = goal.clone().sub(A).normalize();
167
+ this.rotateWorld(l.thigh, this.q2.setFromUnitVectors(from, to));
168
+ }
169
+ /** Apply a world-space rotation to a bone, expressed in its parent's frame. */
170
+ rotateWorld(bone, qWorld) {
171
+ const parentQ = bone.parent.getWorldQuaternion(new THREE.Quaternion());
172
+ const world = parentQ.clone().multiply(bone.quaternion);
173
+ bone.quaternion.copy(parentQ.invert().multiply(qWorld.clone().multiply(world)));
174
+ bone.updateMatrixWorld(true);
175
+ }
176
+ }
@@ -1,220 +1,175 @@
1
+ import * as THREE from 'three';
1
2
  import type { FrameContext, System } from '../character/pipeline.js';
2
3
  import type { EyeGazeDirection, GazeDirection, GazeSource } from './types.js';
3
4
  /**
4
- * Config surface for `GazeSystem`. Task 6 landed the geometric core —
5
- * target -> yaw/pitch -> head/neck split -> time-based, velocity-capped
6
- * smoothing -> `setLookQuat`. Task 7 (this task) adds the aliveness layer:
7
- * idle drift, look-away gaze shifts and turn-bank, all read by `update()`
8
- * below. Task 8 (clip-blending: camera lock, deviation pull-back) extends
9
- * the SAME class and will read the fields still marked "reserved" — they
10
- * are declared now so the class's public config shape does not change
11
- * under that task, but `update()` does not read them yet.
5
+ * Gaze the way people do it: eyes, then neck, then body.
6
+ *
7
+ * · The eyes take every small change on their own (EyeGazeSystem aims each
8
+ * one at `focus`), so the head is not moved by a few degrees of camera.
9
+ * · Beyond that the head follows smoothly, lagging the eyes (smooth pursuit),
10
+ * so a gliding camera is tracked without the head holding and then
11
+ * snapping. A sudden large jump gets a quick eye-led head shift on a
12
+ * minimum-jerk curve, ~80 ms after the eyes, stopping a few degrees short
13
+ * for the eyes to finish.
14
+ * · The torso turns only once the head has had to twist far off the body for
15
+ * a moment; it turns slowly while the neck unwinds by the same amount, so
16
+ * the head stays on target. When the target comes back toward the front,
17
+ * the torso drifts home the same way.
18
+ * · While fixating, most of the clip's fast head sway is cancelled (people
19
+ * hold the head still while looking at something); the slow part of it,
20
+ * and everything below the neck, plays as authored.
21
+ *
22
+ * Every turn is ADDED to the clip's pose, so idle sway and talking gestures
23
+ * stay alive. A bone the clip did not key this frame still holds last
24
+ * frame's output; that is detected and the clip pose restored before adding,
25
+ * so nothing accumulates.
26
+ *
27
+ * Angles are measured from the eyes' own neutral frame (EyeAim, from the
28
+ * body's eye geometry) where the body allows, otherwise from the rig basis.
29
+ * The values are the coffee-shop studio's, tuned on camera.
12
30
  */
13
31
  export interface GazeConfig {
14
- /** Half-angle (rad) the HEAD alone covers before the neck starts sharing
15
- * yaw. Verbatim from `HeadTrackingConfig.DEFAULT_CONFIG.headOnlyAngle`
16
- * (useHeadTracking.ts:107) — NOT the interface JSDoc's PI/3 (line 17),
17
- * which the implementation itself does not use. Per RULING R20: the
18
- * DEFAULTS object is ground truth, and its own comment explains 0.44 was
19
- * a deliberate retune down from a wider zone that put the head near its
20
- * joint limit for a side viewer. */
21
- headOnlyAngle?: number;
22
- /** Beyond this yaw (rad) tracking disengages entirely for the frame
23
- * (target clamped to zero, not clamped-in-range). */
32
+ /** Static construction-time gate. Live control is `setEnabled()`. */
33
+ enabled?: boolean;
34
+ /** Beyond this angle (rad) off the body's front the character stops attending and turns home. Default PI/2. */
24
35
  maxTrackAngle?: number;
25
- /** Vertical (pitch) clamp, radians. */
36
+ /** Changes inside this (degrees) are left to the eyes. Default 5 yaw, 4 pitch. */
37
+ eyeZone?: {
38
+ yaw: number;
39
+ pitch: number;
40
+ };
41
+ /** A target this far (degrees) outside the face gets a quick head shift. Default 18 yaw, 12 pitch. */
42
+ shiftZone?: {
43
+ yaw: number;
44
+ pitch: number;
45
+ };
46
+ /** Seconds the head lags the eyes in smooth pursuit. Default 0.45. */
47
+ followTime?: number;
48
+ /** How far the head and neck turn from the body, degrees. Default 45 yaw, 25 pitch. */
49
+ headLimit?: {
50
+ yaw: number;
51
+ pitch: number;
52
+ };
53
+ /**
54
+ * Torso turn: after the head has been twisted past `trigger` degrees for
55
+ * `hold` seconds the torso turns, at most `limit` degrees. `false` keeps the
56
+ * torso out of it. Default `{ trigger: 32, hold: 0.6, limit: 30 }`.
57
+ */
58
+ torso?: false | {
59
+ trigger?: number;
60
+ hold?: number;
61
+ limit?: number;
62
+ };
63
+ /** Share of the clip's fast head sway cancelled while fixating, 0–1. Default 0.8. */
64
+ steady?: number;
65
+ /**
66
+ * Occasional short glances away while listening (never while speaking or
67
+ * under reduced motion). Default `true`.
68
+ */
69
+ lookAways?: boolean;
70
+ /** @deprecated Ignored since 1.1: the head is driven by pursuit and shifts, see `eyeZone`/`followTime`. */
71
+ headOnlyAngle?: number;
72
+ /** @deprecated Ignored since 1.1; see `headLimit.pitch`. */
26
73
  maxPitchAngle?: number;
27
- /** Ease rate (per-second, before the `* 60` time-based conversion in
28
- * `update()`) toward the target yaw/pitch. Velocity-capped — see
29
- * `update()`. */
74
+ /** @deprecated Ignored since 1.1; see `followTime`. */
30
75
  lerpSpeed?: number;
31
- /** Share of pitch applied to the head. */
76
+ /** @deprecated Ignored since 1.1. */
32
77
  headPitchFactor?: number;
33
- /** Share of pitch applied to each neck bone. */
78
+ /** @deprecated Ignored since 1.1. */
34
79
  neckPitchFactor?: number;
35
- /** Share of the excess (beyond `headOnlyAngle`) yaw applied to the neck.
36
- * Verbatim from `DEFAULT_CONFIG.neckYawFactor` (useHeadTracking.ts:113),
37
- * which is 0.75, not the interface JSDoc's 0.7 (line 30). Per R20. */
80
+ /** @deprecated Ignored since 1.1. */
38
81
  neckYawFactor?: number;
39
- /** Idle micro-drift amplitude on yaw, radians. */
82
+ /** @deprecated Ignored since 1.1: the clip's own motion carries the head's life. */
40
83
  idleYawAmp?: number;
41
- /** Idle micro-drift amplitude on pitch, radians. */
84
+ /** @deprecated Ignored since 1.1. */
42
85
  idlePitchAmp?: number;
43
- /** Idle micro-drift amplitude on roll, radians. */
86
+ /** @deprecated Ignored since 1.1. */
44
87
  idleRollAmp?: number;
45
- /** Time multiplier for the idle-drift clock. */
88
+ /** @deprecated Ignored since 1.1. */
46
89
  idleSpeed?: number;
47
- /** Roll added per radian of (capped) head yaw — banks the head into a
48
- * turn. */
90
+ /** @deprecated Ignored since 1.1. */
49
91
  turnBank?: number;
50
- /** Max yaw (radians) of an occasional look-away gaze shift. */
92
+ /** @deprecated Ignored since 1.1; see `lookAways`. */
51
93
  gazeYawRange?: number;
52
- /** Max pitch (radians) of a gaze shift. */
94
+ /** @deprecated Ignored since 1.1; see `lookAways`. */
53
95
  gazePitchRange?: number;
54
- /** Max roll (radians) of a gaze shift. */
96
+ /** @deprecated Ignored since 1.1. */
55
97
  gazeRollRange?: number;
56
- /** Ease rate (per-second, before the `* 60` time-based conversion) the
57
- * gaze-shift offset eases toward its target at. */
98
+ /** @deprecated Ignored since 1.1. */
58
99
  gazeEase?: number;
59
- /** Minimum seconds a gaze-shift target is held before re-picking. */
100
+ /** @deprecated Ignored since 1.1. */
60
101
  gazeDwellMin?: number;
61
- /** Maximum seconds a gaze-shift target is held before re-picking. */
102
+ /** @deprecated Ignored since 1.1. */
62
103
  gazeDwellMax?: number;
63
- /** Camera-lock strength while idle (0-1): how much of the clip's local
64
- * head rotation is replaced by the look pose. High values mean
65
- * procedural drift/gaze carry the aliveness and the clip barely
66
- * reaches the output. */
104
+ /** @deprecated Ignored since 1.1: turns are added to the clip, never locked over it. */
67
105
  idleLock?: number;
68
- /** Camera-lock strength while `ctx.speaking` — focused on the viewer. */
106
+ /** @deprecated Ignored since 1.1. */
69
107
  talkLock?: number;
70
- /** Clip-deviation (radians, angle between the clip's head pose and the
71
- * look pose) below which the clip flows through untouched. */
108
+ /** @deprecated Ignored since 1.1. */
72
109
  deviationLo?: number;
73
- /** Clip-deviation (radians) by which the pull-back saturates — the head
74
- * is fully locked back to the look pose regardless of `idleLock`/
75
- * `talkLock`. */
110
+ /** @deprecated Ignored since 1.1. */
76
111
  deviationHi?: number;
77
- /** Static construction-time gate. Live control is `setEnabled()`. */
78
- enabled?: boolean;
79
112
  }
80
- /**
81
- * Head/neck gaze pose solve, ported from `useHeadTracking`
82
- * (convai-web-sdk-internal `examples/react-three-fiber/src/hooks/useHeadTracking.ts`).
83
- *
84
- * Stage 2 — the ONLY system permitted to OVERRIDE the mixer's head/neck
85
- * rotation outright (`.copy(...)`, never `.multiply(...)`): a head that
86
- * blends additively with its own animation clip fights itself every frame.
87
- *
88
- * Task 6 ported the geometric core: world target -> yaw/pitch about the
89
- * rig's own basis -> head/neck split -> time-based, velocity-capped ease ->
90
- * `setLookQuat`. Task 7 layered the aliveness on top of that same solve:
91
- * always-on idle micro-drift, occasional look-away gaze shifts with dwell
92
- * timing, and turn-bank roll — all suppressed under `ctx.reducedMotion`.
93
- * Task 8 (this task) adds the clip-blending layer: a smoothed camera-lock
94
- * strength (`idleLock`/`talkLock`, eased rather than switched — see
95
- * `update()`) blended against the mixer's own clip pose, with a deviation
96
- * pull-back (`deviationLo`/`deviationHi`) that guards large clip excursions
97
- * regardless of lock strength; and `calm`, which de-emphasises the idle
98
- * drift/gaze-shift amplitudes while the (smoothed) lock is high, i.e. while
99
- * speaking — a focused speaker stops glancing around.
100
- *
101
- * The HEAD is the only bone that blends with the clip; the neck bones are
102
- * still overridden outright (Task 6/7 behaviour, unchanged) — see
103
- * `update()`.
104
- *
105
- * See `.superpowers/sdd/2026-09-07-convai-embody-motion-and-staging/task-6-report.md`,
106
- * `task-7-report.md` and `task-8-report.md` for the line-by-line comparison
107
- * against the prior art and the places this deliberately differs from it.
108
- */
113
+ /** Minimum-jerk ease: 0→1 with zero velocity and acceleration at both ends. */
114
+ export declare const minJerk: (s: number) => number;
109
115
  export declare class GazeSystem implements System, GazeSource {
110
116
  private readonly cfg;
111
117
  readonly stage: 2;
112
118
  readonly direction: GazeDirection;
113
- /** World-space angle still between the head's forward and the target. */
114
119
  readonly residual: EyeGazeDirection;
120
+ /** See `GazeSource.attending`. */
121
+ attending: boolean;
122
+ /** The world point the character is looking at this frame; see `GazeSource.focus`. */
123
+ readonly focus: THREE.Vector3;
115
124
  private gate;
116
- private yaw;
117
- private pitch;
118
- /** Idle-roll + turn-bank, written by the aliveness layer below. */
119
- private roll;
120
- /** Idle-drift clock, advanced by `dt * idleSpeed`. Randomly seeded so
121
- * multiple GazeSystem instances on stage together don't drift in
122
- * lockstep. */
123
- private driftClock;
124
- /** Current eased gaze-shift offsets (the "look-away" the head is
125
- * easing toward / holding). */
126
- private shiftYaw;
127
- private shiftPitch;
128
- private shiftRoll;
129
- /** The look-away target `shift*` is easing toward, re-picked every
130
- * dwell period. */
131
- private shiftTargetYaw;
132
- private shiftTargetPitch;
133
- private shiftTargetRoll;
134
- /** Seconds remaining before the next gaze-shift target is re-picked. */
135
- private dwell;
136
- /** Smoothed camera-lock strength, eased toward `idleLock`/`talkLock`
137
- * each frame (RULING R39) rather than switched instantly — ported
138
- * from `useHeadTracking.ts:581`'s `lockRef`, but time-based (R40)
139
- * instead of the prior art's bare per-frame `* 0.04`. Initialised to
140
- * `idleLock` so the very first frame doesn't jump. */
141
- private lock;
142
- private readonly vTarget;
143
- private readonly vHead;
144
- private readonly vLocal;
145
- private readonly mInv;
146
- private readonly vAim;
147
- private readonly vFwdW;
148
- private readonly vRightW;
149
- private readonly vUpW;
150
- private readonly qHeadW;
151
- private readonly qRestInv;
152
- /** What the mixer wrote to the head this frame, before we touch it. */
153
- private readonly qClip;
154
- /** The camera-locked look pose computed this frame. */
155
- private readonly qLook;
156
- /** Low-passed (tau ~0.2s) copy of `qClip`. RULING R43 part 2: the
157
- * deviation boost is measured against THIS, not the raw clip pose, so
158
- * one noisy/fast clip frame can't make the boost — and therefore the
159
- * blend — jump within a frame. Ported from useHeadTracking.ts:594-606. */
160
- private readonly animHead;
161
- private animHeadHas;
162
- /** The blended pose, before the output velocity cap below is applied. */
163
- private readonly qOut;
164
- /** The actual pose WRITTEN to the head last frame — the anchor the
165
- * output velocity cap (RULING R43 part 3) rotates from. Ported from
166
- * useHeadTracking.ts:621-633. */
167
- private readonly lastOutHead;
168
- private lastOutHas;
169
- /** RULING R61: the neck bones blend with their OWN clip pose too — same
170
- * low-pass, same shared head-derived `blend`, same output cap — ported
171
- * from useHeadTracking.ts:598-606 (low-pass) and :635-671 (blend +
172
- * cap). One low-pass/cap pair per neck bone, each independent (the
173
- * clip poses and, after the cap, the written poses differ per bone). */
174
- private readonly animNeck1;
175
- private animNeck1Has;
176
- private readonly animNeck2;
177
- private animNeck2Has;
178
- private readonly qOutNeck;
179
- private readonly lastOutNeck1;
180
- private lastOutNeck1Has;
181
- private readonly lastOutNeck2;
182
- private lastOutNeck2Has;
125
+ private eyes;
126
+ private headYaw;
127
+ private headPitch;
128
+ private torsoYaw;
129
+ private outside;
130
+ private twisted;
131
+ private readonly sway;
132
+ private glance;
133
+ private readonly base;
134
+ private readonly out;
135
+ private readonly q;
136
+ private readonly pq;
137
+ private readonly up;
138
+ /** The body's lateral axis turned with the head: pitch nods about it. */
139
+ private readonly lateral;
140
+ private readonly fwd;
141
+ private readonly v;
142
+ private readonly w;
143
+ private readonly headPos;
144
+ private readonly m;
145
+ private readonly qHead;
183
146
  constructor(cfg?: GazeConfig);
184
- /**
185
- * Live on/off gate, read every frame. While false the system writes
186
- * NOTHING to the rig — the mixer's clips own the head/neck — and eases
187
- * the internal yaw/pitch/roll state toward neutral so re-enabling does
188
- * not snap the head back onto the camera.
189
- */
190
147
  setEnabled(on: boolean): void;
191
148
  /**
192
- * Time-based exponential ease toward `tgt`, velocity-capped so a frame
193
- * hitch can't yank the head. Ported verbatim from
194
- * useHeadTracking.ts:480-484 — NOT the bare `x += (target - x) * lerp`
195
- * form, which is frame-rate dependent by construction (a 144Hz monitor
196
- * would track visibly faster than a 60Hz one). Per RULING R22.
197
- *
198
- * Hoisted to a method (RULING R25) rather than a closure allocated inside
199
- * `update()` — a per-frame closure is exactly the steady-state allocation
200
- * this class's zero-allocation rule forbids. `useHeadTracking.ts` can
201
- * afford the closure because it's a React hook body re-created rarely and
202
- * garbage-collected on unmount; embody's `update()` runs every frame for
203
- * the lifetime of the character.
149
+ * Put back the clip pose on bones this system wrote and the mixer is not
150
+ * about to rewrite. Registered before the mixer (stage 0): at a loop seam
151
+ * or crossfade the mixer can blend from a bone's current value, which
152
+ * would otherwise still hold last frame's turn.
204
153
  */
205
- private approach;
154
+ beforeMixer(): void;
155
+ update(dt: number, ctx: FrameContext): void;
156
+ /** Smooth pursuit beyond the eyes' zone, and a quick shift on a large jump. */
157
+ private pursue;
158
+ /** The torso comes round after the neck has been twisted far for a while, and drifts home. */
159
+ private turnTorso;
160
+ /** Occasional short glances away while listening. */
161
+ private planGlances;
162
+ /** Rotate `focus` about the head by yaw (about up) and pitch (about right), degrees. */
163
+ private offsetFocus;
206
164
  /**
207
- * Smooth, non-repeating drift in ~[-1, 1] — three incommensurate sines
208
- * (an fbm-lite). Ported VERBATIM from useHeadTracking.ts:144-150
209
- * (function `drift`) — NOT the brief's single-sine placeholder
210
- * (`Math.sin(t * 0.31)`), which is a different, un-sourced waveform.
211
- * The three fixed frequencies/weights are shared by every axis; only
212
- * the per-axis `seed` (and its own amplitude) differs, exactly as in
213
- * the prior art's `drift(t, 1.0|7.3|13.1)` calls at lines 556-559.
214
- *
215
- * Hoisted to a method rather than a closure (RULING R25, same as
216
- * `approach` above) — no steady-state allocation.
165
+ * Where `target` (world) sits from the face now: yaw (+ toward the
166
+ * character's left) and pitch (+ up), degrees.
217
167
  */
218
- private drift;
219
- update(dt: number, ctx: FrameContext): void;
168
+ private angleTo;
169
+ private publish;
170
+ /** Bones the clip did not rewrite this frame still hold our output: put the clip pose back. */
171
+ private restoreClipPose;
172
+ private rememberOutput;
173
+ /** Rotate a chain of bones by `deg` about a world axis, shared by weight. */
174
+ private turn;
220
175
  }