@voqalize/avatar 0.4.4 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +65 -154
  2. package/dist/index.d.ts +37 -26
  3. package/dist/index.js +48 -25
  4. package/dist/pin.d.ts +2 -0
  5. package/dist/pin.js +3 -0
  6. package/dist/react.d.ts +21 -6
  7. package/dist/react.js +24 -6
  8. package/package.json +8 -67
  9. package/LICENSE-APACHE-2.0 +0 -201
  10. package/LICENSE-CC-BY-4.0 +0 -396
  11. package/assets/README.md +0 -60
  12. package/assets/tanvi.glb +0 -0
  13. package/assets/tanya.glb +0 -0
  14. package/assets/tara.glb +0 -0
  15. package/assets/tess.glb +0 -0
  16. package/assets/tushar.glb +0 -0
  17. package/client/Avatar.tsx +0 -57
  18. package/client/AvatarClient.ts +0 -818
  19. package/client/createAvatar.ts +0 -177
  20. package/client/index.ts +0 -44
  21. package/client/internal.ts +0 -132
  22. package/client/playout.ts +0 -95
  23. package/client/react.ts +0 -13
  24. package/client/supports.ts +0 -27
  25. package/client/three/assets.ts +0 -26
  26. package/client/three/budgets.ts +0 -27
  27. package/client/three/character-rig.ts +0 -1746
  28. package/client/three/createCharacter.ts +0 -109
  29. package/client/three/holds.ts +0 -69
  30. package/client/three/internal.ts +0 -55
  31. package/client/three/motion-limits.json +0 -124
  32. package/client/three/sequences.ts +0 -281
  33. package/client/three/tanvi-asset.ts +0 -6
  34. package/client/three/tanvi.ts +0 -30
  35. package/client/three/tanya-asset.ts +0 -6
  36. package/client/three/tanya.ts +0 -29
  37. package/client/three/tara-asset.ts +0 -25
  38. package/client/three/tara.ts +0 -36
  39. package/client/three/tess-asset.ts +0 -6
  40. package/client/three/tess.ts +0 -30
  41. package/client/three/tushar-asset.ts +0 -6
  42. package/client/three/tushar.ts +0 -28
  43. package/client/types.ts +0 -235
  44. package/client/useAvatar.ts +0 -74
  45. package/dist/Avatar.d.ts +0 -44
  46. package/dist/Avatar.d.ts.map +0 -1
  47. package/dist/Avatar.js +0 -7
  48. package/dist/Avatar.js.map +0 -1
  49. package/dist/AvatarClient.d.ts +0 -274
  50. package/dist/AvatarClient.d.ts.map +0 -1
  51. package/dist/AvatarClient.js +0 -712
  52. package/dist/AvatarClient.js.map +0 -1
  53. package/dist/createAvatar.d.ts +0 -135
  54. package/dist/createAvatar.d.ts.map +0 -1
  55. package/dist/createAvatar.js +0 -75
  56. package/dist/createAvatar.js.map +0 -1
  57. package/dist/index.d.ts.map +0 -1
  58. package/dist/index.js.map +0 -1
  59. package/dist/internal.d.ts +0 -34
  60. package/dist/internal.d.ts.map +0 -1
  61. package/dist/internal.js +0 -56
  62. package/dist/internal.js.map +0 -1
  63. package/dist/playout.d.ts +0 -42
  64. package/dist/playout.d.ts.map +0 -1
  65. package/dist/playout.js +0 -85
  66. package/dist/playout.js.map +0 -1
  67. package/dist/react.d.ts.map +0 -1
  68. package/dist/react.js.map +0 -1
  69. package/dist/supports.d.ts +0 -19
  70. package/dist/supports.d.ts.map +0 -1
  71. package/dist/supports.js +0 -25
  72. package/dist/supports.js.map +0 -1
  73. package/dist/three/assets.d.ts +0 -10
  74. package/dist/three/assets.d.ts.map +0 -1
  75. package/dist/three/assets.js +0 -23
  76. package/dist/three/assets.js.map +0 -1
  77. package/dist/three/budgets.d.ts +0 -17
  78. package/dist/three/budgets.d.ts.map +0 -1
  79. package/dist/three/budgets.js +0 -21
  80. package/dist/three/budgets.js.map +0 -1
  81. package/dist/three/character-rig.d.ts +0 -366
  82. package/dist/three/character-rig.d.ts.map +0 -1
  83. package/dist/three/character-rig.js +0 -1644
  84. package/dist/three/character-rig.js.map +0 -1
  85. package/dist/three/createCharacter.d.ts +0 -60
  86. package/dist/three/createCharacter.d.ts.map +0 -1
  87. package/dist/three/createCharacter.js +0 -84
  88. package/dist/three/createCharacter.js.map +0 -1
  89. package/dist/three/holds.d.ts +0 -38
  90. package/dist/three/holds.d.ts.map +0 -1
  91. package/dist/three/holds.js +0 -61
  92. package/dist/three/holds.js.map +0 -1
  93. package/dist/three/internal.d.ts +0 -52
  94. package/dist/three/internal.d.ts.map +0 -1
  95. package/dist/three/internal.js +0 -49
  96. package/dist/three/internal.js.map +0 -1
  97. package/dist/three/motion-limits.json +0 -124
  98. package/dist/three/sequences.d.ts +0 -71
  99. package/dist/three/sequences.d.ts.map +0 -1
  100. package/dist/three/sequences.js +0 -262
  101. package/dist/three/sequences.js.map +0 -1
  102. package/dist/three/tanvi-asset.d.ts +0 -7
  103. package/dist/three/tanvi-asset.d.ts.map +0 -1
  104. package/dist/three/tanvi-asset.js +0 -7
  105. package/dist/three/tanvi-asset.js.map +0 -1
  106. package/dist/three/tanvi.d.ts +0 -24
  107. package/dist/three/tanvi.d.ts.map +0 -1
  108. package/dist/three/tanvi.js +0 -24
  109. package/dist/three/tanvi.js.map +0 -1
  110. package/dist/three/tanya-asset.d.ts +0 -7
  111. package/dist/three/tanya-asset.d.ts.map +0 -1
  112. package/dist/three/tanya-asset.js +0 -7
  113. package/dist/three/tanya-asset.js.map +0 -1
  114. package/dist/three/tanya.d.ts +0 -23
  115. package/dist/three/tanya.d.ts.map +0 -1
  116. package/dist/three/tanya.js +0 -23
  117. package/dist/three/tanya.js.map +0 -1
  118. package/dist/three/tara-asset.d.ts +0 -26
  119. package/dist/three/tara-asset.d.ts.map +0 -1
  120. package/dist/three/tara-asset.js +0 -26
  121. package/dist/three/tara-asset.js.map +0 -1
  122. package/dist/three/tara.d.ts +0 -30
  123. package/dist/three/tara.d.ts.map +0 -1
  124. package/dist/three/tara.js +0 -30
  125. package/dist/three/tara.js.map +0 -1
  126. package/dist/three/tess-asset.d.ts +0 -7
  127. package/dist/three/tess-asset.d.ts.map +0 -1
  128. package/dist/three/tess-asset.js +0 -7
  129. package/dist/three/tess-asset.js.map +0 -1
  130. package/dist/three/tess.d.ts +0 -24
  131. package/dist/three/tess.d.ts.map +0 -1
  132. package/dist/three/tess.js +0 -24
  133. package/dist/three/tess.js.map +0 -1
  134. package/dist/three/tushar-asset.d.ts +0 -7
  135. package/dist/three/tushar-asset.d.ts.map +0 -1
  136. package/dist/three/tushar-asset.js +0 -7
  137. package/dist/three/tushar-asset.js.map +0 -1
  138. package/dist/three/tushar.d.ts +0 -22
  139. package/dist/three/tushar.d.ts.map +0 -1
  140. package/dist/three/tushar.js +0 -22
  141. package/dist/three/tushar.js.map +0 -1
  142. package/dist/types.d.ts +0 -149
  143. package/dist/types.d.ts.map +0 -1
  144. package/dist/types.js +0 -140
  145. package/dist/types.js.map +0 -1
  146. package/dist/useAvatar.d.ts +0 -39
  147. package/dist/useAvatar.d.ts.map +0 -1
  148. package/dist/useAvatar.js +0 -35
  149. package/dist/useAvatar.js.map +0 -1
  150. package/src/avatar.d.ts +0 -436
  151. package/src/avatar.js +0 -1378
  152. package/src/behavior.d.ts +0 -42
  153. package/src/behavior.js +0 -114
  154. package/src/camera.js +0 -29
  155. package/src/clips.js +0 -155
  156. package/src/conformance.js +0 -119
  157. package/src/emotions.js +0 -58
  158. package/src/face-core.js +0 -180
  159. package/src/face-myna.d.ts +0 -7
  160. package/src/face-myna.js +0 -786
  161. package/src/face-peep-control-plane.js +0 -167
  162. package/src/face-peep.d.ts +0 -7
  163. package/src/face-peep.js +0 -925
  164. package/src/face-wren.d.ts +0 -7
  165. package/src/face-wren.js +0 -468
  166. package/src/faces.d.ts +0 -12
  167. package/src/faces.js +0 -53
  168. package/src/gaze.js +0 -504
  169. package/src/hand.js +0 -709
  170. package/src/head.js +0 -134
  171. package/src/idle.js +0 -668
  172. package/src/interjections.js +0 -747
  173. package/src/line-art.js +0 -137
  174. package/src/params.js +0 -180
  175. package/src/perform.js +0 -107
  176. package/src/prosody.js +0 -741
  177. package/src/rig.d.ts +0 -31
  178. package/src/rig.js +0 -32
  179. package/src/speech-timing.js +0 -51
  180. package/src/visemes.js +0 -410
@@ -1,1746 +0,0 @@
1
-
2
- /**
3
- * The Blender characters' renderer: the `AvatarRig` contract (`apply(frame)` /
4
- * `destroy()`) over a Blender-authored GLB. One of these drives every compiled
5
- * character, and `createCharacter.ts` is what hands it one.
6
- *
7
- * The whole file is one idea — **the pose channel is the interface, and every
8
- * mapping here is a translation of one channel into the one control that
9
- * renders it.** `scripts/morphs.py` authored the shape keys under the library's
10
- * own channel names precisely so this file never has to interpret a viseme, a
11
- * state or an emotion; it receives a fully mixed pose and moves geometry.
12
- */
13
-
14
- import { JAW_OF_OPEN, REST, VISEME_SHAPES } from "../internal.js";
15
- import type { AvatarFrame, AvatarRig, RigPose } from "../internal.js";
16
- import * as THREE from "three";
17
- import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
18
- import { HARD_BUDGET, pixelRatioFor } from "./budgets.js";
19
-
20
- /**
21
- * The camera, from `scripts/render_frame.py`. It is orthographic and that is
22
- * forced by the albedo rather than chosen: the atlas is a front-orthographic
23
- * projection of the shell, so an atlas texel sits at face-space (u, v) whatever
24
- * depth the triangle carrying it has. A perspective camera disagrees by
25
- * (offset from the axis) x (depth / distance) — which put the hair rim 3 px
26
- * above the hairline it is textured to and opened a black band across the
27
- * forehead. `build_character.setup_scene` has the measurement.
28
- */
29
- const FRAME = { bottom: -0.52, top: 1.46 };
30
- const FRAME_HEIGHT = FRAME.top - FRAME.bottom;
31
- const FRAME_CENTRE = (FRAME.top + FRAME.bottom) / 2;
32
-
33
- /**
34
- * Head motion, in degrees at the channel's own clamp, so the envelope is
35
- * unreachable by construction rather than by a second clamp nobody runs.
36
- *
37
- * Yaw is the tight axis: the albedo is a front-orthographic projection of a
38
- * shallow shell, and a large turn is where that reads as a cardboard cutout
39
- * rather than a head. A ladder rendered at 9/12/15/18/21/25/30 and read at crop
40
- * is clean to 21° on tara and to 18° on the tightest of the others, so 15°
41
- * keeps headroom; pitch and roll opened once `NECK_QUAD` made the neck's follow
42
- * exact. How far a pose may be *held* is a stricter question, measured per
43
- * character (`motion-limits.json`, applied through `holds.ts`).
44
- *
45
- * The yaw twist's two fields are an expansion in the angle, so their error
46
- * grows as θ²/6 — 0.85 % at 15°, on a displacement that is itself a fraction of
47
- * the neck's radius (`morphs.neck_twist`).
48
- *
49
- * TARA-SPECIFIC: reach before an artefact; see 3d-avatar-tara-specific.md.
50
- */
51
- // Exported through `internal.ts` for the instruments that need to put a real
52
- // angle *into* a channel, which is this scaling run backwards. The mocap
53
- // instrument kept its own copy for want of that export and said in a comment
54
- // that the copy would lie the day the envelope moved; it moved.
55
- export const HEAD_CLAMP = 1.4;
56
- export const HEAD_DEG = { yaw: 15, pitch: 24, roll: 8 };
57
-
58
- /**
59
- * Where the head turns about. v 0.36 is the jaw angle and the earlobe, and it
60
- * sits a fifth of a face height *behind* the face plane — a pivot on the
61
- * surface spins the face in place, where a real yaw swings the chin across as
62
- * well as around, which is most of what makes a small turn read.
63
- *
64
- * The asset carries this (`stamp_abi`, as `head_pivot`); this is the fallback
65
- * for a GLB built before the stamp.
66
- */
67
- export const PIVOT = new THREE.Vector3(0.0, 0.36, -0.22);
68
-
69
- /**
70
- * Where the head *tilts* about, from `morphs.ROLL_PIVOT`: the midline just
71
- * above the chin. A roll is a bend of the whole neck, so its centre is far
72
- * below the ear, and a drawn head sells it by holding the chin and swinging
73
- * the crown — Live2D's sample rigs tilt about this same point. About PIVOT
74
- * instead, the chin swung 7 px the other way at 8° and the head read as a
75
- * pendulum hung from the ears.
76
- *
77
- * It sits inside the yaw and pitch, so a turned head still tilts about its own
78
- * chin. The asset carries it (`stamp_abi`, as `roll_pivot`); this is the
79
- * fallback for a GLB built before the stamp, and `morphs.ROLL_PIVOT` has what
80
- * sets the height.
81
- */
82
- export const ROLL_PIVOT = new THREE.Vector3(0.0, 0.05, -0.22);
83
-
84
- /**
85
- * What the head takes with it, from `build_character.HEAD_PARTS`. `Body` stays
86
- * behind, and so does `Neck` — but the neck is not *static*: it follows the
87
- * skull through morph targets of its own (`NECK_QUAD` below).
88
- *
89
- * Without that follow a turn dragged the skull's jaw rim across a throat that
90
- * had not moved, and the rim landed mid-neck as a second jawline — invisible at
91
- * the 400 × 300 tile, obvious at a 3× crop.
92
- *
93
- * The asset carries its own list (`stamp_abi`, as `head_parts`); this is the
94
- * fallback for a GLB built before the stamp.
95
- */
96
- export const HEAD_PARTS = ["Head", "Ears", "Hair", "Eye_L", "Eye_R", "Cavity",
97
- "Teeth_Upper", "Teeth_Lower", "Tongue", "HairLayer"];
98
-
99
- /** One stamped vector, in glTF's frame already, or the rig's own fallback. */
100
- function stampedVec(extras: Record<string, unknown>, key: string, fallback: THREE.Vector3) {
101
- const v = extras[key];
102
- return Array.isArray(v) && v.length === 3 && v.every((n) => typeof n === "number")
103
- ? new THREE.Vector3(v[0], v[1], v[2]) : fallback.clone();
104
- }
105
-
106
- /**
107
- * The body, in face-space units (glTF y is face-space v) and degrees.
108
- *
109
- * Every mechanism is the SVG faces' (`face-core.poseTransforms`), but anatomy
110
- * is the floor here, not a fraction of a cartoon: at 0.6 of peep's travel a
111
- * 30-second listening hold moved the shoulders 2 px and read as a still with a
112
- * tremor — the head moving more than the body carrying it. These put that hold
113
- * at 4-5 px, still slow, and still well under the 1.5 Hz ceiling.
114
- *
115
- * breath A swell, not a slide (`docs/research-biomechanics.md` §6.1): the
116
- * torso scales about a point below the frame, as peep's does about its
117
- * hem. The rise is a little more than the widening because what a
118
- * breath shows in this crop is the upper ribs and clavicles lifting as
119
- * much as the rib cage widening. The neck and head ride the lift at the
120
- * collar, derived rather than tuned (peep's `neckLift`), so the neck
121
- * cannot telescope — the 2-3 mm a seated head really moves with a
122
- * breath.
123
- * lean `torsoLean` as a deformation of the trunk, on the shell itself
124
- * (`morphs.torso_targets`) — hem pinned at the frame's lower edge, the
125
- * shoulders spreading and tipping as they come nearer. Only the head's
126
- * *ride* is here: above the collar the field is flat, so the neck and
127
- * head take a pure translation of `leanRide` and nothing else. That is
128
- * Live2D's measured behaviour rather than a simplification — body angle
129
- * moves every head part by 1.00 ± 0.02 and adds no differential motion
130
- * inside the head (`docs/research-head-rotation.md` § 3.1).
131
- * sway `torsoTurn` as the seated body's inverted pendulum: the whole figure
132
- * rolls about the hips, ~45 cm below the collar, so the trunk shifts
133
- * sideways and tips by a fraction of a degree together. peep slides
134
- * its torso under a head that stays; a photograph cannot, because a
135
- * neck joins them, so here the head rides the trunk and is rolled back
136
- * level by exactly the trunk's tilt — its roll is `headRoll`'s alone,
137
- * as a person shifting their weight goes on holding their eyes level on
138
- * the person they are listening to. One degree at full `torsoTurn`: a
139
- * weight shift (0.16-0.42) moves the collar 1.1-2.9 px, and following
140
- * a full head turn ~7 px — 1.7 cm at the collar, what a seated body
141
- * shifting onto one hip really does.
142
- */
143
- const BODY = {
144
- swellPivot: FRAME.bottom - 0.35 * FRAME_HEIGHT,
145
- rise: 0.016,
146
- widen: 0.012,
147
- /** `landmarks.SHOULDER.top`: where the neck meets the torso. */
148
- collar: -0.24,
149
- /**
150
- * The head group's rigid drop at a full lean. `morphs.py` parses this and
151
- * plateaus `torsoLean`'s field at it above the collar, so the body's
152
- * deformation and the head's transform are one number and meet without a
153
- * seam — the same arrangement `lift.position.y` already has with the breath.
154
- * A uniform scale of the whole figure in its place is a zoom and not a lean:
155
- * the crown travels 4.56× what the eyes do, which is a head being scaled
156
- * rather than carried.
157
- */
158
- leanRide: 0.021,
159
- hip: -2.9,
160
- swayDeg: 1.0,
161
- };
162
-
163
- /**
164
- * Gaze, as iris travel in face heights at `pupilX/Y` = 1, converted to a globe
165
- * rotation on load. Authored as travel, not as an angle, because travel is what
166
- * anyone can measure off the render and the angle is an artefact of this
167
- * build's globe radius (0.24, some 4x anatomical — the sphere is a curved
168
- * backing for a painted iris, not an eyeball).
169
- *
170
- * 0.042 across is ~7 mm on a face 165 mm tall, the iris travel of a ~35°
171
- * version, and 6.4 px at the 400 px tile; the first value (0.026, ~4 px) was
172
- * under the threshold where a look to the screen edge reads as a look rather
173
- * than a drift. Vertical is shorter because a person looking up or down
174
- * carries most of it with the lids and the head.
175
- */
176
- const GAZE_TRAVEL = { x: 0.042, y: 0.026 };
177
- /** `head_mesh.GLOBE_R`. The arc a rotation moves the iris through is r·θ. */
178
- const GLOBE_RADIUS = 0.24;
179
- /** `face_texture.EYE_EXTENT`'s span: one eye tile covers 0.30 face heights. */
180
- const EYE_TILE = 0.3;
181
-
182
- /** Degrees per pose unit: the globe turns `pupil * GAZE_TRAVEL / GLOBE_RADIUS`
183
- * radians, and the head reaches `HEAD_DEG` at the clamp.
184
- *
185
- * Exported through `internal.ts` for the same reason as the head envelope. */
186
- export const EYE_DEG = { x: (GAZE_TRAVEL.x / GLOBE_RADIUS) * 180 / Math.PI, y: (GAZE_TRAVEL.y / GLOBE_RADIUS) * 180 / Math.PI };
187
- const HEAD_UNIT_DEG = { x: HEAD_DEG.yaw / HEAD_CLAMP, y: HEAD_DEG.pitch / HEAD_CLAMP };
188
-
189
- /**
190
- * The mixer's per-rig calibration, passed by the character's entry point and by
191
- * the motion audit so both measure the same face. A pose unit is an angle here
192
- * and a pixel count on an SVG face, so the speech layer's amplitudes are tuned
193
- * per rig rather than in the library.
194
- *
195
- * The shared look table is drawn for a line face, whose pupils cross most of an
196
- * eye, so it put every look in the eyes: a thinking look away was the iris
197
- * parked in the corner of the socket for two thirds of the state, which is
198
- * side-eye, not thought. Here the head carries about 60 % of a look and the
199
- * eyes land a third of the way off centre, where a real eye-head shift leaves
200
- * them (Freedman & Sparks; Pejsa & Andrist). The comment on each target is its
201
- * world angle, x right and y down.
202
- *
203
- * vor Real gain in the light is close to 1. A little under leaves the
204
- * head some say, so a nod carries the eyes a touch with it rather
205
- * than pinning them to the lens. Vertically it is well under:
206
- * her pitch is a shell tipping on a photograph and
207
- * reads as a fraction of what it is, so the eyes' full answer to
208
- * it read as the eyes moving on their own — at 0.8, THINKING's
209
- * up-look rolled the iris to the lid with white beneath it, and
210
- * the reply that followed dropped it into a downcast look with the
211
- * lid riding down, while the head was still coming back. At 0.35
212
- * the eyes mostly go where the head takes them.
213
- * range How far the reflex may carry the eye in the socket. Unlimited,
214
- * an up-look's onset put the whole 8° in the eye before the neck
215
- * moved — the iris pinned under the lid, white beneath it, which
216
- * on this photograph reads as an eye-roll. Up 0.45 is 2.8° and
217
- * down 0.5 is 3.1°; the head covers the rest, as it does once the
218
- * eye nears its effective range. Both were looser (0.55, 0.85)
219
- * and a gap between turns hit both ends of them every time.
220
- * lidFollow Down 0.38 keeps the upper lid on the iris: the lid travels
221
- * 0.068 face units per lid unit and the iris 0.026 per pupil unit.
222
- * Up a little less, so an upward look opens the eye a hair instead
223
- * of dragging the lid along with it.
224
- * avert A conversational look away keeps under half its size in the
225
- * eyes and gives the neck 0.6 of it — on this face the eyes alone
226
- * could only make it a glance sideways.
227
- * head The follow's launch and cruise, in head units. A real head
228
- * reaches a 7-8° shift in about 0.4 s, its speed scaling with the
229
- * size of the shift; the shared amble took a thinking look 1.2 s,
230
- * and the reflex, doing its job, held the eyes in the corner of
231
- * the socket the whole way.
232
- *
233
- * `trunkFollow` is 0.3, under the line faces' 0.45. Live2D's face-tracking
234
- * sample gives the body a third of the head's yaw (BodyAngleX 10 against
235
- * AngleX 30; docs/research-head-rotation.md § 5 item 6), and that is the
236
- * benchmark to sit at, not above. Here it matters for a reason a line face
237
- * does not have: the neck's outline holds under a twist by construction
238
- * (`morphs.neck_twist`), so the trunk's sway is what is left moving it — a
239
- * quarter to a third of it at the yaw peak in the recorded call, read as the
240
- * neck sliding.
241
- */
242
- export const CHARACTER_TUNING = {
243
- prosodyHeadGain: 1.0, prosodyFaceGain: 1, saccadeGain: 2.4, aversionGain: 1.8,
244
- trunkFollow: 0.3,
245
- // **The speaking face's upper half, sized for a photograph.** A reviewer read
246
- // her speech as delivered "with zero emotion or movement in her eyes or
247
- // forehead", against the same reviewer's praise for `CANT_HEAR`. The two are
248
- // the same channels at different values, and the gap is measurable in the
249
- // expression maps: `CANT_HEAR` above is knit 0.14/0.45 plus inner 0.22/0.5,
250
- // about 0.75 of map weight, where speech averaged 0.19.
251
- //
252
- // `floor` is the larger half of the answer. `browRaise` is split by sign into
253
- // two maps and *both* read as nothing near zero, so the shared [-0.14, 0.18]
254
- // range spends most phrases in a dead band recruiting no light at all. The
255
- // band is skipped rather than the range widened, because what carries the
256
- // read is the sign a phrase commits to, not how far it goes — and the ceiling
257
- // stays under a beat's 0.34 for the reason `POSE.brow` gives.
258
- //
259
- // `forms` moves draws off the plain raise and onto the inner lift, which on
260
- // this asset is the only brow shape besides the raise that recruits light at
261
- // all (`EXPRESSION_WEIGHT`: there is no map for `browAngle`, and its 0.026 of
262
- // travel is the smallest of the three). `prosody.js` makes the inner lift the
263
- // rarest form deliberately, because a face that keeps lifting its inner brows
264
- // reads as worried — that rule is written for the line faces, and the face it
265
- // is being relaxed for holds `browInner` at 0.22 for the whole of `CANT_HEAR`
266
- // and was praised for it. Still a transient on a 0.55 s envelope, never a
267
- // held shape: the prohibition is on the hold, not the event.
268
- brows: {
269
- range: [-0.28, 0.24],
270
- floor: 0.11,
271
- forms: [
272
- { p: 0.38, inner: 0.00, angle: 0.00 },
273
- { p: 0.34, inner: 0.26, angle: 0.00 },
274
- { p: 0.28, inner: 0.00, angle: 0.30 },
275
- ],
276
- },
277
- states: {
278
- // The shared WORKING pose without its AU4 brows. On peep, brows-down is
279
- // what makes reading read as effort rather than a blank face; on a
280
- // photograph, over lids that are already following the eyes down, it
281
- // closes them to a squint the owner read as straining at the screen, not
282
- // working. Her reading scan carries the state instead: eyes off the user,
283
- // stepping along a line, the way a person at their own display looks.
284
- WORKING: {
285
- pose: { headPitch: 0.04, lidL: -0.08, lidR: -0.08, shoulderL: 0.06, shoulderR: 0.06 },
286
- },
287
- // Straining to hear, without the squint. Its AU7 lifts her lower lid and
288
- // pushes the cheek up under it: at 0.75 the eyes closed to slits over a
289
- // dark band, and under brows at -0.45, with the eyes countered into the
290
- // corner of the socket (`USER_EAR` below), she read as giving the user a
291
- // suspicious side-eye. The lean in and the ear offered carry the state,
292
- // which the shared comment already says of them. The brows keep a small
293
- // knit with the inner ends up: effort that is also asking. The mouth is
294
- // pressed at a photograph's scale; the shared -0.22 corners clear peep's
295
- // drawn smile, and a compiled face rests neutral.
296
- //
297
- // The lean is attentive-sized, not the shared 0.70: 0.22 sits in the
298
- // sustained band (+0.15–0.25, research-biomechanics.md §6.3), and the ear
299
- // and chin carry the rest.
300
- CANT_HEAR: {
301
- pose: {
302
- torsoLean: 0.22, headPitch: 0.10,
303
- browRaiseL: -0.14, browRaiseR: -0.14, browInnerL: 0.22, browInnerR: 0.18,
304
- mouthPress: 0.40, mouthCornerL: -0.10, mouthCornerR: -0.10,
305
- },
306
- },
307
- // Hunting for a control, without the squint: the same lower lid and cheek
308
- // at 0.40 left the eyes half-lidded from below over a dark band. The hunt
309
- // is the wander and the flick; the face only has to be not smiling, and
310
- // on a mouth that rests neutral that is a small press, not peep's -0.25
311
- // corners.
312
- SEARCHING_SCREEN: {
313
- pose: { mouthPress: 0.40, mouthCornerL: -0.08, mouthCornerR: -0.08,
314
- browRaiseL: -0.10, browRaiseR: -0.06 },
315
- },
316
- },
317
- oculomotor: {
318
- angles: { eye: EYE_DEG, head: HEAD_UNIT_DEG },
319
- vor: { x: 0.8, y: 0.35 },
320
- range: { x: 0.8, up: 0.45, down: 0.5 },
321
- lidFollow: { down: 0.38, up: 0.30 },
322
- avert: { eye: 0.45, head: 0.6 },
323
- head: { accel: 16, speed: 3.5 },
324
- targets: {
325
- // 2.0° right, 2.5° down: her own display, read level and a little to
326
- // the side (WORKING, and OFFLINE's wait). The upper lid follows the eye
327
- // down (`lidFollow`), and a photographed eye 6.8° down — where this
328
- // sat — hooded to a lid of 0.25-0.43 against listening's 0.14, read as
329
- // a squint rather than reading. Here it holds 0.12-0.18 through the
330
- // scan, and being off the user to the side is what says "busy".
331
- OWN_SCREEN: { px: 0.14, py: 0.24, hx: 0.10, hy: 0.06 },
332
- // 2.6° of head turn toward the user's side and 3.1° of roll, the eyes
333
- // countered 2.0° back onto them: the ear offered, contact held from
334
- // inside the socket. The shared 0.42 counter held her iris against the
335
- // corner of the socket with white on one side, which on a photograph is
336
- // side-eye; the roll is the cue that says "ear", so it keeps its size.
337
- USER_EAR: { px: -0.20, py: 0.05, hx: 0.40, hy: 0.02, roll: 0.55 },
338
- // 2.8° up, nearly all of it the head — the same total the shared target
339
- // gives on her, redistributed. The shared split puts 1.0° on the eyes,
340
- // and the middle of a screen is the one place a hunt keeps returning to,
341
- // so it is the worst place to sit with the iris off-centre.
342
- SCREEN_CENTER: { px: 0.00, py: -0.06, hx: 0.00, hy: -0.14 },
343
- // 9.3° to the side, 1.5° up, the head carrying over half. The shared
344
- // ones put the eyes at the edge of her socket (0.78 of 0.8) — white on
345
- // one side again, and a hunt that reads as shifty. SEARCHING_SCREEN,
346
- // REVIEWING_SCREEN and DISTRACTED look here.
347
- SCREEN_LEFT: { px: -0.42, py: -0.10, hx: -0.80, hy: -0.05 },
348
- SCREEN_RIGHT: { px: 0.42, py: -0.10, hx: 0.80, hy: -0.05 },
349
- // 6.4° up, three quarters of it the head. The shared target is 9.6° on
350
- // her — higher than her own AWAY_THINKING, so it read as looking over
351
- // the monitor rather than at the top of it, and it made the two longest
352
- // hops in SEARCHING_SCREEN's set (14.6° from SCREEN_WORK, 12.4° from
353
- // either side) on a face whose looks are all scaled to a head-and-
354
- // shoulders crop.
355
- SCREEN_TOP: { px: 0.00, py: -0.26, hx: 0.00, hy: -0.28 },
356
- // 7.0° left, 3.2° down, for the same reason, split evenly.
357
- SCREEN_WORK: { px: -0.35, py: 0.18, hx: -0.55, hy: 0.12 },
358
- // 7.8° left, 8.1° up.
359
- AWAY_THINKING: { px: -0.30, py: -0.40, hx: -0.75, hy: -0.33, roll: 0.08 },
360
- // 7.6° right, 7.4° up.
361
- AWAY_RIGHT: { px: 0.30, py: -0.36, hx: 0.72, hy: -0.30, roll: -0.06 },
362
- // 6.8° left, 5.5° down: shallower than the up-looks, since down on a
363
- // face this real is the one read as downcast.
364
- AWAY_DOWN: { px: -0.28, py: 0.45, hx: -0.62, hy: 0.16, roll: 0.04 },
365
- // 7.8° right, level.
366
- AWAY_SIDE: { px: 0.32, py: 0.08, hx: 0.72, hy: 0.02, roll: -0.03 },
367
- },
368
- },
369
- } as const;
370
-
371
- /** How far a closing lid darkens the eye it covers, from where it starts to.
372
- * The lid's margin and lashes throw the strip of white still showing into
373
- * shadow; the atlas's socket shade was measured with the eye open and cannot
374
- * know that, so without this the last frames before a blink closes show a
375
- * bright line under a dark lash band — what a reviewer called the sclera
376
- * tearing across the lid. */
377
- const LID_SHADE = 0.55;
378
- const LID_SHADE_FROM = 0.3;
379
-
380
- /**
381
- * The eye material, taught to hold its socket still while the globe turns.
382
- *
383
- * The eye atlas is two tiles (`face_texture.write_eye_atlas`): the globe, and
384
- * a linear multiplier holding what the lids and the socket put on the eye —
385
- * the shadow under the upper lid, the caruncle, the shading into each corner.
386
- * A real eye turns under all of that, and when it was baked into the globe a
387
- * sideways look swung the caruncle into the middle of the white. So the
388
- * globe's texel is read where the surface point *was* and the socket's where
389
- * it *is*: the planar UV of the rotated position, which is the unrotated UV
390
- * plus the rotation's displacement through the UV's own gradient. One extra
391
- * texture read on two small meshes; no pass, no draw call.
392
- */
393
- function socketed(base: THREE.MeshStandardMaterial, side: number,
394
- gaze: { value: THREE.Matrix3 }, deep: boolean,
395
- lid: { value: number }) {
396
- const material = base.clone();
397
- // d(u)/dx and d(v)/dy of `head_mesh.eye_uvs`: the globe is half the atlas
398
- // wide, the right eye reads it mirrored, and glTF flips v.
399
- const socket = { value: new THREE.Vector2((0.5 * -side) / EYE_TILE, -1 / EYE_TILE) };
400
- material.onBeforeCompile = (shader) => {
401
- shader.uniforms.uGaze = gaze;
402
- shader.uniforms.uSocket = socket;
403
- shader.uniforms.uLidShade = lid;
404
- // The globe's hidden skirt turns at the frame's motion depth like the skin
405
- // that hides it; its visible cap carries a field of exactly zero, so the
406
- // socket, the iris and gaze below are unaffected by this. It turns at the
407
- // *frame's* rotation and not its own, which is what `undoGaze` reads back
408
- // out of `uGaze` — hence this sitting after that uniform is bound.
409
- if (deep) turnDeep(shader, true);
410
- shader.vertexShader = shader.vertexShader
411
- .replace("#include <uv_pars_vertex>",
412
- "#include <uv_pars_vertex>\nuniform mat3 uGaze;\nuniform vec2 uSocket;\nvarying vec2 vSocketUv;")
413
- .replace("#include <uv_vertex>",
414
- "#include <uv_vertex>\nvSocketUv = vMapUv + vec2(0.5, 0.0)"
415
- + " + uSocket * ((uGaze * position).xy - position.xy);");
416
- shader.fragmentShader = shader.fragmentShader
417
- .replace("#include <uv_pars_fragment>",
418
- "#include <uv_pars_fragment>\nvarying vec2 vSocketUv;\nuniform float uLidShade;")
419
- .replace("#include <map_fragment>",
420
- "vec3 socket = uLidShade * 2.0 * texture2D( map, vSocketUv ).rgb;\n#include <map_fragment>\ndiffuseColor.rgb *= socket;")
421
- .replace("#include <emissivemap_fragment>",
422
- "#include <emissivemap_fragment>\ntotalEmissiveRadiance *= socket;");
423
- };
424
- material.customProgramCacheKey = () => (deep ? "tara-eye-socket-deep" : "tara-eye-socket");
425
- return material;
426
- }
427
-
428
- /**
429
- * tess's "ah" photograph, down the midline of her open mouth: the colour at
430
- * each depth into the lip opening, 0 at the upper lip's inner rim and 1 at the
431
- * lower's, in sRGB as the photograph has it. Read at MediaPipe's inner-lip
432
- * points 13 and 14, which is the same rim `build_character.lip_aperture` stamps.
433
- *
434
- * The shape is the thing to keep, and every reviewer's "pink and flat" is its
435
- * absence: dark in the shadow of the upper arch, rising to the tongue's front
436
- * two-thirds of the way down, and falling again into the floor of the mouth
437
- * above the lower arch. Where either arch covers a stretch of this, what the
438
- * photograph measured there is enamel and was left out — the arches draw
439
- * themselves.
440
- */
441
- const INTERIOR: ReadonlyArray<readonly [number, readonly [number, number, number]]> = [
442
- [0.15, [60, 22, 26]],
443
- [0.30, [84, 35, 41]],
444
- [0.45, [122, 62, 68]],
445
- [0.60, [159, 87, 93]],
446
- [0.70, [144, 60, 61]],
447
- [0.80, [52, 17, 15]],
448
- [0.90, [40, 12, 12]],
449
- ];
450
-
451
- /**
452
- * The photograph's enamel is 158 of 255 and the rendered upper arch's is about
453
- * 232, so the profile is carried over at the ratio of the two: the interior is
454
- * as dark *against the teeth* as hers is, which is the only comparison anyone
455
- * makes looking into a mouth.
456
- */
457
- const INTERIOR_GAIN = 232 / 158;
458
-
459
- /** The same, applied to the photograph in linear light, where its tile is read. */
460
- const INTERIOR_GAIN_LINEAR = INTERIOR_GAIN ** 2.2;
461
-
462
- /**
463
- * How open her mouth is in that photograph: 185 px between the inner rims at the
464
- * midline over 229 between the inner corners (MediaPipe 13/14 and 78/308). A
465
- * speaking mouth is rarely a third of that, and the light reaching into it
466
- * falls with the opening, so the profile is dimmed by the square root of the
467
- * ratio — between the solid angle's own square law, which turned every
468
- * conversational viseme into a hole, and none at all, which left each one the
469
- * lit pink band reviewers called flat.
470
- */
471
- const INTERIOR_OPEN = 185 / 229;
472
-
473
- /** That dimming, in GLSL, over the `uAperture` the mouth's shaders share. */
474
- const OPENNESS = `sqrt(clamp(uAperture.w / (2.0 * uAperture.z * ${INTERIOR_OPEN.toFixed(3)}), 0.0, 1.0))`;
475
-
476
- /**
477
- * Where a raised tongue is read from: `tongue` = 1 paints the surface it lifts
478
- * into the opening from `from` down to `at` — `at` the photograph's brightest
479
- * row — rather than in the shadow of the upper arch it has moved into. A span
480
- * and not a row: read at one row, every height of it took the same texels,
481
- * and the photograph's tongue drew as vertical streaks down a slab. Squeezed
482
- * into the span it keeps its crown, rolling off toward the teeth.
483
- */
484
- const TONGUE_LIFT = { from: 0.25, at: 0.60 };
485
-
486
- /** An `INTERIOR` knot as linear light, at `INTERIOR_GAIN`. */
487
- function interiorKnot(rgb: readonly [number, number, number]) {
488
- return new THREE.Color().setRGB(
489
- ...(rgb.map((v) => Math.min(1, (v * INTERIOR_GAIN) / 255)) as [number, number, number]), THREE.SRGBColorSpace);
490
- }
491
-
492
- const glslColor = (c: THREE.Color) => `vec3(${c.r.toFixed(5)}, ${c.g.toFixed(5)}, ${c.b.toFixed(5)})`;
493
-
494
- /** GLSL: `vec3 interior`, the profile at depth `mouthA` in the opening. */
495
- function interiorProfile() {
496
- let profile = `vec3 interior = ${glslColor(interiorKnot(INTERIOR[0][1]))};\n`;
497
- for (let i = 1; i < INTERIOR.length; i++) {
498
- const [a0] = INTERIOR[i - 1];
499
- const [a1, rgb] = INTERIOR[i];
500
- profile += `interior = mix(interior, ${glslColor(interiorKnot(rgb))}, clamp((mouthA - ${a0.toFixed(3)}) / ${(a1 - a0).toFixed(3)}, 0.0, 1.0));\n`;
501
- }
502
- return profile;
503
- }
504
-
505
- /**
506
- * The inside of the mouth — the tongue and the cavity behind it — painted from
507
- * a photograph of one, at where each point sits in the lip opening the pose
508
- * has made.
509
- *
510
- * The opening, and not the surface, because that is how the photograph's
511
- * profile arises: the light reaching into a mouth is gated by the lips and
512
- * shadowed by the upper arch, so the same point of tongue is bright when the
513
- * jaw drops and dark when it closes. A shade stuck to the surface — what this
514
- * replaced — reads as a lit object behind a hole, which at viseme D was a flat
515
- * mauve plate, and at C and H put the tongue's bright crest directly under the
516
- * upper teeth with dark below it: the order of the photograph, inverted.
517
- *
518
- * Emitted rather than lit, and entirely: this is sampled light, like the 75%
519
- * of the face that is the photograph verbatim, and a key from above lights the
520
- * dorsum brightest at its back, which is the one thing a mouth never looks
521
- * like. Both surfaces read the same picture, so her mouth's absence of any
522
- * tongue-to-cavity edge carries over; only a raised tongue is told apart, by
523
- * `lift`, because a tongue tip at the teeth is what viseme H is.
524
- */
525
- function mouthInterior(base: THREE.MeshStandardMaterial,
526
- aperture: { value: THREE.Vector4 }, lift: { value: number },
527
- extent: readonly number[] | null) {
528
- const material = base.clone();
529
- // The photograph itself where the asset carries it (`project_albedo.project_interior`),
530
- // and its midline where it does not.
531
- const tiled = material.map !== null && extent !== null;
532
- let paint: string;
533
- if (tiled) {
534
- const [s0, s1, a0, a1] = extent!;
535
- paint = `vec2 tileAt = clamp(vec2((mouthX - ${s0.toFixed(3)}) / ${(s1 - s0).toFixed(3)},`
536
- + ` (mouthA - ${a0.toFixed(3)}) / ${(a1 - a0).toFixed(3)}), 0.0, 1.0);\n`
537
- + `vec3 interior = min(texture2D(map, tileAt).rgb * ${INTERIOR_GAIN_LINEAR.toFixed(4)}, vec3(1.0));\n`;
538
- } else {
539
- paint = interiorProfile();
540
- }
541
- material.onBeforeCompile = (shader) => {
542
- shader.uniforms.uAperture = aperture;
543
- shader.uniforms.uMouthLift = lift;
544
- shader.vertexShader = shader.vertexShader
545
- .replace("#include <common>", "#include <common>\nvarying vec2 vMouthAt;")
546
- // `transformed` after the morphs, unlike every other shade in this file:
547
- // where the surface *is* in the opening is the whole question.
548
- .replace("#include <morphtarget_vertex>",
549
- "#include <morphtarget_vertex>\nvMouthAt = transformed.xy;");
550
- shader.fragmentShader = shader.fragmentShader
551
- .replace("#include <common>",
552
- "#include <common>\nvarying vec2 vMouthAt;\nuniform vec4 uAperture;\nuniform float uMouthLift;")
553
- .replace("#include <map_fragment>", "diffuseColor.rgb = vec3(0.0);")
554
- .replace("#include <emissivemap_fragment>",
555
- "#include <emissivemap_fragment>\n"
556
- + "float mouthA0 = (uAperture.y - vMouthAt.y) / uAperture.w;\n"
557
- + `float mouthA = mouthA0 < ${TONGUE_LIFT.at.toFixed(2)} ? mix(mouthA0, ${TONGUE_LIFT.from.toFixed(2)}`
558
- + ` + max(mouthA0, 0.0) * ${((TONGUE_LIFT.at - TONGUE_LIFT.from) / TONGUE_LIFT.at).toFixed(4)}, uMouthLift) : mouthA0;\n`
559
- + "float mouthX = (vMouthAt.x - uAperture.x) / uAperture.z;\n"
560
- + paint
561
- + `totalEmissiveRadiance = interior * ${OPENNESS};`);
562
- };
563
- material.customProgramCacheKey = () => (tiled ? "mouth-interior-tile" : "mouth-interior");
564
- return material;
565
- }
566
-
567
- /**
568
- * The lower arch, lit by the light the tongue beside it is lit by.
569
- *
570
- * Its tile is toned against the upper arch on a smile (`project_albedo`), a
571
- * mouth open wide and pulled back, where the lower teeth take nearly the upper
572
- * ones' light. Speaking, they are the deepest thing the opening shows, and
573
- * tess's "ah" — open wider than any viseme — shows no lower crown at all: the
574
- * row above where they would be is the darkest in the photograph. Left at the
575
- * smile's tone, the sliver a C or D uncovers was a lit grey rule between that
576
- * dark and the lip, which reads as a wire and not as teeth.
577
- *
578
- * So the arch takes the interior's light where it stands: the profile at its
579
- * depth in the opening over the profile's brightest row, which is the tongue
580
- * lit as well as anything in a mouth is, and the same openness dimming. The
581
- * ratio is in linear light and per channel, so the enamel goes as dark and as
582
- * warm as the mouth around it, and its top edge — higher in the opening —
583
- * keeps the most. The upper arch, at the lip and in the light, keeps its own.
584
- */
585
- function lowerArch(base: THREE.MeshStandardMaterial, aperture: { value: THREE.Vector4 }) {
586
- const material = base.clone();
587
- const peak = interiorKnot(INTERIOR.reduce((a, b) => (b[1][0] + b[1][1] + b[1][2] > a[1][0] + a[1][1] + a[1][2] ? b : a))[1]);
588
- material.onBeforeCompile = (shader) => {
589
- shader.uniforms.uAperture = aperture;
590
- shader.vertexShader = shader.vertexShader
591
- .replace("#include <common>", "#include <common>\nvarying vec2 vMouthAt;")
592
- .replace("#include <morphtarget_vertex>", "#include <morphtarget_vertex>\nvMouthAt = transformed.xy;");
593
- shader.fragmentShader = shader.fragmentShader
594
- .replace("#include <common>", "#include <common>\nvarying vec2 vMouthAt;\nuniform vec4 uAperture;")
595
- // Both terms: the teeth are three-quarters sampled light (`emissive`
596
- // in the build's `flat_material`), so dimming the lit part alone does
597
- // almost nothing.
598
- .replace("#include <map_fragment>",
599
- "#include <map_fragment>\n"
600
- + "float mouthA = (uAperture.y - vMouthAt.y) / uAperture.w;\n"
601
- + interiorProfile()
602
- + `vec3 archLight = min(interior / ${glslColor(peak)}, vec3(1.0)) * ${OPENNESS};\n`
603
- + "diffuseColor.rgb *= archLight;")
604
- .replace("#include <emissivemap_fragment>", "#include <emissivemap_fragment>\ntotalEmissiveRadiance *= archLight;");
605
- };
606
- material.customProgramCacheKey = () => "lower-arch";
607
- return material;
608
- }
609
-
610
- /**
611
- * The upper arch, with the gaps between its teeth showing the mouth.
612
- *
613
- * `project_albedo.project_teeth` paints everything under the incisal edge
614
- * that is not a tooth — the notches between the tips — in `notch`, a neutral
615
- * near-black, because the tile cannot know what the mouth behind it will be.
616
- * Rendered, that strip came out a hard grey saw along the bottom of the arch:
617
- * neutral against a warm interior, and at the arch's light rather than the
618
- * mouth's. A gap in a row of teeth is a window onto the cavity, so it takes
619
- * the cavity's own light at that height in the opening, and only the gap does
620
- * — `notch` is darker than any enamel the tile carries, shaded overhang
621
- * included, so the test is the texel's own brightness, and a filtered texel
622
- * on a tip's edge takes a share of each.
623
- */
624
- function upperArch(base: THREE.MeshStandardMaterial, aperture: { value: THREE.Vector4 }) {
625
- const material = base.clone();
626
- material.onBeforeCompile = (shader) => {
627
- shader.uniforms.uAperture = aperture;
628
- shader.vertexShader = shader.vertexShader
629
- .replace("#include <common>", "#include <common>\nvarying vec2 vMouthAt;")
630
- .replace("#include <morphtarget_vertex>", "#include <morphtarget_vertex>\nvMouthAt = transformed.xy;");
631
- shader.fragmentShader = shader.fragmentShader
632
- .replace("#include <common>", "#include <common>\nvarying vec2 vMouthAt;\nuniform vec4 uAperture;")
633
- .replace("#include <map_fragment>",
634
- "#include <map_fragment>\n"
635
- + "float archGap = 1.0 - smoothstep(0.01, 0.30, dot(diffuseColor.rgb, vec3(0.2126, 0.7152, 0.0722)));\n"
636
- + "diffuseColor.rgb *= 1.0 - archGap;")
637
- .replace("#include <emissivemap_fragment>",
638
- "#include <emissivemap_fragment>\n"
639
- + "float mouthA = (uAperture.y - vMouthAt.y) / uAperture.w;\n"
640
- + interiorProfile()
641
- + `totalEmissiveRadiance = mix(totalEmissiveRadiance, interior * ${OPENNESS}, archGap);`);
642
- };
643
- material.customProgramCacheKey = () => "upper-arch";
644
- return material;
645
- }
646
-
647
- /** `scripts/head_mesh.MOTION_DEPTH_ATTR`, as GLTFLoader names it: lowercased. */
648
- const MOTION_DEPTH = "_motion_depth";
649
-
650
- /**
651
- * The vertex half of `motionDepth`, which the eye's socket shader needs too.
652
- *
653
- * It is a free function rather than inheritance because `Material.copy` does not
654
- * carry `onBeforeCompile`: cloning a motion-depth material and giving the clone
655
- * a second injection drops the first one silently, with no error and a rest pose
656
- * that looks right. The two are composed by hand instead.
657
- */
658
- function turnDeep(shader: { vertexShader: string }, undoGaze = false) {
659
- // `undoGaze` is for the globes, and without it the cure draws a worse defect
660
- // than the one it removes. A globe is *rotated* for gaze, so its own
661
- // `modelViewMatrix` carries that rotation: building the offset from it tilts
662
- // (0, 0, Δz) into the screen plane and slides the hidden skirt out past the
663
- // temple on a look alone, head square on, where the skin it hides behind has
664
- // not moved at all. The frame's rotation is the one the skirt must follow, and
665
- // `uGaze` is exactly the extra rotation to take back out — multiplying the
666
- // vector from the left is its inverse, so this needs no second uniform.
667
- const depth = `vec3(0.0, 0.0, ${MOTION_DEPTH})`;
668
- shader.vertexShader = shader.vertexShader
669
- .replace("#include <common>", `#include <common>\nattribute float ${MOTION_DEPTH};`)
670
- .replace("#include <project_vertex>",
671
- "#include <project_vertex>\n"
672
- + `mvPosition.xy += (modelViewMatrix * vec4(${undoGaze ? `${depth} * uGaze` : depth}, 0.0)).xy;\n`
673
- + "gl_Position = projectionMatrix * mvPosition;");
674
- }
675
-
676
- /**
677
- * The head's frame, taught to turn as if it sat deeper than it does.
678
- *
679
- * Rotated at their real depth, the ears, crown, side hair and outline move
680
- * nearly as far as the nose, and a nod reads as the whole head dropping
681
- * (docs/research-head-rotation.md § 1). A real head's frame sits near the axis
682
- * and barely moves, and that differential is what reads as rotation. Each
683
- * vertex of the skin, hair and ears carries a Δz toward the viewer
684
- * (`head_mesh.motion_depth`, ≤ 0 and zero across the features), and this moves
685
- * it on screen by the rotated Δz: the view-space xy of modelView · (0, 0, Δz).
686
- * Depth, draw order and lighting keep the real position. At rest that xy is
687
- * exactly zero through the orthographic camera, so the drawing is untouched.
688
- *
689
- * The attribute decides, not the mesh name: the neck shares the skin material
690
- * and has no field, which is why each head shell gets a clone. The same clone
691
- * wears the expression maps, when the asset has them: the shells textured from
692
- * the face atlas are exactly the ones those maps are registered to.
693
- */
694
- function motionDepth(base: THREE.MeshStandardMaterial, expression?: Expression, shut?: Shut) {
695
- const material = base.clone();
696
- material.onBeforeCompile = (shader) => {
697
- turnDeep(shader);
698
- if (shut) {
699
- Object.assign(shader.uniforms, shut.uniforms);
700
- shader.vertexShader = shader.vertexShader
701
- .replace("#include <common>", `#include <common>\n${SHUT_VERTEX_HEAD}`)
702
- .replace("#include <uv_vertex>", `#include <uv_vertex>\n${SHUT_VERTEX}`);
703
- shader.fragmentShader = shader.fragmentShader
704
- .replace("#include <common>", `#include <common>\n${SHUT_UNIFORMS}`)
705
- .replace("#include <map_fragment>", `#include <map_fragment>\n${SHUT_FRAGMENT}`)
706
- .replace("#include <emissivemap_fragment>",
707
- "#include <emissivemap_fragment>\n"
708
- + "totalEmissiveRadiance = mix(totalEmissiveRadiance, emissive * shutColour, shutWeight);");
709
- }
710
- if (!expression) return;
711
- const n = expression.names.length;
712
- Object.assign(shader.uniforms, expression.uniforms);
713
- shader.fragmentShader = shader.fragmentShader
714
- .replace("#include <common>", `#include <common>\n${EXPRESSION_UNIFORMS(n)}`)
715
- .replace("#include <map_fragment>", `#include <map_fragment>\n${EXPRESSION_FRAGMENT(n)}`)
716
- .replace("#include <emissivemap_fragment>",
717
- "#include <emissivemap_fragment>\ntotalEmissiveRadiance *= expression;");
718
- };
719
- material.customProgramCacheKey = () =>
720
- ["tara-motion-depth", expression && `expression-${expression.names.length}`, shut && "shut"]
721
- .filter(Boolean).join("-");
722
- return material;
723
- }
724
-
725
- /**
726
- * Expression maps: the light a smile or a raised brow changes, which the
727
- * geometry cannot.
728
- *
729
- * A pixel of this face is 75% photograph, emitted, so the most a morph can
730
- * change by moving skin is a quarter of its shading — and that shading is the
731
- * neutral photograph's, which has no nasolabial fold to deepen and no forehead
732
- * line to show. A cheek morph measured 0.00% of the tile changed. So an asset
733
- * can carry, per expression, a grey ratio taken from a photograph of the same
734
- * face making it (`scripts/expression_maps.py`), read at the texel's *rest*
735
- * position: the morph that lifts the cheek carries the lifted cheek's light up
736
- * with its texture, so light and shape arrive together.
737
- *
738
- * Per side, because the channels are. The weights cross over at the midline
739
- * rather than switching there, so a one-sided smile does not cut its fold's
740
- * light off in a line down the philtrum. An asset with no maps is a face whose
741
- * light never changes, which is a correct face.
742
- */
743
- const EXPRESSION_SPLIT = 0.03;
744
- /**
745
- * Each map's weight, from one side's channels, as a fraction of the channel
746
- * value at which it is at its photograph's full strength. The smile's is the
747
- * mixer's broad smile (0.48-0.58 in the happy states) with room above it; a
748
- * raise at the mixer's highest (0.42) gets most of the lines; a knit at
749
- * `CANT_HEAR`'s −0.14 gets a third of the crease and the deepest frown the
750
- * mixer asks for (−0.45) gets all of it; the inner brow's concern is full at
751
- * its 0.55 and a press at the 0.40-0.55 most states hold gets most of the chin.
752
- *
753
- * A map named here that the asset lacks is simply not read, and a map the
754
- * asset has that is not named here stays at weight 0: an asset newer than its
755
- * rig loses the light the rig cannot place, not the face.
756
- */
757
- const EXPRESSION_WEIGHT: Record<string, (at: (channel: string) => number) => number> = {
758
- smile: (at) => at("mouthCorner") / 0.8,
759
- raise: (at) => at("browRaise") / 0.6,
760
- knit: (at) => -at("browRaise") / 0.45,
761
- inner: (at) => at("browInner") / 0.5,
762
- press: (at) => at("mouthPress") / 0.7,
763
- };
764
- const EXPRESSION_UNIFORMS = (n: number) => [
765
- "uniform sampler2D uExpression;", "uniform vec4 uExpressionTile;", "uniform vec2 uExpressionU;",
766
- "uniform vec4 uExpressionStrip;", "uniform float uExpressionStep;",
767
- "uniform float uExpressionScale;",
768
- `uniform float uExpressionL[${n}];`, `uniform float uExpressionR[${n}];`,
769
- ].join("\n");
770
- // Every tap is read whatever its weight: a texture read under a branch that
771
- // differs across the midline has no derivatives to choose its mip from.
772
- const EXPRESSION_FRAGMENT = (n: number) => [
773
- "vec2 expressionAt = clamp(uExpressionTile.xz + uExpressionTile.yw * vMapUv, 0.0, 1.0);",
774
- "vec2 expressionUv = uExpressionStrip.zw + expressionAt * uExpressionStrip.xy;",
775
- "float expressionSide = smoothstep(",
776
- ` -${EXPRESSION_SPLIT}, ${EXPRESSION_SPLIT}, uExpressionU.x + uExpressionU.y * vMapUv.x);`,
777
- "float expressionLog = 0.0;",
778
- `for (int i = 0; i < ${n}; i++) {`,
779
- " float ratio = texture2D(uExpression, expressionUv + vec2(float(i) * uExpressionStep, 0.0)).r;",
780
- " expressionLog += mix(uExpressionL[i], uExpressionR[i], expressionSide)",
781
- " * log(max(ratio * uExpressionScale, 0.01));",
782
- "}",
783
- "float expression = exp(expressionLog);",
784
- "diffuseColor.rgb *= expression;",
785
- ].join("\n");
786
-
787
- interface Expression {
788
- readonly map: THREE.Texture;
789
- readonly names: readonly string[];
790
- readonly left: Float32Array;
791
- readonly right: Float32Array;
792
- readonly uniforms: Record<string, THREE.IUniform>;
793
- }
794
-
795
- /** The maps from their carrier's node, or `null` for an asset whose carrier
796
- * does not say what this rig reads — a face with no expression maps is a
797
- * correct face, and a map read with the wrong layout is not. */
798
- function expressive(carrier: THREE.Mesh): Expression | null {
799
- const { expression_tile: tile, expression_u: u, expression_maps: names,
800
- expression_layout: layout, expression_unity: unity } = carrier.userData;
801
- const map = (carrier.material as THREE.MeshStandardMaterial).map;
802
- if (!map || !Array.isArray(tile) || !Array.isArray(u) || !Array.isArray(layout)
803
- || typeof unity !== "number" || !Array.isArray(names) || !names.length) return null;
804
- // A ratio, not a colour. The loader tags every base colour sRGB.
805
- map.colorSpace = THREE.NoColorSpace;
806
- map.needsUpdate = true;
807
- const [width, height, pad, stripWidth, stripHeight] = layout as number[];
808
- const left = new Float32Array(names.length);
809
- const right = new Float32Array(names.length);
810
- return {
811
- map, names: names.map(String), left, right,
812
- uniforms: {
813
- uExpression: { value: map },
814
- uExpressionTile: { value: new THREE.Vector4(tile[0], tile[1], tile[2], tile[3]) },
815
- uExpressionU: { value: new THREE.Vector2(u[0], u[1]) },
816
- uExpressionStrip: { value: new THREE.Vector4(width / stripWidth, height / stripHeight,
817
- pad / stripWidth, pad / stripHeight) },
818
- uExpressionStep: { value: (width + pad) / stripWidth },
819
- uExpressionScale: { value: 255 / unity },
820
- uExpressionL: { value: left },
821
- uExpressionR: { value: right },
822
- },
823
- };
824
- }
825
-
826
- /** `scripts/build_character.LID_SHUT_ATTR`'s fields, as GLTFLoader names
827
- * them: the displacement's u and v, and the weight. */
828
- const LID_SHUT_ATTRS = ["_lid_shut_u", "_lid_shut_v", "_lid_shut_w"] as const;
829
-
830
- /**
831
- * The shut lids: a closing lid shows the photograph of the eyes closed.
832
- *
833
- * A 2.5-D shell has no hidden skin, so the lid that closes is the open eye's
834
- * lid, stretched: the lash band and the fold drawn down over the eyeball in
835
- * pale streaks, and a white edge where the stretched margin met the sclera. A
836
- * reviewer watching a real call called it the eye tearing mid-blink, and at
837
- * any distance it read as an eye that did not shut at all.
838
- *
839
- * So the build carries the neutral edited to close the eyes
840
- * (`scripts/shut_lids.py`), and each lid vertex says where its texel lands
841
- * when the lid is shut. A lid fragment reads that photograph *there*: shut,
842
- * every pixel the lid covers is the closed photograph's own; half-shut, the
843
- * margin already wears the closed lash line and the band above it closed lid
844
- * skin, because that is what those texels become. The blend follows the lid's
845
- * own influence, so a lid held part-way on purpose — a downward glance, a
846
- * degraded link — changes a little of its content, and a blink all of it.
847
- *
848
- * How much each vertex may show is the build's to say (`_lid_shut_w`,
849
- * `morphs.lid_shut_weight`): all of the upper lid, fading out above the crease
850
- * with the lid's own pull, so the rest of the face is the photograph to the
851
- * byte at any lid value, and not at rest at all.
852
- */
853
- // The lid's influence over which the closed photograph arrives. Not from 0, so
854
- // the lid a glance lowers keeps nearly all of its own texture; full before the
855
- // lid is, so the frames a blink is seen on are the closed photograph's.
856
- const SHUT_FROM = 0.1;
857
- const SHUT_FULL = 0.55;
858
- // ...and over which the lower lid's lash fringe does (`_lid_shut_w` below 0).
859
- // The lower lid hardly moves, so its fringe can only arrive with the upper
860
- // margin: any earlier and a half-open eye wears it as a heavy lower liner.
861
- const SHUT_FRINGE_FROM = 0.7;
862
- const SHUT_FRINGE_FULL = 0.95;
863
- const SHUT_VERTEX_HEAD = [
864
- ...LID_SHUT_ATTRS.map((name) => `attribute float ${name};`),
865
- "uniform vec2 uShutScale;", "varying vec2 vLidShut;", "varying float vLidShare;",
866
- ].join("\n");
867
- const SHUT_VERTEX = [
868
- `vLidShut = vec2(${LID_SHUT_ATTRS[0]}, ${LID_SHUT_ATTRS[1]}) * uShutScale;`,
869
- `vLidShare = ${LID_SHUT_ATTRS[2]};`,
870
- ].join("\n");
871
- const SHUT_UNIFORMS = [
872
- "uniform sampler2D uShut;", "uniform vec4 uShutTile;", "uniform vec2 uShutU;",
873
- "uniform vec2 uShutLid;", "uniform vec2 uShutFringe;", "varying vec2 vLidShut;", "varying float vLidShare;",
874
- ].join("\n");
875
- const SHUT_FRAGMENT = [
876
- "vec2 shutAt = clamp(uShutTile.xz + uShutTile.yw * (vMapUv + vLidShut), 0.0, 1.0);",
877
- "vec3 shutColour = texture2D(uShut, shutAt).rgb;",
878
- `float shutSide = smoothstep(-${EXPRESSION_SPLIT}, ${EXPRESSION_SPLIT}, uShutU.x + uShutU.y * vMapUv.x);`,
879
- "float shutWeight = max(vLidShare, 0.0) * mix(uShutLid.x, uShutLid.y, shutSide)",
880
- " + max(-vLidShare, 0.0) * mix(uShutFringe.x, uShutFringe.y, shutSide);",
881
- "diffuseColor.rgb = mix(diffuseColor.rgb, diffuse * shutColour, shutWeight);",
882
- ].join("\n");
883
-
884
- interface Shut {
885
- readonly map: THREE.Texture;
886
- readonly lid: THREE.Vector2;
887
- readonly fringe: THREE.Vector2;
888
- readonly uniforms: Record<string, THREE.IUniform>;
889
- }
890
-
891
- /** The closed photograph from its carrier's node, or `null` for an asset
892
- * without one, whose lids shut on their own texels as they always have. */
893
- function shutLids(carrier: THREE.Mesh): Shut | null {
894
- const { shut_tile: tile, shut_u: u, shut_scale: scale } = carrier.userData;
895
- const map = (carrier.material as THREE.MeshStandardMaterial).map;
896
- if (!map || !Array.isArray(tile) || !Array.isArray(u) || !Array.isArray(scale)) return null;
897
- const lid = new THREE.Vector2();
898
- const fringe = new THREE.Vector2();
899
- return {
900
- map, lid, fringe,
901
- uniforms: {
902
- uShut: { value: map },
903
- uShutTile: { value: new THREE.Vector4(tile[0], tile[1], tile[2], tile[3]) },
904
- uShutU: { value: new THREE.Vector2(u[0], u[1]) },
905
- uShutScale: { value: new THREE.Vector2(scale[0], scale[1]) },
906
- uShutLid: { value: lid },
907
- uShutFringe: { value: fringe },
908
- },
909
- };
910
- }
911
-
912
- /**
913
- * The neck, taught to wear the jaw's shadow where the jaw is.
914
- *
915
- * The photograph paints the shadow the chin casts on the throat, and a painted
916
- * shadow stays where it was painted: under a 9° turn the jaw crossed the top of
917
- * the neck by 8 px and its shadow did not, which read as the neck sliding out
918
- * from under the head. So the build lifts it into a ratio tile
919
- * (`project_albedo.lift_jaw_shadow`) and each neck fragment reads it at the
920
- * *turned* head's rim, offset by how far the jaw has dropped — a rigid
921
- * transform does not carry a morph, which was the other half of the same
922
- * defect: an open mouth left the shadow banded across the throat at the closed
923
- * rim with nothing casting it.
924
- *
925
- * The tile's edges are white and the lookup clamped to it, so a ray landing
926
- * past it reads "no shadow" rather than the hair or the iris beside it.
927
- */
928
- function jawShadow(base: THREE.MeshStandardMaterial, uv: number[], extent: number[],
929
- rimZ: number, headInverse: { value: THREE.Matrix4 },
930
- jawDrop: { value: number }) {
931
- const material = base.clone();
932
- const tile = { value: new THREE.Vector4(uv[0], uv[1], uv[2], uv[3]) };
933
- const bounds = { value: new THREE.Vector4(extent[0], extent[1], extent[2], extent[3]) };
934
- const rim = { value: rimZ };
935
- material.onBeforeCompile = (shader) => {
936
- shader.uniforms.uHeadInverse = headInverse;
937
- shader.uniforms.uJawTile = tile;
938
- shader.uniforms.uJawBounds = bounds;
939
- shader.uniforms.uJawRim = rim;
940
- shader.uniforms.uJawDrop = jawDrop;
941
- shader.vertexShader = shader.vertexShader
942
- .replace("#include <common>", "#include <common>\nvarying vec3 vJawView;")
943
- .replace("#include <project_vertex>", "#include <project_vertex>\nvJawView = mvPosition.xyz;");
944
- shader.fragmentShader = shader.fragmentShader
945
- .replace("#include <common>",
946
- "#include <common>\nvarying vec3 vJawView;\nuniform mat4 uHeadInverse;\n"
947
- + "uniform vec4 uJawTile;\nuniform vec4 uJawBounds;\nuniform float uJawRim;\n"
948
- + "uniform float uJawDrop;")
949
- .replace("#include <map_fragment>",
950
- "#include <map_fragment>\n"
951
- + "vec3 jawFrom = (uHeadInverse * vec4(vJawView, 1.0)).xyz;\n"
952
- + "vec3 jawRay = (uHeadInverse * vec4(0.0, 0.0, 1.0, 0.0)).xyz;\n"
953
- + "vec2 jawAt = jawFrom.xy + jawRay.xy * ((uJawRim - jawFrom.z) / jawRay.z);\n"
954
- + "jawAt.y += uJawDrop;\n"
955
- + "jawAt = clamp(jawAt, uJawBounds.xz, uJawBounds.yw);\n"
956
- + "vec3 jawShadow = texture2D(map, uJawTile.xz + uJawTile.yw * jawAt).rgb;\n"
957
- + "diffuseColor.rgb *= jawShadow;")
958
- .replace("#include <emissivemap_fragment>",
959
- "#include <emissivemap_fragment>\ntotalEmissiveRadiance *= jawShadow;");
960
- };
961
- material.customProgramCacheKey = () => "tara-jaw-shadow";
962
- return material;
963
- }
964
-
965
- /** Lamps, standing in for the four area lights `build_character.setup_scene` uses.
966
- *
967
- * The albedo is a photograph and already holds this face's light, so 75% of it
968
- * is emitted verbatim (`emissiveFactor` in the GLB) and only the remaining 25%
969
- * is handed to these. They exist for the quarter that makes a yaw read as a
970
- * head turning rather than a picture sliding; their sum along the face normal
971
- * is π, which is exactly the irradiance that renders that quarter back at full
972
- * albedo — so the rest frame is the reference photograph, and only a turned
973
- * head departs from it.
974
- */
975
- const AMBIENT = 0.62 * Math.PI;
976
- const KEY = 0.26 * Math.PI;
977
- const WRAP = 0.10 * Math.PI;
978
-
979
- /** Frames drawn per second, capped rather than left at the display's rate.
980
- *
981
- * Measured on an M1 over four paired reps against `peep`, which is the SVG
982
- * avatar that already ships: uncapped at 60 the 3-D face cost 13.8 points of
983
- * one core more than peep; capped at 30 the difference was inside the noise
984
- * (−0.9 points over three reps). The cap is most of the runtime cost of being
985
- * 3-D at all.
986
- *
987
- * It throttles *drawing* only. The mixer runs its own rAF loop and keeps its
988
- * own clock, so cue timing is exactly as accurate as it was — what drops is how
989
- * often that clock is looked at, not how well it is kept.
990
- */
991
- const RENDER_FPS = 30;
992
- const MIN_FRAME_MS = 1000 / RENDER_FPS;
993
-
994
- export interface CharacterRigOptions {
995
- /** Called once the GLB is in the scene, for a capture tool that must wait. */
996
- readonly onReady?: () => void;
997
- /** The character's GLB. Every build fact the rig reads (jaw-shadow tile, rim
998
- * depth, morph names) travels in the GLB's own extras, so a character built
999
- * by the same scripts needs nothing else.
1000
- *
1001
- * Required, and deliberately not defaulted: a default would make this module
1002
- * import one character's URL, and a bundler emits assets per module — so
1003
- * every consumer of *any* character would ship that one's GLB whether or not
1004
- * they mounted it. The caller knows which character it is building; this
1005
- * file must not ([tara-asset.ts](./tara-asset.ts)). */
1006
- readonly url: string;
1007
- /** `false` leaves an asset's expression maps unread, for a capture tool
1008
- * comparing the face with and without them. */
1009
- readonly expression?: boolean;
1010
- /** Keeps the drawing buffer readable after the browser has composited it, so
1011
- * a caller can copy the canvas out at any moment rather than only from
1012
- * inside the frame that drew it.
1013
- *
1014
- * A pose sheet reuses one context, holding a pose still and copying the
1015
- * canvas into a tile; copy outside the drawing frame without this and the
1016
- * buffer has already been cleared, which is a blank tile and not an error.
1017
- * A consumer must not have it: the buffer can no longer be discarded, which
1018
- * on some drivers means a second copy of every frame. */
1019
- readonly readback?: boolean;
1020
- }
1021
-
1022
- const radians = (deg: number) => (deg * Math.PI) / 180;
1023
-
1024
- /** The renderer, or `null` where the browser will not give a context.
1025
- *
1026
- * Three.js throws out of the constructor rather than returning anything, and it
1027
- * has already written its own line to the console by then; that line is kept
1028
- * because it names the underlying reason, which this one does not. */
1029
- function webglRenderer(readback = false): THREE.WebGLRenderer | null {
1030
- try {
1031
- return new THREE.WebGLRenderer({ antialias: true, alpha: true, preserveDrawingBuffer: readback });
1032
- } catch {
1033
- return null;
1034
- }
1035
- }
1036
-
1037
- /**
1038
- * The lower lid follows the eye down. Its retractor is tied to the inferior
1039
- * rectus, so a look down pulls the lower margin down with it by a millimetre
1040
- * or two — the upper lid's half of this is the mixer's `lidBias`, and a face
1041
- * whose upper lid follows while the lower one stays reads as a drowsy droop
1042
- * rather than a glance. Driven through the squint target run backwards,
1043
- * which is the lower lid and nothing else: at the notes gaze (`pupilY` 0.72)
1044
- * the margin drops ~0.006, about 1.7 px at the 400 px tile.
1045
- */
1046
- const LOWER_LID_FOLLOW = 0.35;
1047
-
1048
- /**
1049
- * Where a lid channel's influence reaches a shut eye. Not at 1, because the
1050
- * mixer never asks for 1: a blink is a triangle (`idle.BLINK_DUR`), the lid
1051
- * channel's 18 ms smoothing rounds its peak off, and this rig draws at 30 fps.
1052
- * `LID_SHUT` was set when blinks ran 0.11–0.15 s, at the fifth percentile of
1053
- * the peak a viewer actually saw (median 0.74). At the present 0.19–0.23 s
1054
- * that peak is 0.85 median and 0.81 at the fifth percentile — simulated over
1055
- * the same smoothing and frame phase — so every blink still lands shut, with
1056
- * margin. A lid morph that shut only at 1 left every blink a quarter open — the
1057
- * lid came down and the iris was still there.
1058
- *
1059
- * So above `LID_KNEE` the influence is eased up to meet 1 at `LID_SHUT`, and
1060
- * held there: past that point the lid has landed on the lower one. Below the
1061
- * knee it is untouched, and that is every lid held part-way on purpose — the
1062
- * lowered lid of a degraded link, the lid following a downward gaze — so those
1063
- * look exactly as they did. The ease is quadratic from the knee, so both the
1064
- * value and its slope are continuous there.
1065
- *
1066
- * `scripts/morphs.py:influence` parses both numbers out of this file.
1067
- */
1068
- const LID_KNEE = 0.35;
1069
- const LID_SHUT = 0.66;
1070
-
1071
- /**
1072
- * A squint's influence rises faster than its channel. The mixer's values are
1073
- * set where a line face's lower lid reads — a smile's squint is 0.30 — and a
1074
- * photographic lower lid rising 0.30 of its travel is a pixel or two, so the
1075
- * squint that makes a smile real was not there. The lower-lid follow, which
1076
- * runs this target backwards, is added after the curve and stays linear.
1077
- *
1078
- * The curve needs a ceiling as well as a lift, because the squint stacks and
1079
- * the mouth does not. A *silent* smile takes squint from three layers at once —
1080
- * an approval clip, the encouraging emotion and prosody's warmth — which reach
1081
- * 0.51 together, while the smile map saturates at a mouth corner of 0.8 and no
1082
- * layer drives the jaw, so the last third of a smile arrives as narrowing eyes
1083
- * over lips that cannot part any further. On a line face that reads as warmth.
1084
- * On a photograph it reads as sedation: a reviewer watching a recorded call
1085
- * read those two moments as the avatar falling asleep or heavily medicated, and
1086
- * named the eyes, not the mouth. The ceiling is where the face stops reading
1087
- * drugged, judged at crop. The knee is low enough that an ordinary one-layer
1088
- * smile is untouched (0.22 renders 0.402 against 0.403 with no ceiling), and
1089
- * the approach is exponential rather than a clamp so the slope is continuous
1090
- * where the two meet and the lower lid never visibly sticks.
1091
- */
1092
- const SQUINT_CURVE = 0.6;
1093
- const SQUINT_KNEE = 0.2;
1094
- const SQUINT_CEIL = 0.54;
1095
-
1096
- const squintCurve = (i: number): number => {
1097
- if (i <= 0) return i;
1098
- if (i <= SQUINT_KNEE) return i ** SQUINT_CURVE;
1099
- const knee = SQUINT_KNEE ** SQUINT_CURVE;
1100
- const slope = SQUINT_CURVE * SQUINT_KNEE ** (SQUINT_CURVE - 1);
1101
- const head = SQUINT_CEIL - knee;
1102
- return SQUINT_CEIL - head * Math.exp((-slope * (i - SQUINT_KNEE)) / head);
1103
- };
1104
-
1105
- const lidClosure = (i: number): number => {
1106
- if (i <= LID_KNEE) return i;
1107
- if (i >= LID_SHUT) return 1;
1108
- const t = (i - LID_KNEE) / (LID_SHUT - LID_KNEE);
1109
- return i + (1 - LID_SHUT) * t * t;
1110
- };
1111
-
1112
- /**
1113
- * The neck's follow targets, and the one place a morph is not driven by
1114
- * `influence`.
1115
- *
1116
- * `scripts/morphs.neck_targets` authors two fields per head axis — `A P` and
1117
- * `A^2 P` for that axis's skew matrix — because a rigid rotation is exactly
1118
- * `sin(th)` of the first plus `1 - cos(th)` of the second. A single field scaled
1119
- * by the pose is the *linear* approximation of that, and it stretches the neck
1120
- * away from the pivot by `radius x (1 - cos th)`: fine at six degrees, five
1121
- * pixels at the envelope a nod that lands actually needs.
1122
- *
1123
- * So under pitch and roll the throat tracks the skull exactly at any angle,
1124
- * and — the part worth having — the asset stops depending on the envelope.
1125
- * These fields carry no angle, so `HEAD_DEG` above is a runtime number that can
1126
- * move without leaving a built GLB stale. Yaw's pair is a partial twist rather
1127
- * than the skull's own rotation, and the build weights its two fields so these
1128
- * same two influences drive it (`morphs.neck_twist`).
1129
- */
1130
- const NECK_QUAD = "_q";
1131
- const HEAD_AXIS: Record<string, keyof typeof HEAD_DEG> = {
1132
- headYaw: "yaw", headPitch: "pitch", headRoll: "roll",
1133
- };
1134
-
1135
- const neckInfluence = (channel: string, pose: RigPose): number | null => {
1136
- const quad = channel.endsWith(NECK_QUAD);
1137
- const axis = HEAD_AXIS[quad ? channel.slice(0, -NECK_QUAD.length) : channel];
1138
- if (!axis) return null;
1139
- const value = pose[quad ? channel.slice(0, -NECK_QUAD.length) : channel] ?? 0;
1140
- const radians = ((value / HEAD_CLAMP) * HEAD_DEG[axis] * Math.PI) / 180;
1141
- return quad ? 1 - Math.cos(radians) : Math.sin(radians);
1142
- };
1143
-
1144
- /**
1145
- * The hair's roll, which is the one thing in this rig that is not a function of
1146
- * the pose alone. `hold` is the share of the head's roll the hanging hair
1147
- * declines to take, and `hz`/`damping` are how it gets there.
1148
- *
1149
- * A hank that hangs past the jaw is lying on a shoulder, and a shoulder does
1150
- * not tilt when the head does: rolled rigidly with the skull it lifts off the
1151
- * collar and the page shows through behind it, so the shell gives up `hold` of
1152
- * the roll at its lowest rows and none at the crown (`morphs.hair_hold`).
1153
- *
1154
- * The other half is why roll read as a hinge at all. A rigid rotation about a
1155
- * fixed point *is* a hinge, and what a real tilt has that this lacked is hair
1156
- * that arrives late and settles (`docs/research-head-rotation.md` § 3.1:
1157
- * mobility ~0.95, delay 0.8-0.9, one clear overshoot). At 0.65 that is a single
1158
- * visible overshoot, inside 5% of the hold in 450 ms — faster reads as a flick,
1159
- * slower as wet hair.
1160
- *
1161
- * It is on the hair and not on `headRoll` on purpose: every clip in the library
1162
- * is authored pre-compensated for the mixer's per-channel time constants, so a
1163
- * spring on the channel would silently re-time every nod ever authored.
1164
- */
1165
- export const HAIR_ROLL = { hold: 0.85, hz: 1.5, damping: 0.65 };
1166
-
1167
- /**
1168
- * One step of `HAIR_ROLL`'s spring: semi-implicit, the rate taking the frame's
1169
- * acceleration before the angle takes the rate. Explicit Euler rings at this
1170
- * stiffness and 30 fps; this does not, which is the only reason the order of
1171
- * those two lines is worth a sentence.
1172
- *
1173
- * Exported for `test/nods.test.ts`: settle time and overshoot are numbers, not
1174
- * something a still frame can show.
1175
- */
1176
- export const hairRollStep = (angle: number, rate: number, target: number, dt: number) => {
1177
- const w = 2 * Math.PI * HAIR_ROLL.hz;
1178
- const next = rate + (w * w * (target - angle) - 2 * HAIR_ROLL.damping * w * rate) * dt;
1179
- return { angle: angle + next * dt, rate: next };
1180
- };
1181
-
1182
- /** The hair pair's two influences, from the *extra* angle the shell is turned
1183
- * by — the same `sin` / `1 - cos` terms the neck's fields are driven with. */
1184
- const hairInfluence = (channel: string, extra: number): number | null => {
1185
- if (channel === "headRoll") return Math.sin(extra);
1186
- if (channel === "headRoll" + NECK_QUAD) return 1 - Math.cos(extra);
1187
- return null;
1188
- };
1189
-
1190
- /**
1191
- * A channel's morph influence — the same law `scripts/morphs.py:influence`
1192
- * uses, so a Blender preview and the browser pose the face identically.
1193
- *
1194
- * It is allowed to go negative, which is what lets one target serve a
1195
- * bidirectional channel: `mouthCornerL` at −1.4 is the smile target run
1196
- * backwards into a frown, and `lidL` below its rest opens the eye wider than
1197
- * neutral. A rig that clamped this at 0 would silently delete the negative half
1198
- * of every channel that has one.
1199
- */
1200
- const influence = (channel: string, value: number): number => {
1201
- const rest = (REST as Record<string, number>)[channel] ?? 0;
1202
- const i = (value - rest) / (1 - rest);
1203
- if (channel === "lidL" || channel === "lidR") return lidClosure(i);
1204
- return channel === "squintL" || channel === "squintR" ? squintCurve(i) : i;
1205
- };
1206
-
1207
- /**
1208
- * The jaw a closure lets through. This shell hangs the lower lip from the
1209
- * mandible (`morphs._mandible`), and the jaw is the slower of the two to settle
1210
- * (`JAW_RESPONSE_TAU_S`), so into an [m] the lips have shut while the jaw is
1211
- * still coming up. That pulls the lower lip back off the upper, and the teeth
1212
- * show through the seam. A real lower lip closes over a jaw that is still
1213
- * open. A press is what a seal looks like on these channels, so in proportion
1214
- * to the press past rest the jaw is held to what the lips' own aperture
1215
- * implies. The cost is a chin that arrives with the lips rather than just
1216
- * after them, and that is not what a viewer looks at during an [m].
1217
- */
1218
- const SEAL_FROM = VISEME_SHAPES.X.mouthPress ?? REST.mouthPress;
1219
- const SEAL_FULL = VISEME_SHAPES.A.mouthPress ?? 1;
1220
- function sealed(pose: RigPose): RigPose {
1221
- const { jaw, mouthOpen = 0, mouthPress } = pose;
1222
- if (jaw === undefined || mouthPress === undefined) return pose;
1223
- const seal = Math.min(1, Math.max(0, (mouthPress - SEAL_FROM) / (SEAL_FULL - SEAL_FROM)));
1224
- const excess = jaw - JAW_OF_OPEN * mouthOpen;
1225
- return seal > 0 && excess > 0 ? { ...pose, jaw: jaw - seal * excess } : pose;
1226
- }
1227
-
1228
- /** One side's weights for the asset's expression maps, in its order. A
1229
- * channel with no side (`mouthPress`) weighs the same on both. */
1230
- const expressionWeights = (pose: RigPose, side: "L" | "R", expression: Expression,
1231
- into: Float32Array) => {
1232
- const at = (channel: string) => {
1233
- const name = pose[channel + side] === undefined ? channel : channel + side;
1234
- const value = pose[name];
1235
- return value === undefined ? 0 : influence(name, value);
1236
- };
1237
- expression.names.forEach((map, i) => {
1238
- const weight = EXPRESSION_WEIGHT[map]?.(at) ?? 0;
1239
- into[i] = Math.min(Math.max(weight, 0), 1);
1240
- });
1241
- };
1242
-
1243
- export function createCharacterRig(mount: HTMLElement, options?: unknown): AvatarRig {
1244
- const { onReady, url, expression: readExpression = true, readback = false } =
1245
- (options ?? {}) as CharacterRigOptions;
1246
- // A missing `url` is a caller's defect, not a browser condition — the WebGL
1247
- // path below degrades because a driver is nobody's fault, whereas this would
1248
- // otherwise be a 404 on a path spelled `undefined`.
1249
- if (!url) throw new TypeError("createCharacterRig: `url` is required");
1250
- const scene = new THREE.Scene();
1251
- const camera = new THREE.OrthographicCamera(-1, 1, 1, -1, 0.1, 100);
1252
- camera.position.set(0, FRAME_CENTRE, 6);
1253
- camera.lookAt(0, FRAME_CENTRE, 0);
1254
-
1255
- const renderer = webglRenderer(readback);
1256
- // `createAvatar` is synchronous, so anything thrown here lands in the
1257
- // consumer's window and takes the call page with it over a browser condition
1258
- // that is nobody's defect. The right outcome is a call that still has audio,
1259
- // captions and states, with an empty tile where the head would be.
1260
- //
1261
- // `warn` rather than `error` on purpose: `[avatar]` console errors mean a
1262
- // defect in this package, and the capture tools fail a run on any of them.
1263
- // This one says the environment cannot draw, which is a different sentence.
1264
- if (!renderer) {
1265
- console.warn("[avatar] no WebGL context; this character will not render in this browser");
1266
- return { apply() { /* nothing to pose */ }, destroy() { /* nothing to release */ } };
1267
- }
1268
- renderer.outputColorSpace = THREE.SRGBColorSpace;
1269
- // `setSize(…, false)` below leaves CSS alone, so the canvas has to be told to
1270
- // fill the mount. Without this it displays at its backing-store size, which on
1271
- // a 2x screen is a head twice the tile, cropped to its top-left quarter —
1272
- // invisible at devicePixelRatio 1, which is what every capture tool ran at.
1273
- renderer.domElement.style.cssText = "display:block;width:100%;height:100%";
1274
- mount.append(renderer.domElement);
1275
-
1276
- scene.add(new THREE.AmbientLight(0xffffff, AMBIENT));
1277
- const key = new THREE.DirectionalLight(0xffffff, KEY);
1278
- key.position.set(0, 1.0, 3.0);
1279
- scene.add(key);
1280
- for (const x of [-1.9, 1.9]) {
1281
- const wrap = new THREE.DirectionalLight(0xffffff, WRAP);
1282
- wrap.position.set(x, 1.0, 2.5);
1283
- scene.add(wrap);
1284
- }
1285
-
1286
- // The head swings as a group; the neck and torso do not (see HEAD_PARTS).
1287
- // Parenting to a pivot object rather than rotating each part is not a
1288
- // convenience — the parts have to rotate about a shared point, and a per-mesh
1289
- // rotation about each mesh's own origin would slide them apart.
1290
- //
1291
- // The body's two transforms nest outside it, one group each (see BODY):
1292
- // `trunk` sways everything, and `lift` carries the neck and head on the
1293
- // breath the torso takes inside `trunk` — and on the lean's rigid share,
1294
- // which reaches the head the same way for the same reason.
1295
- const trunk = new THREE.Group();
1296
- const lift = new THREE.Group();
1297
- const head = new THREE.Group();
1298
- // The asset's own, once it has loaded; these hold the shape of the hierarchy
1299
- // until then, and nothing draws before that.
1300
- let pivot = PIVOT.clone();
1301
- let rollPivot = ROLL_PIVOT.clone();
1302
- head.position.copy(pivot);
1303
- // Yaw and pitch turn `head`; roll turns `tilt`, which rides inside them at
1304
- // the chin (`rollPivot`), so the parts hang from `tilt`.
1305
- const tilt = new THREE.Group();
1306
- tilt.position.subVectors(rollPivot, pivot);
1307
- scene.add(trunk);
1308
- trunk.add(lift);
1309
- lift.add(head);
1310
- head.add(tilt);
1311
- let torso: THREE.Object3D | null = null;
1312
-
1313
- let destroyed = false;
1314
- let pending: RigPose | null = null;
1315
- let loaded = false;
1316
- let warnedAboutRuntimeBudget = false;
1317
- // Every mesh that owns morph targets, paired with the dictionary three.js
1318
- // built from the GLB's `extras.targetNames` — i.e. the pose channel names
1319
- // `scripts/morphs.py` authored them under.
1320
- const morphed: THREE.Mesh[] = [];
1321
- const eyes: THREE.Object3D[] = [];
1322
- // Both globes turn together, so one rotation serves both sockets.
1323
- const gaze = { value: new THREE.Matrix3() };
1324
- const lidShade = { L: { value: 1 }, R: { value: 1 } };
1325
- const turn = new THREE.Matrix4();
1326
- let expression: Expression | null = null;
1327
- let shut: Shut | null = null;
1328
- // The `Hair` shell, on a character whose hair hangs low enough to have the
1329
- // roll pair, and the `HairLayer` over the body, on one whose hair is a layer
1330
- // of its own; empty where the hair stops beside the temple (`HAIR_ROLL`).
1331
- const hairMeshes: THREE.Mesh[] = [];
1332
- // The head's roll in the asset's own frame — what both the neck's follow and
1333
- // the hair's are authored about — and the hair's own, which chases it.
1334
- let headRollRad = 0;
1335
- let hairRollRad = 0;
1336
- let hairRate = 0;
1337
- let hairSeeded = false;
1338
- // How far the jaw's rim travels at jaw = 1 on this asset, and the uniform the
1339
- // neck's shadow reads it through at the pose's influence (`jawShadow`).
1340
- let jawRimTravel = 0;
1341
- const jawDrop = { value: 0 };
1342
- // The lip opening at rest and each channel's move of it, off the shell
1343
- // (`build_character.lip_aperture`), as [top, bottom, left, right]; and what
1344
- // `mouthInterior` reads it through, as (centre, top, half-width, height).
1345
- let aperture: { rest: number[]; deltas: [string, number[]][] } | null = null;
1346
- const mouthOpening = { value: new THREE.Vector4(0, 0, 1, 1) };
1347
- const tongueLift = { value: 0 };
1348
-
1349
- /** The share of the head's roll the hair settles at. */
1350
- const hairTarget = () => headRollRad * (1 - HAIR_ROLL.hold);
1351
-
1352
- const writeHair = () => {
1353
- const extra = hairRollRad - headRollRad;
1354
- for (const mesh of hairMeshes) {
1355
- const dictionary = mesh.morphTargetDictionary;
1356
- const influences = mesh.morphTargetInfluences;
1357
- if (!dictionary || !influences) continue;
1358
- for (const [channel, index] of Object.entries(dictionary)) {
1359
- const term = hairInfluence(channel, extra);
1360
- if (term !== null) influences[index] = term;
1361
- }
1362
- }
1363
- };
1364
-
1365
- const stepHair = (dt: number) => {
1366
- if (!hairMeshes.length || !hairSeeded) return;
1367
- ({ angle: hairRollRad, rate: hairRate } =
1368
- hairRollStep(hairRollRad, hairRate, hairTarget(), dt));
1369
- };
1370
-
1371
- const resize = () => {
1372
- const width = Math.max(1, mount.clientWidth);
1373
- const height = Math.max(1, mount.clientHeight || Math.round((width * 3) / 4));
1374
- renderer.setPixelRatio(pixelRatioFor(width, height, window.devicePixelRatio));
1375
- renderer.setSize(width, height, false);
1376
- // The framed *height* is fixed and the width follows the mount, so a tile
1377
- // of the wrong aspect shows more or less background rather than a face of
1378
- // the wrong shape. The atlas cannot be stretched: it is a photograph.
1379
- // Symmetric about the camera, which is *already* at the frame's centre
1380
- // height — offsetting the frustum by that centre as well applies it twice,
1381
- // and the face rendered 0.47 face heights low, at exactly the right size,
1382
- // which reads as a framing choice rather than as the arithmetic error it
1383
- // was.
1384
- const halfHeight = FRAME_HEIGHT / 2;
1385
- const halfWidth = (halfHeight * width) / height;
1386
- camera.top = halfHeight;
1387
- camera.bottom = -halfHeight;
1388
- camera.left = -halfWidth;
1389
- camera.right = halfWidth;
1390
- camera.updateProjectionMatrix();
1391
- };
1392
- const observer = new ResizeObserver(resize);
1393
- observer.observe(mount);
1394
- resize();
1395
-
1396
- const applyPose = (given: RigPose) => {
1397
- const pose = sealed(given);
1398
- headRollRad = radians(((pose.headRoll ?? 0) / HEAD_CLAMP) * HEAD_DEG.roll);
1399
- // The first pose is a starting point, not a movement: seed the hair where it
1400
- // would have settled, so a tool that sets one pose and screenshots it gets
1401
- // the hold with no transient, and only a *change* of roll swings the hank.
1402
- if (!hairSeeded) { hairSeeded = true; hairRollRad = hairTarget(); hairRate = 0; }
1403
- for (const mesh of morphed) {
1404
- const dictionary = mesh.morphTargetDictionary;
1405
- const influences = mesh.morphTargetInfluences;
1406
- if (!dictionary || !influences) continue;
1407
- const neck = mesh.name === "Neck";
1408
- const hair = hairMeshes.includes(mesh);
1409
- for (const [channel, index] of Object.entries(dictionary)) {
1410
- // The hair carries the head's roll channel too, and it is neither a
1411
- // channel value nor the head's own angle: it is how much *further* than
1412
- // the skull this shell is turned, which is negative while it holds and
1413
- // swings either side of that while it settles (`HAIR_ROLL`).
1414
- if (hair) {
1415
- const term = hairInfluence(channel, hairRollRad - headRollRad);
1416
- if (term !== null) { influences[index] = term; continue; }
1417
- }
1418
- // The neck's head targets are the rotation's two terms, not a channel
1419
- // scaled by the influence law. Only on the neck and the hair layer's
1420
- // yaw hold (`morphs.hair_layer_targets`), whose field carries the hold
1421
- // and takes the skull's own angle: the same three channel names on the
1422
- // head group are a rigid transform, and nowhere else.
1423
- if (neck || (hair && mesh.name === "HairLayer")) {
1424
- const term = neckInfluence(channel, pose);
1425
- if (term !== null) { influences[index] = term; continue; }
1426
- }
1427
- const value = pose[channel];
1428
- const follow = channel.startsWith("squint") ? LOWER_LID_FOLLOW * Math.max(pose.pupilY ?? 0, 0) : 0;
1429
- if (value === undefined && !follow) continue;
1430
- influences[index] = (value === undefined ? 0 : influence(channel, value)) - follow;
1431
- }
1432
- }
1433
- // The chin's shadow on the throat rides the same channel the chin does.
1434
- jawDrop.value = jawRimTravel
1435
- * (pose.jaw === undefined ? 0 : influence("jaw", pose.jaw));
1436
- if (aperture) {
1437
- const at = aperture.rest.slice();
1438
- for (const [channel, delta] of aperture.deltas) {
1439
- const value = pose[channel];
1440
- if (value === undefined) continue;
1441
- const k = influence(channel, value);
1442
- for (let i = 0; i < 4; i++) at[i] += k * delta[i];
1443
- }
1444
- // A floor on the height so a shut mouth divides by something: the seam
1445
- // it shows is a line, and what colour a line is does not read.
1446
- mouthOpening.value.set((at[2] + at[3]) / 2, at[0], Math.max((at[3] - at[2]) / 2, 1e-3),
1447
- Math.max(at[0] - at[1], 2e-3));
1448
- }
1449
- tongueLift.value = pose.tongue === undefined ? 0 : influence("tongue", pose.tongue);
1450
- // Blender's Z is face-space v, so its yaw is about Z, its pitch about X and
1451
- // its roll about Y. The export maps Blender (x, y, z) to glTF (x, z, −y),
1452
- // so Blender +Z *is* glTF +Y and Blender +X is glTF +X: yaw and pitch carry
1453
- // across with their sign intact. Only roll changes sign, because Blender +Y
1454
- // is glTF −Z, and that is a statement about two axes and not about the
1455
- // channel.
1456
- //
1457
- // Yaw is not negated, and a negation here survived for a long time because
1458
- // `build_character.pose_head` had the same one: the Blender preview and the
1459
- // browser agreed with each other and only disagreed with the library.
1460
- //
1461
- // YXZ because that is the order a neck composes in — yaw carrying the pitch
1462
- // — rather than the order three.js defaults to.
1463
- // The body first, because the head's roll is stated against it.
1464
- //
1465
- // Positive `torsoTurn` takes the trunk to the viewer's right, which about a
1466
- // pivot below it is a clockwise roll as the camera sees it: negative about
1467
- // glTF +Z, the axis pointing at the camera.
1468
- const sway = -radians((pose.torsoTurn ?? 0) * BODY.swayDeg);
1469
- trunk.rotation.z = sway;
1470
- trunk.position.set(BODY.hip * Math.sin(sway), BODY.hip * (1 - Math.cos(sway)), 0);
1471
- // `torsoLean` itself is not applied here at all — it is two morph targets on
1472
- // the shell, driven by the generic loop above like any other channel that
1473
- // rests at 0. What is left for the rig is the head's ride: above the collar
1474
- // the field is flat, so the neck and head take a pure translation and the
1475
- // head does not deform under a body lean (research § 8 item 4). `morphs.py`
1476
- // parses `leanRide`, so this is the same number the field plateaus at.
1477
- const breath = pose.breath ?? 0;
1478
- if (torso) {
1479
- torso.scale.set(1 + breath * BODY.widen, 1 + breath * BODY.rise, 1);
1480
- torso.position.y = -BODY.swellPivot * breath * BODY.rise;
1481
- }
1482
- lift.position.y = breath * BODY.rise * (BODY.collar - BODY.swellPivot)
1483
- - (pose.torsoLean ?? 0) * BODY.leanRide;
1484
-
1485
- head.rotation.order = "YXZ";
1486
- head.rotation.set(
1487
- radians(((pose.headPitch ?? 0) / HEAD_CLAMP) * HEAD_DEG.pitch),
1488
- radians(((pose.headYaw ?? 0) / HEAD_CLAMP) * HEAD_DEG.yaw),
1489
- 0,
1490
- );
1491
- // Less the trunk's roll, so the head's world roll is `headRoll` and
1492
- // nothing else. Exact about Z alone; with the yaw and pitch it composes
1493
- // with, the error is the product of two sub-degree angles.
1494
- tilt.rotation.z = -headRollRad - sway;
1495
- for (const globe of eyes) {
1496
- globe.rotation.order = "YXZ";
1497
- globe.rotation.set(
1498
- (pose.pupilY ?? 0) * GAZE_TRAVEL.y / GLOBE_RADIUS,
1499
- (pose.pupilX ?? 0) * GAZE_TRAVEL.x / GLOBE_RADIUS,
1500
- 0,
1501
- );
1502
- }
1503
- if (eyes.length) gaze.value.setFromMatrix4(turn.makeRotationFromEuler(eyes[0].rotation));
1504
- for (const side of ["L", "R"] as const) {
1505
- const value = pose[`lid${side}`];
1506
- const i = value === undefined ? 0 : influence(`lid${side}`, value);
1507
- lidShade[side].value = 1 - LID_SHADE * THREE.MathUtils.smoothstep(i, LID_SHADE_FROM, 1);
1508
- }
1509
- if (expression) {
1510
- expressionWeights(pose, "L", expression, expression.left);
1511
- expressionWeights(pose, "R", expression, expression.right);
1512
- }
1513
- if (shut) {
1514
- const closure = (channel: "lidL" | "lidR") => {
1515
- const value = pose[channel];
1516
- return value === undefined ? 0 : influence(channel, value);
1517
- };
1518
- const [l, r] = [closure("lidL"), closure("lidR")];
1519
- const { smoothstep } = THREE.MathUtils;
1520
- shut.lid.set(smoothstep(l, SHUT_FROM, SHUT_FULL), smoothstep(r, SHUT_FROM, SHUT_FULL));
1521
- shut.fringe.set(smoothstep(l, SHUT_FRINGE_FROM, SHUT_FRINGE_FULL),
1522
- smoothstep(r, SHUT_FRINGE_FROM, SHUT_FRINGE_FULL));
1523
- }
1524
- };
1525
-
1526
- new GLTFLoader().load(url, (gltf) => {
1527
- if (destroyed) return;
1528
- // Where this head turns and tilts, and what rides its skull, are facts
1529
- // about this head — measured by the build, stamped into the scene, and read
1530
- // here before anything is reparented into the frames they define. An asset
1531
- // that predates the stamp keeps the constants above, which are the numbers
1532
- // it was built with.
1533
- const stamp = (gltf.scene.userData ?? {}) as Record<string, unknown>;
1534
- pivot = stampedVec(stamp, "head_pivot", PIVOT);
1535
- rollPivot = stampedVec(stamp, "roll_pivot", ROLL_PIVOT);
1536
- head.position.copy(pivot);
1537
- tilt.position.subVectors(rollPivot, pivot);
1538
- const headParts = Array.isArray(stamp.head_parts)
1539
- && stamp.head_parts.every((n) => typeof n === "string")
1540
- ? stamp.head_parts as string[] : HEAD_PARTS;
1541
- trunk.add(gltf.scene);
1542
- // Collect the morphed meshes *before* reparenting: every one of them is a
1543
- // head part, so a traverse of `gltf.scene` after the move finds only the
1544
- // neck and the torso and the whole face goes rigid — silently, because a
1545
- // rig with nothing to drive still renders a perfectly good rest pose.
1546
- gltf.scene.traverse((object) => {
1547
- const mesh = object as THREE.Mesh;
1548
- if (mesh.isMesh && mesh.morphTargetDictionary) morphed.push(mesh);
1549
- // Kept aside as well: its roll pair is driven by a clock and not only by a
1550
- // pose, so the render loop has to reach it between poses (`HAIR_ROLL`).
1551
- if (mesh.isMesh && (mesh.name === "Hair" || mesh.name === "HairLayer")
1552
- && mesh.morphTargetDictionary) hairMeshes.push(mesh);
1553
- // Hair that lies over the body is in front of everything the portrait
1554
- // has, and it turns with the skull where the chest does not: tested
1555
- // against depth, a nod swings the lower hank back through the chest.
1556
- if (mesh.isMesh && mesh.name === "HairLayer") {
1557
- (mesh.material as THREE.Material).depthTest = false;
1558
- mesh.renderOrder = 1;
1559
- }
1560
- });
1561
- // Out of the scene before anything draws it: it carries the maps, and is
1562
- // one triangle behind the body that nothing should pay a draw call for.
1563
- const carrier = gltf.scene.getObjectByName("Expression") as THREE.Mesh | undefined;
1564
- if (carrier) {
1565
- carrier.removeFromParent();
1566
- expression = readExpression ? expressive(carrier) : null;
1567
- if (!expression) (carrier.material as THREE.MeshStandardMaterial).map?.dispose();
1568
- carrier.geometry.dispose();
1569
- (carrier.material as THREE.Material).dispose();
1570
- }
1571
- const shutCarrier = gltf.scene.getObjectByName("Shut") as THREE.Mesh | undefined;
1572
- if (shutCarrier) {
1573
- shutCarrier.removeFromParent();
1574
- shut = shutLids(shutCarrier);
1575
- if (!shut) (shutCarrier.material as THREE.MeshStandardMaterial).map?.dispose();
1576
- shutCarrier.geometry.dispose();
1577
- (shutCarrier.material as THREE.Material).dispose();
1578
- }
1579
- for (const name of headParts) {
1580
- const part = gltf.scene.getObjectByName(name);
1581
- // Reparenting moves the object into the tilt's frame, whose origin is
1582
- // the roll pivot, so subtract that to leave the part where it was
1583
- // authored. `attach()` would do this from the world matrix, which has
1584
- // not been computed yet at load.
1585
- if (part) {
1586
- part.position.sub(rollPivot);
1587
- tilt.add(part);
1588
- }
1589
- }
1590
- // One clone per source material, shared by every shell that carries the
1591
- // field; a mesh without it keeps the material it came with.
1592
- // The shell with the lids is a clone of its own: the ears and the hair
1593
- // share its material and have no lid attributes to read.
1594
- const turned = new Map<THREE.Material, THREE.MeshStandardMaterial>();
1595
- const lidded = new Map<THREE.Material, THREE.MeshStandardMaterial>();
1596
- head.traverse((object) => {
1597
- const mesh = object as THREE.Mesh;
1598
- if (!mesh.isMesh || !mesh.geometry.getAttribute(MOTION_DEPTH)) return;
1599
- const base = mesh.material as THREE.MeshStandardMaterial;
1600
- const withLids = shut !== null && LID_SHUT_ATTRS.every((name) => mesh.geometry.getAttribute(name) !== undefined);
1601
- const cache = withLids ? lidded : turned;
1602
- const own = cache.get(base)
1603
- ?? motionDepth(base, expression ?? undefined, withLids ? shut ?? undefined : undefined);
1604
- cache.set(base, own);
1605
- mesh.material = own;
1606
- });
1607
- // The neck rides the breath with the head; the torso *is* the breath. Both
1608
- // are authored in world space with identity nodes, and `lift` sits at the
1609
- // origin until a pose arrives, so moving the neck needs no correction.
1610
- const neck = gltf.scene.getObjectByName("Neck") as THREE.Mesh | undefined;
1611
- if (neck) lift.add(neck);
1612
- // The shadow's map comes from the build (`build_character.py`, the neck); an
1613
- // asset without it keeps the shadow painted on, as every build did before.
1614
- const skull = head.getObjectByName("Head");
1615
- const { jaw_shadow_uv: jawUv, jaw_shadow_extent: jawExtent, jaw_shadow_rim_z: jawRim,
1616
- jaw_shadow_drop: jawTravel } = neck?.userData ?? {};
1617
- if (neck && skull && Array.isArray(jawUv) && Array.isArray(jawExtent) && typeof jawRim === "number") {
1618
- const headInverse = { value: new THREE.Matrix4() };
1619
- // A build that predates the travel keeps the shadow rotating but not
1620
- // opening, which is where this started and is still better than no tile.
1621
- jawRimTravel = typeof jawTravel === "number" ? jawTravel : 0;
1622
- neck.material = jawShadow(neck.material as THREE.MeshStandardMaterial, jawUv, jawExtent,
1623
- jawRim, headInverse, jawDrop);
1624
- neck.onBeforeRender = (_renderer, _scene, camera) => {
1625
- headInverse.value.multiplyMatrices(camera.matrixWorldInverse, skull.matrixWorld).invert();
1626
- };
1627
- }
1628
- torso = gltf.scene.getObjectByName("Body") ?? null;
1629
- for (const name of ["Eye_L", "Eye_R"]) {
1630
- const globe = head.getObjectByName(name);
1631
- if (!globe) continue;
1632
- // The globe's geometry is authored in world space with no node transform,
1633
- // so it has to be rotated about its own centre rather than about the
1634
- // origin. Re-centre the geometry once and put the offset on the node; the
1635
- // catchlight is already a child and rides the globe, which is what a
1636
- // reflection on a wet cornea does.
1637
- const geometry = (globe as THREE.Mesh).geometry;
1638
- geometry.computeBoundingSphere();
1639
- const centre = geometry.boundingSphere?.center.clone() ?? new THREE.Vector3();
1640
- geometry.translate(-centre.x, -centre.y, -centre.z);
1641
- globe.position.add(centre);
1642
- for (const child of globe.children) child.position.sub(centre);
1643
- const mesh = globe as THREE.Mesh;
1644
- // Whether the globe turns deep is the attribute's to say, not the name's —
1645
- // the same rule the frame's shells are found by above. An asset built
1646
- // before the globes carried the field keeps the socket shader alone.
1647
- mesh.material = socketed(mesh.material as THREE.MeshStandardMaterial,
1648
- name === "Eye_L" ? -1 : 1, gaze,
1649
- mesh.geometry.getAttribute(MOTION_DEPTH) !== undefined,
1650
- name === "Eye_L" ? lidShade.L : lidShade.R);
1651
- eyes.push(globe);
1652
- }
1653
- // The mouth's inside, painted against the opening the lips make
1654
- // (`mouthInterior`). Found by *mesh* name rather than material name:
1655
- // `flat_material` hard-codes a `tara_` prefix, so tushar's cavity material
1656
- // is called `tara_cavity` as well, and the mesh is what distinguishes it.
1657
- // An asset built before the shell carried its opening keeps the flat fill.
1658
- const opening = skull?.userData?.aperture as
1659
- { rest?: unknown; deltas?: Record<string, unknown> } | undefined;
1660
- if (opening && Array.isArray(opening.rest) && opening.rest.length === 4) {
1661
- aperture = { rest: opening.rest as number[],
1662
- deltas: Object.entries(opening.deltas ?? {})
1663
- .filter((e): e is [string, number[]] => Array.isArray(e[1]) && e[1].length === 4) };
1664
- for (const [name, lift] of [["Cavity", { value: 0 }], ["Tongue", tongueLift]] as const) {
1665
- const mesh = head.getObjectByName(name) as THREE.Mesh | undefined;
1666
- if (!mesh) continue;
1667
- const extent = mesh.userData?.interior_extent;
1668
- mesh.material = mouthInterior(mesh.material as THREE.MeshStandardMaterial, mouthOpening, lift,
1669
- Array.isArray(extent) && extent.length === 4 ? extent : null);
1670
- }
1671
- const lower = head.getObjectByName("Teeth_Lower") as THREE.Mesh | undefined;
1672
- if (lower) lower.material = lowerArch(lower.material as THREE.MeshStandardMaterial, mouthOpening);
1673
- const upper = head.getObjectByName("Teeth_Upper") as THREE.Mesh | undefined;
1674
- if (upper) upper.material = upperArch(upper.material as THREE.MeshStandardMaterial, mouthOpening);
1675
- }
1676
- // A rig with no morph targets still renders a perfectly good rest pose, so
1677
- // the failure mode of losing them is a face that simply never moves — which
1678
- // a still frame cannot show. Say so, loudly enough that the Gate B capture
1679
- // (which fails on any `[avatar]` console error) catches it.
1680
- if (!morphed.length) console.error("[avatar] this character has no morph targets; the face will not move", url);
1681
- loaded = true;
1682
- if (pending) applyPose(pending);
1683
- onReady?.();
1684
- }, undefined, (error: unknown) => console.error("[avatar] could not load this character", url, error));
1685
-
1686
- // rAF fires on the panel's own grid, so elapsed lands *near* the frame
1687
- // interval and never on it, and a bare `elapsed < MIN_FRAME_MS` rejects the
1688
- // frame it wants and waits for the next — 20 fps, not 30. The tolerance is
1689
- // under half the interval at 120, 90 and 60 Hz, so it can never admit two
1690
- // frames where one belongs.
1691
- const GRID_TOLERANCE_MS = 4;
1692
- let lastFrameMs = 0;
1693
- renderer.setAnimationLoop(() => {
1694
- const now = performance.now();
1695
- const elapsed = now - lastFrameMs;
1696
- if (elapsed < MIN_FRAME_MS - GRID_TOLERANCE_MS) return;
1697
- lastFrameMs = now;
1698
- // The hair lags the skull and settles behind it, which is a state and not a
1699
- // function of the pose, so it advances on the clock. Clamped, because a tab
1700
- // that was in the background hands back a gap and not a frame.
1701
- stepHair(Math.min(elapsed / 1000, 0.1));
1702
- writeHair();
1703
- renderer.render(scene, camera);
1704
- if (!warnedAboutRuntimeBudget &&
1705
- (renderer.info.render.calls > HARD_BUDGET.drawCalls ||
1706
- renderer.info.render.triangles > HARD_BUDGET.triangles)) {
1707
- warnedAboutRuntimeBudget = true;
1708
- console.warn("[avatar] runtime budget exceeded", {
1709
- calls: renderer.info.render.calls,
1710
- triangles: renderer.info.render.triangles,
1711
- });
1712
- }
1713
- });
1714
-
1715
- return {
1716
- apply(frame: AvatarFrame) {
1717
- // The mixer smooths every channel before it gets here (each has its own
1718
- // τ, `docs/internal-rig.md`), so the rig eases nothing itself. Poses that
1719
- // arrive before the GLB does are not queued, only remembered: the newest
1720
- // one is the only one that was ever going to be shown.
1721
- if (!loaded) { pending = frame.pose; return; }
1722
- applyPose(frame.pose);
1723
- },
1724
- destroy() {
1725
- if (destroyed) return;
1726
- destroyed = true;
1727
- observer.disconnect();
1728
- renderer.setAnimationLoop(null);
1729
- scene.traverse((object) => {
1730
- const mesh = object as THREE.Mesh;
1731
- mesh.geometry?.dispose();
1732
- for (const item of Array.isArray(mesh.material) ? mesh.material : [mesh.material]) {
1733
- const material = item as THREE.MeshStandardMaterial | undefined;
1734
- material?.map?.dispose();
1735
- material?.emissiveMap?.dispose();
1736
- material?.dispose();
1737
- }
1738
- });
1739
- // A uniform, not a material's map, so the traverse above never meets it.
1740
- expression?.map.dispose();
1741
- shut?.map.dispose();
1742
- renderer.dispose();
1743
- renderer.domElement.remove();
1744
- },
1745
- };
1746
- }