@voqalize/avatar 0.4.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +65 -154
  2. package/dist/index.d.ts +37 -26
  3. package/dist/index.js +48 -25
  4. package/dist/pin.d.ts +2 -0
  5. package/dist/pin.js +3 -0
  6. package/dist/react.d.ts +21 -6
  7. package/dist/react.js +24 -6
  8. package/package.json +8 -67
  9. package/LICENSE-APACHE-2.0 +0 -201
  10. package/LICENSE-CC-BY-4.0 +0 -396
  11. package/assets/README.md +0 -60
  12. package/assets/tanvi.glb +0 -0
  13. package/assets/tanya.glb +0 -0
  14. package/assets/tara.glb +0 -0
  15. package/assets/tess.glb +0 -0
  16. package/assets/tushar.glb +0 -0
  17. package/client/Avatar.tsx +0 -57
  18. package/client/AvatarClient.ts +0 -818
  19. package/client/createAvatar.ts +0 -177
  20. package/client/index.ts +0 -44
  21. package/client/internal.ts +0 -132
  22. package/client/playout.ts +0 -95
  23. package/client/react.ts +0 -13
  24. package/client/supports.ts +0 -27
  25. package/client/three/assets.ts +0 -26
  26. package/client/three/budgets.ts +0 -27
  27. package/client/three/character-rig.ts +0 -1713
  28. package/client/three/createCharacter.ts +0 -109
  29. package/client/three/holds.ts +0 -69
  30. package/client/three/internal.ts +0 -55
  31. package/client/three/motion-limits.json +0 -124
  32. package/client/three/sequences.ts +0 -281
  33. package/client/three/tanvi-asset.ts +0 -6
  34. package/client/three/tanvi.ts +0 -30
  35. package/client/three/tanya-asset.ts +0 -6
  36. package/client/three/tanya.ts +0 -29
  37. package/client/three/tara-asset.ts +0 -25
  38. package/client/three/tara.ts +0 -36
  39. package/client/three/tess-asset.ts +0 -6
  40. package/client/three/tess.ts +0 -30
  41. package/client/three/tushar-asset.ts +0 -6
  42. package/client/three/tushar.ts +0 -28
  43. package/client/types.ts +0 -235
  44. package/client/useAvatar.ts +0 -74
  45. package/dist/Avatar.d.ts +0 -44
  46. package/dist/Avatar.d.ts.map +0 -1
  47. package/dist/Avatar.js +0 -7
  48. package/dist/Avatar.js.map +0 -1
  49. package/dist/AvatarClient.d.ts +0 -274
  50. package/dist/AvatarClient.d.ts.map +0 -1
  51. package/dist/AvatarClient.js +0 -712
  52. package/dist/AvatarClient.js.map +0 -1
  53. package/dist/createAvatar.d.ts +0 -135
  54. package/dist/createAvatar.d.ts.map +0 -1
  55. package/dist/createAvatar.js +0 -75
  56. package/dist/createAvatar.js.map +0 -1
  57. package/dist/index.d.ts.map +0 -1
  58. package/dist/index.js.map +0 -1
  59. package/dist/internal.d.ts +0 -34
  60. package/dist/internal.d.ts.map +0 -1
  61. package/dist/internal.js +0 -56
  62. package/dist/internal.js.map +0 -1
  63. package/dist/playout.d.ts +0 -42
  64. package/dist/playout.d.ts.map +0 -1
  65. package/dist/playout.js +0 -85
  66. package/dist/playout.js.map +0 -1
  67. package/dist/react.d.ts.map +0 -1
  68. package/dist/react.js.map +0 -1
  69. package/dist/supports.d.ts +0 -19
  70. package/dist/supports.d.ts.map +0 -1
  71. package/dist/supports.js +0 -25
  72. package/dist/supports.js.map +0 -1
  73. package/dist/three/assets.d.ts +0 -10
  74. package/dist/three/assets.d.ts.map +0 -1
  75. package/dist/three/assets.js +0 -23
  76. package/dist/three/assets.js.map +0 -1
  77. package/dist/three/budgets.d.ts +0 -17
  78. package/dist/three/budgets.d.ts.map +0 -1
  79. package/dist/three/budgets.js +0 -21
  80. package/dist/three/budgets.js.map +0 -1
  81. package/dist/three/character-rig.d.ts +0 -366
  82. package/dist/three/character-rig.d.ts.map +0 -1
  83. package/dist/three/character-rig.js +0 -1611
  84. package/dist/three/character-rig.js.map +0 -1
  85. package/dist/three/createCharacter.d.ts +0 -60
  86. package/dist/three/createCharacter.d.ts.map +0 -1
  87. package/dist/three/createCharacter.js +0 -84
  88. package/dist/three/createCharacter.js.map +0 -1
  89. package/dist/three/holds.d.ts +0 -38
  90. package/dist/three/holds.d.ts.map +0 -1
  91. package/dist/three/holds.js +0 -61
  92. package/dist/three/holds.js.map +0 -1
  93. package/dist/three/internal.d.ts +0 -52
  94. package/dist/three/internal.d.ts.map +0 -1
  95. package/dist/three/internal.js +0 -49
  96. package/dist/three/internal.js.map +0 -1
  97. package/dist/three/motion-limits.json +0 -124
  98. package/dist/three/sequences.d.ts +0 -71
  99. package/dist/three/sequences.d.ts.map +0 -1
  100. package/dist/three/sequences.js +0 -262
  101. package/dist/three/sequences.js.map +0 -1
  102. package/dist/three/tanvi-asset.d.ts +0 -7
  103. package/dist/three/tanvi-asset.d.ts.map +0 -1
  104. package/dist/three/tanvi-asset.js +0 -7
  105. package/dist/three/tanvi-asset.js.map +0 -1
  106. package/dist/three/tanvi.d.ts +0 -24
  107. package/dist/three/tanvi.d.ts.map +0 -1
  108. package/dist/three/tanvi.js +0 -24
  109. package/dist/three/tanvi.js.map +0 -1
  110. package/dist/three/tanya-asset.d.ts +0 -7
  111. package/dist/three/tanya-asset.d.ts.map +0 -1
  112. package/dist/three/tanya-asset.js +0 -7
  113. package/dist/three/tanya-asset.js.map +0 -1
  114. package/dist/three/tanya.d.ts +0 -23
  115. package/dist/three/tanya.d.ts.map +0 -1
  116. package/dist/three/tanya.js +0 -23
  117. package/dist/three/tanya.js.map +0 -1
  118. package/dist/three/tara-asset.d.ts +0 -26
  119. package/dist/three/tara-asset.d.ts.map +0 -1
  120. package/dist/three/tara-asset.js +0 -26
  121. package/dist/three/tara-asset.js.map +0 -1
  122. package/dist/three/tara.d.ts +0 -30
  123. package/dist/three/tara.d.ts.map +0 -1
  124. package/dist/three/tara.js +0 -30
  125. package/dist/three/tara.js.map +0 -1
  126. package/dist/three/tess-asset.d.ts +0 -7
  127. package/dist/three/tess-asset.d.ts.map +0 -1
  128. package/dist/three/tess-asset.js +0 -7
  129. package/dist/three/tess-asset.js.map +0 -1
  130. package/dist/three/tess.d.ts +0 -24
  131. package/dist/three/tess.d.ts.map +0 -1
  132. package/dist/three/tess.js +0 -24
  133. package/dist/three/tess.js.map +0 -1
  134. package/dist/three/tushar-asset.d.ts +0 -7
  135. package/dist/three/tushar-asset.d.ts.map +0 -1
  136. package/dist/three/tushar-asset.js +0 -7
  137. package/dist/three/tushar-asset.js.map +0 -1
  138. package/dist/three/tushar.d.ts +0 -22
  139. package/dist/three/tushar.d.ts.map +0 -1
  140. package/dist/three/tushar.js +0 -22
  141. package/dist/three/tushar.js.map +0 -1
  142. package/dist/types.d.ts +0 -149
  143. package/dist/types.d.ts.map +0 -1
  144. package/dist/types.js +0 -140
  145. package/dist/types.js.map +0 -1
  146. package/dist/useAvatar.d.ts +0 -39
  147. package/dist/useAvatar.d.ts.map +0 -1
  148. package/dist/useAvatar.js +0 -35
  149. package/dist/useAvatar.js.map +0 -1
  150. package/src/avatar.d.ts +0 -436
  151. package/src/avatar.js +0 -1370
  152. package/src/behavior.d.ts +0 -42
  153. package/src/behavior.js +0 -114
  154. package/src/camera.js +0 -29
  155. package/src/clips.js +0 -155
  156. package/src/conformance.js +0 -119
  157. package/src/emotions.js +0 -58
  158. package/src/face-core.js +0 -180
  159. package/src/face-myna.d.ts +0 -7
  160. package/src/face-myna.js +0 -786
  161. package/src/face-peep-control-plane.js +0 -167
  162. package/src/face-peep.d.ts +0 -7
  163. package/src/face-peep.js +0 -925
  164. package/src/face-wren.d.ts +0 -7
  165. package/src/face-wren.js +0 -468
  166. package/src/faces.d.ts +0 -12
  167. package/src/faces.js +0 -53
  168. package/src/gaze.js +0 -504
  169. package/src/hand.js +0 -709
  170. package/src/head.js +0 -134
  171. package/src/idle.js +0 -668
  172. package/src/interjections.js +0 -747
  173. package/src/line-art.js +0 -137
  174. package/src/params.js +0 -180
  175. package/src/perform.js +0 -107
  176. package/src/prosody.js +0 -741
  177. package/src/rig.d.ts +0 -31
  178. package/src/rig.js +0 -32
  179. package/src/speech-timing.js +0 -32
  180. package/src/visemes.js +0 -410
package/src/avatar.js DELETED
@@ -1,1370 +0,0 @@
1
- /**
2
- * The mixer — a programmable talking head.
3
- *
4
- * Not the public surface: that is `createAvatar({ mount, client })` in
5
- * `packages/avatar/client/createAvatar.ts`, and this is what it drives. Everything below is
6
- * addressed by our own runtime, our tooling, and an avatar author who chose to
7
- * build on the SVG renderer (`@voqalize/avatar/internal`, no semver promise).
8
- *
9
- * Per frame the mixer runs a fixed layer order. Earlier layers are overwritten
10
- * by later ones on the channels they touch; the gesture and idle layers are
11
- * additive so they compose rather than fight.
12
- *
13
- * base pose (state + emotion) -> gaze -> visemes -> clip -> idle
14
- */
15
-
16
- import { REST, CHANNELS, TAU, RANGE, GROUPS, clamp, approach } from './params.js';
17
- import { EMOTIONS, emotionPose } from './emotions.js';
18
- import { GazeLayer, GAZE_TARGETS, AVERSION } from './gaze.js';
19
- import { IdleLayer, ListeningEngine } from './idle.js';
20
- import { ClipPlayer } from './clips.js';
21
- import { ACTIONS, INTERNAL_CLIPS } from './interjections.js';
22
- import { VisemeTrack, shapeFor, SILENT } from './visemes.js';
23
- import { SpeechProsody, UNCALIBRATED_HEAD_GAIN } from './prosody.js';
24
- import { HEAD_AXES, soften } from './head.js';
25
- import { PerformTrack } from './perform.js';
26
- import { createHand, HAND_GESTURES, HAND_ACTION_TO_FRAME_GESTURE } from './hand.js';
27
- import { avatarFrame, createSvgRig } from './rig.js';
28
-
29
- const rand = ([a, b]) => a + Math.random() * (b - a);
30
- const pick = (xs) => xs[(Math.random() * xs.length) | 0];
31
-
32
- // Each state's `idle` is a profile for the liveness layer (see DEFAULT_PROFILE
33
- // in idle.js). Blink gaps come from docs/research-biomechanics.md §5: the rate
34
- // alone separates listening (~16/min) from thinking (~25/min) from visually
35
- // busy (~9/min), and it is the cheapest state signal the rig has. The gap is
36
- // the budget for every blink, paired ones included (about one in six), so it
37
- // sits a little longer than 60/rate.
38
- export const STATES = {
39
- // Idle means present and relaxed, not attending. It is also the face before
40
- // a call connects — the first one anyone sees — so it rests on the user and
41
- // leaves for soft, unhurried looks (AVERSION.IDLE) rather than parking the
42
- // eyes somewhere. It used to open on AWAY_THINKING and wander between it,
43
- // AWAY_RIGHT and NOTES from the first frame: a face that loads looking up
44
- // and to the left, at nothing, reads as broken rather than idle. What keeps
45
- // it apart from LISTENING is the longer, lazier looks, the slower scan and
46
- // the missing engagement lean, not refusing to look at the user.
47
- IDLE: { gaze: 'USER', emotion: 'neutral', engagement: false,
48
- aversion: 'IDLE', scan: [0.8, 2.0],
49
- idle: { sway: 0.72, blinkGap: [3.6, 5.2] } },
50
- // `aversion` is why this state does not stare. Continuous eye contact is not
51
- // the attentive pose it looks like — it is a demand for more talk (Rossano)
52
- // and it measures as *tense*, not attentive (Wang & Gratch). See AVERSION in
53
- // gaze.js for the numbers; the mixer holds it off near a turn boundary.
54
- // At 3.2-4.4 s the blink timer measured 20-23/min.
55
- LISTENING: { gaze: 'USER', emotion: 'neutral', engagement: true,
56
- aversion: 'LISTEN',
57
- idle: { sway: 1.0, blinkGap: [3.6, 4.8] },
58
- pose: { browRaiseL: 0.06, browRaiseR: 0.06, lidL: -0.04, lidR: -0.04 } },
59
- // Faster, shallower breath is the measured cognitive-load signature, and the
60
- // occasional dead-still hold is the strongest "working on it" cue a rig this
61
- // simple can make — deliberate stillness, not more motion. The look away is
62
- // Andrist's cognitive aversion (§4.2): one look held ~3.5 s (SD 1.26) that
63
- // drifts a little where it sits, a check-in with the user, then the next.
64
- //
65
- // In a call this state mostly lasts under two seconds — the gap between the
66
- // user finishing and the reply starting — so its opening is what gets seen,
67
- // and the opening used to be wrong. The eyes left the instant the user
68
- // stopped, down and to the left, every turn: a listener who drops their
69
- // eyes the moment you finish reads as ashamed or done with you. Now the
70
- // eyes hold the user for `opening` first (a person takes in the end of
71
- // what was said before they go to think), and a fast reply never looks
72
- // away at all. The look is mostly up-and-aside or level-aside: that is the
73
- // one audiences read as thinking, whatever the measured split, and down
74
- // is kept to a fifth of looks because on a face this real it reads as
75
- // downcast. The pose takes back `thoughtful`'s lid drop: a thinking face is
76
- // alert, and the two together measured past the 0.15 that reads drowsy.
77
- THINKING: { gaze: 'AWAY_SIDE', emotion: 'thoughtful', engagement: false,
78
- // Fixational jumps rare and small: a thinker's eyes rest
79
- // where they land. At the default gap the look jittered
80
- // about its spot every second, and with a drift on top
81
- // the eyes never held anything — searching the wall, not
82
- // thinking.
83
- scan: [1.2, 2.6, 0.35],
84
- idle: { sway: 0.7, blinkGap: [2.6, 3.2], slowBlink: 0.3,
85
- breathRate: 1.18, breathAmp: 0.7,
86
- hold: { every: [4.5, 9.0], dur: [0.8, 1.5] } },
87
- // `stick`: people have a side they look to when they
88
- // think and keep to it (Day; Kinsbourne's lateral eye
89
- // movements) — one who alternates sides on every look is
90
- // scanning the room. `dart`: now and then, once in a
91
- // look, the eyes move on to a second spot with a flick of
92
- // the brows, which is the thought moving; the random
93
- // drift this replaces never stopped and read as roving.
94
- // `blinkTo`: the look back to the user always blinks —
95
- // the re-engagement is the beat to see.
96
- glance: { to: 'USER', opening: [0.35, 0.7], every: [2.2, 4.4], hold: [0.8, 1.3],
97
- back: ['AWAY_THINKING', 'AWAY_THINKING', 'AWAY_THINKING',
98
- 'AWAY_SIDE', 'AWAY_SIDE', 'AWAY_SIDE',
99
- 'AWAY_RIGHT', 'AWAY_RIGHT', 'AWAY_DOWN', 'AWAY_DOWN'],
100
- stick: 0.55, dart: { p: 0.4, mag: 0.08, brow: 0.12 }, blinkTo: true },
101
- // **The chin comes up**: thinking is the one stretch of
102
- // a call where the avatar owes the user feedback and
103
- // has no mouth to give it with, the eyes being off the
104
- // user by design. A head tipping back as the gaze
105
- // leaves is the swing-up of research-biomechanics.md
106
- // § 3.3. It rides the looks that go up, as the head's
107
- // *share* of them: held as a pose it lifted the chin on
108
- // the level and downward looks too, so the user was
109
- // looked at down the nose.
110
- pose: { lidL: -0.10, lidR: -0.10 } },
111
- // Eyes on the user for the whole turn, and no `aversion`: gaze.js has why a
112
- // speaker's measured looks away are not this rig's to render. What moves
113
- // while it talks is the head it holds per phrase and the trunk under it
114
- // (prosody.js), never the eyes leaving the user.
115
- SPEAKING: { gaze: 'USER', emotion: 'neutral',
116
- idle: { sway: 0.55 }, engagement: false },
117
- REVIEWING_SCREEN: { gaze: 'SCREEN_CENTER', emotion: 'thoughtful', engagement: false,
118
- idle: { sway: 0.8, blinkGap: [4.0, 6.5] },
119
- wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_RIGHT', 'SCREEN_TOP', 'SCREEN_WORK'],
120
- every: [1.8, 5.0] } },
121
- // The head cant is the state's signature cue, and it has to clear the roll
122
- // multiplier to exist at all: 0.05 here renders as 0.3° of rotation, which
123
- // is no tilt whatever the number says. 0.30 renders ~1.7° — visible at tile
124
- // size, still gentle.
125
- WAITING_FOR_USER: { gaze: 'USER', emotion: 'encouraging', engagement: true,
126
- idle: { sway: 1.0, blinkGap: [3.1, 4.2] },
127
- pose: { headRoll: 0.30, browRaiseL: 0.16, browRaiseR: 0.12 } },
128
- // Straining to hear. The one state where the amplitude constraint yields,
129
- // because the lean IS the message: torsoLean well past what the engagement
130
- // layer ever spends, head cheated aside on USER_EAR so an ear
131
- // favors the speaker while the eyes hold contact, and a concentration
132
- // squint with knit brows. Stillness does the rest — straining people
133
- // freeze — so holds are frequent and there is no engagement lean: you don't
134
- // nod along to what you can't hear. Server sends it on soft/low-SNR user
135
- // audio, typically followed by SORRY or a "could you repeat" utterance.
136
- CANT_HEAR: {
137
- gaze: 'USER_EAR', emotion: 'neutral', engagement: false,
138
- idle: { sway: 0.5, blinkGap: [4.5, 6.5],
139
- hold: { every: [2.5, 5.5], dur: [1.0, 1.8] } },
140
- // A minimal line face swallows small deltas — the ink moves whole units
141
- // or it doesn't move. These values are set from the contact sheet's
142
- // extremes row, not from what a fleshed rig would need: brows DOWN
143
- // (corrugator effort, not the browInner worry-lift), a real squint, and
144
- // the resting smile pressed flat — nobody smiles while straining to hear.
145
- // The squint is the lower lid's (AU7), with no upper lid dropped on top:
146
- // the two together narrowed a photographic eye from above and read as
147
- // drowsy, and a line face's squint is its lower lid already.
148
- pose: {
149
- torsoLean: 0.70, headPitch: 0.10,
150
- squintL: 0.75, squintR: 0.75,
151
- browRaiseL: -0.45, browRaiseR: -0.45, browInnerL: 0.15, browInnerR: 0.12,
152
- mouthPress: 0.45, mouthCornerL: -0.22, mouthCornerR: -0.22,
153
- },
154
- },
155
- // The user's microphone is closed, and the agent is the one that closed it.
156
- // Authored against CANT_HEAR as its exact inverse, because that contrast is
157
- // the whole read: straining leans *in* to get more of the user, this settles
158
- // *back* — nothing more is coming through and the avatar knows why. The eyes
159
- // stay on the user, because a deliberate hold is not inattention; what
160
- // carries it is the mouth, pressed shut and staying shut, and the slow,
161
- // unhurried blink of waiting rather than working. No engagement lean: you
162
- // cannot nod along to a channel you have muted yourself. No filter, ever —
163
- // DEGRADED and OFFLINE own "something is broken", and this is a decision.
164
- MUTED: {
165
- gaze: 'USER', emotion: 'neutral', engagement: false,
166
- idle: { sway: 0.45, blinkGap: [4.5, 7.0],
167
- hold: { every: [3.0, 6.0], dur: [0.9, 1.6] } },
168
- // Line-face scaled (see CANT_HEAR): peep's resting mouth is drawn smiling,
169
- // so a closed mouth has to be authored clearly past flat to read as closed
170
- // at all. Corners were -0.20 first, which rendered as a straight line and
171
- // at 130 px against LISTENING said nothing — the delta a viewer gets is
172
- // curvature, and flat is the halfway point of it, not the end. -0.34 is as
173
- // far as it goes before the hold starts reading as sulking. No squint and
174
- // no lid drop: one is straining, the other was asleep. browInner carries
175
- // the "one moment" without the worry lift.
176
- pose: {
177
- torsoLean: -0.28, headPitch: 0.04,
178
- browRaiseL: 0.08, browRaiseR: 0.06, browInnerL: 0.28, browInnerR: 0.22,
179
- mouthPress: 0.78, mouthCornerL: -0.34, mouthCornerR: -0.34,
180
- },
181
- },
182
- // --- application state ---------------------------------------------------
183
- // "Momentarily busy on the thing you asked for." No hands in frame, so the
184
- // whole read comes from cheap cues (docs/research-biomechanics.md §6.4,
185
- // recommendation 19): gaze down on one stable target with a reading scan,
186
- // blinks suppressed to task-focus rate (~9/min), shoulders slightly raised
187
- // and *held* with brief micro-freezes, and a glance back up to the user.
188
- // The glance is the tell that the user has not been forgotten; without it,
189
- // busy is just absent, and much more often than that it is fidgeting. There
190
- // is no shoulder rhythm: at 2.2 Hz it ran over the 1.5 Hz ceiling every
191
- // other motion here keeps, and held is what §6.4 describes. The trunk sits
192
- // out the glance — checking on someone is a look, not a turn toward them.
193
- WORKING: {
194
- // OWN_SCREEN: eyes down at the agent's own display, head nearly level.
195
- // This was SCREEN_WORK, which turned the head down and to the left and
196
- // held it there for the whole task, with a random drift too small to see
197
- // on it — a head turned away from the user, staring at a point. `read` is
198
- // what makes it busy instead: fixations stepping along a line, a sweep
199
- // back, the next line, a pause on the result (see readStep). The lids
200
- // take back most of the down look's follow so the eyes stay awake.
201
- gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
202
- scan: [0.9, 2.0, 0.12],
203
- // §6.4's ~9/min is a count of blinks: at the table's own 6-7.5 s gap it
204
- // measured 12-13/min, which is not focus.
205
- idle: { sway: 0.5, blinkGap: [6.8, 8.6], breathRate: 1.05, breathAmp: 0.8,
206
- hold: { every: [5.0, 9.0], dur: [0.6, 1.1] } },
207
- // The look up to check is brows-first and blinkless, the lids leading it
208
- // (their 18 ms against the eye's 32 ms); the blink comes on the way back
209
- // down, where the eyes re-engage with the work. A blink going up hid the
210
- // one moment the user is actually looked at.
211
- glance: { to: 'USER', every: [6, 11], hold: [0.8, 1.2], brow: 0.10,
212
- blinkTo: false, blinkBack: true },
213
- // Fewer, longer fixations than a page reader's. At 3-5 steps a quarter
214
- // second apart the scan measured a shift a second, and over a webcam that
215
- // is a jiggle, not reading.
216
- read: { width: 0.20, steps: [2, 3], fix: [0.35, 0.7], lines: [2, 4], line: 0.05,
217
- pause: [0.9, 1.8] },
218
- // The brows draw down and together — AU4, the one facial action that
219
- // reliably marks effortful concentration. Without it the face over a
220
- // reading scan is blank, and blank over a moving eye reads as bored.
221
- pose: { headPitch: 0.04, lidL: -0.08, lidR: -0.08,
222
- shoulderL: 0.06, shoulderR: 0.06,
223
- browRaiseL: -0.08, browRaiseR: -0.08, browInnerL: -0.10, browInnerR: -0.10 },
224
- },
225
- // The audio channel is broken and the agent is typing in the chat window to
226
- // communicate. The glance looks up and HOLDS 1.2–2 s, expectant, because the
227
- // chat (and the user's face) is now the only channel there is. A touch of
228
- // browInner carries the apology. Relation to DEGRADED is by semantics, not merger:
229
- // DEGRADED says "my feed is broken", TYPING_CHAT says "I'm working around
230
- // it" — a server will typically sequence DEGRADED → TYPING_CHAT.
231
- TYPING_CHAT: {
232
- gaze: 'SCREEN_WORK', emotion: 'neutral', engagement: false,
233
- idle: { sway: 0.6, blinkGap: [5.5, 7.0], breathRate: 1.05,
234
- rhythm: { amp: 0.055, freq: 2.5 } },
235
- glance: { to: 'USER', every: [3.2, 5.5], hold: [1.2, 2.0] },
236
- // Line-face scaled (see CANT_HEAR); the apology has to survive the rig's
237
- // baked resting smile, so the corners go clearly negative.
238
- pose: { headPitch: 0.10, lidL: -0.04, lidR: -0.04,
239
- shoulderL: 0.06, shoulderR: 0.06,
240
- browInnerL: 0.45, browInnerR: 0.38,
241
- mouthPress: 0.50, mouthCornerL: -0.28, mouthCornerR: -0.28 },
242
- },
243
- // Attention genuinely elsewhere. What separates this from WORKING is target
244
- // *stability* (§6.4): busy is one steady off-user target, distracted is
245
- // wandering ones, held long (aversion >3s), with no engagement lean — the
246
- // missing nod is as diagnostic as the look-away. Sway is looser than
247
- // LISTENING because attention is what was holding the body still. The
248
- // widget only looks away; deciding when to snap back is the server's call.
249
- DISTRACTED: {
250
- gaze: 'AWAY_RIGHT', emotion: 'neutral', engagement: false,
251
- idle: { sway: 1.15, blinkGap: [1.8, 4.2] },
252
- // Sideways and up, never steep-down: lateral is where real intimacy/
253
- // distraction aversions live, and a steep down target seals this rig's
254
- // eyes (see WORKING).
255
- wander: { targets: ['AWAY_RIGHT', 'AWAY_THINKING', 'SCREEN_LEFT', 'SCREEN_TOP'],
256
- every: [2.8, 6.8] },
257
- },
258
- // The buying-time move: hunting for a control on screen. Distinct from
259
- // REVIEWING_SCREEN by *hunt* quality — reading dwells (1.8–5 s) become
260
- // search saccades (0.8–2 s) with revisits (targets repeat in the wander
261
- // set), plus the idle layer's flick: the tiny "no, not this one" yaw
262
- // wiggle nobody makes while merely reading. Server semantics: a filler
263
- // while an async activity completes; the server exits it when done.
264
- SEARCHING_SCREEN: {
265
- gaze: 'SCREEN_CENTER', emotion: 'neutral', engagement: false,
266
- idle: { sway: 0.65, blinkGap: [5.0, 6.8], breathRate: 1.05,
267
- flick: { amp: 0.30, every: [3.5, 7.0] } },
268
- // The only screen state that never named its own scan, so it ran the
269
- // default [0.5, 1.6] at full amplitude: a second full-size jump landing on
270
- // top of a wander hop, which is most of what reads as snapping. A hunt's
271
- // refixations are faster and much smaller than a reader's — quick little
272
- // checks around the thing being looked at — so this is the other end of
273
- // the dial from WORKING's [0.9, 2.0, 0.12] rather than a copy of it.
274
- scan: [0.4, 1.1, 0.55],
275
- wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_TOP', 'SCREEN_WORK',
276
- 'SCREEN_RIGHT', 'SCREEN_CENTER', 'SCREEN_LEFT'],
277
- every: [0.8, 2.0] },
278
- // Line-face scaled (see CANT_HEAR). Note the corners: peep's REST mouth
279
- // is drawn smiling, so "not smiling" is a clearly negative net corner,
280
- // not zero.
281
- pose: { squintL: 0.40, squintR: 0.40, mouthPress: 0.65,
282
- mouthCornerL: -0.25, mouthCornerR: -0.25, browRaiseL: -0.26, browRaiseR: -0.20 },
283
- },
284
- // --- floor management ----------------------------------------------------
285
- // Turn-taking is the part of a voice call that goes wrong most often: the
286
- // user either talks over the agent or sits in silence waiting for a signal
287
- // that never comes. These are states rather than clips because the floor is a
288
- // condition and not an event — WANTS_IN in particular has to hold for as long
289
- // as it takes the other person to notice it.
290
- //
291
- // Each of these lifts the shoulders and parts the lips, because that is what an
292
- // inbreath looks like from outside, and an inbreath is the cue humans actually
293
- // use to predict that someone is about to speak. The head comes *up* rather
294
- // than down: a lowered head is deferential and reads as yielding.
295
- TAKING_FLOOR: {
296
- gaze: 'USER', emotion: 'neutral', idle: { sway: 0.6 }, engagement: false,
297
- pose: {
298
- browRaiseL: 0.26, browRaiseR: 0.22, lidL: -0.10, lidR: -0.10,
299
- headPitch: -0.10, torsoLean: 0.22, shoulderL: 0.30, shoulderR: 0.30,
300
- mouthOpen: 0.10, mouthPress: -0.10,
301
- },
302
- },
303
- // The one signal the rig had no way to give at all. An agent needs to be
304
- // able to say "I'd like to come in" without talking over the user, and
305
- // every part of this pose is doing that job: held still (idle is low on
306
- // purpose — stillness is what makes it read as intent rather than as fidget),
307
- // leaning in, lips apart and staying apart.
308
- WANTS_IN: {
309
- gaze: 'USER', emotion: 'neutral', idle: { sway: 0.45 }, engagement: false,
310
- pose: {
311
- browRaiseL: 0.42, browRaiseR: 0.38, lidL: -0.14, lidR: -0.14,
312
- headPitch: -0.14, torsoLean: 0.42, shoulderL: 0.45, shoulderR: 0.45,
313
- mouthOpen: 0.16, mouthWidth: 0.30, mouthPress: -0.14,
314
- },
315
- },
316
- // Interrupted mid-word. The mouth shutting is the whole message, and it has to
317
- // happen faster than anything else on the face — see YIELD_FLOOR, which is
318
- // what actually delivers the snap.
319
- YIELDED: {
320
- gaze: 'USER', emotion: 'neutral', idle: { sway: 0.9 }, engagement: false,
321
- pose: {
322
- browRaiseL: 0.10, browRaiseR: 0.08,
323
- torsoLean: -0.18, shoulderL: -0.12, shoulderR: -0.12,
324
- },
325
- },
326
-
327
- // --- connection ----------------------------------------------------------
328
- // Neither of these may close the eyes. A lid at 0.3 reads sleepy and one at
329
- // 0.95 reads asleep (research-perception.md §6), and on a photographic face
330
- // a disconnected call rendered as someone falling asleep on camera. The SVG
331
- // faces also desaturate (`filter`); a rig that does not is carried by pose
332
- // and gaze alone, so those have to say it.
333
- //
334
- // Our side of the call is struggling: still with the user, a little worried
335
- // about it — inner brows up, mouth pressed — and quieter than listening.
336
- DEGRADED: { gaze: 'USER', emotion: 'neutral', engagement: false,
337
- aversion: 'LISTEN',
338
- idle: { sway: 0.4, blinkGap: [3.4, 5.0] },
339
- pose: { browInnerL: 0.30, browInnerR: 0.26, browRaiseL: -0.04, browRaiseR: -0.04,
340
- mouthPress: 0.20, mouthCornerL: -0.10, mouthCornerR: -0.10 },
341
- filter: 'grayscale(.55) brightness(.82)' },
342
- // The call is gone. What a person does when their call drops: eyes on their
343
- // own screen, waiting, and now and then a look back up to see whether it
344
- // has come back. Still, with long holds — not in conversation — but awake.
345
- OFFLINE: { gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
346
- scan: [1.2, 2.6],
347
- idle: { sway: 0.3, blinkGap: [4.5, 7.0],
348
- hold: { every: [4.0, 8.0], dur: [1.0, 2.0] } },
349
- glance: { to: 'USER', every: [7, 12], hold: [0.8, 1.3] },
350
- pose: { lidL: -0.08, lidR: -0.08, mouthPress: 0.25 },
351
- filter: 'grayscale(1) brightness(.6)' },
352
- };
353
-
354
- export const STATE_NAMES = Object.keys(STATES);
355
-
356
- /**
357
- * The shoulder line's share of a held tilt, per unit of `headRoll`.
358
- *
359
- * Roll is the one head axis with nothing under it: a turn recruits the trunk
360
- * and a nod bends the neck, but a tilt on a 2.5-D head is a rotation about a
361
- * point near the chin and every other pixel stays exactly where it was. That is
362
- * the read reported as "a hinge". Anatomically a tilt is lower-cervical
363
- * (docs/research-head-rotation.md §2.1) — spread down the neck, ending at the
364
- * girdle — so the shoulder line tips a little with it, and that tip is what says
365
- * the neck bent rather than the head swinging off a pin.
366
- *
367
- * Sized to stay an accompaniment: at the roll clamp the line tips about a
368
- * quarter of the trunk's own share of a turn, which lifts a shoulder ~0.11 of
369
- * its shrug — inside the band a weight shift already occupies. A third, the
370
- * figure Live2D gives its body angles, put 6 px of shoulder on an 8 degree tilt
371
- * and read as a shrug arriving with the head.
372
- *
373
- * Exported for the rig instruments.
374
- */
375
- export const SHOULDER_TILT = 0.08;
376
-
377
- export function createAvatar(opts = {}) {
378
- const mount = typeof opts.mount === 'string' ? document.querySelector(opts.mount) : opts.mount;
379
- if (!mount) throw new Error('createAvatar: mount element required');
380
-
381
- // `opts.face` is a Face record — `{ create, meta }`. Passed in rather than
382
- // named: a name needs a table, and a table imports every drawing to answer
383
- // one lookup (`src/faces.js` has that table, for tooling that wants them).
384
- const entry = opts.rig ? null : opts.face;
385
- if (!opts.rig && !entry) {
386
- throw new Error('createAvatar: a `face` (see src/faces.js) or a `rig` is required');
387
- }
388
- const face = entry ? entry.create(mount, opts.theme) : null;
389
- // A renderer-neutral rig needs no SVG descriptor: `meta` is what SVG hosts
390
- // and tools frame the drawing with, never a requirement of the rig contract.
391
- // A face carries its own — a `{ create }` with no META used to be tolerated
392
- // here and the viewBox re-read off the produced svg, which meant a face could
393
- // ship half a descriptor and nothing would say so.
394
- const meta = face ? entry.meta : null;
395
- const gaze = new GazeLayer();
396
- const idle = new IdleLayer();
397
- const speech = new VisemeTrack();
398
- const prosody = new SpeechProsody({ brows: opts.brows });
399
-
400
- let gazeOverrideByClip = null;
401
- const clip = new ClipPlayer({
402
- onGaze: (g) => { gazeOverrideByClip = g; applyGaze(); },
403
- onBlink: () => idle.blink(),
404
- });
405
- const engagement = new ListeningEngine();
406
- const performTrack = new PerformTrack();
407
- // The current SVG hand is a renderer adapter for the first-class `frame.hand`
408
- // control. `hand: false` only disables its SVG rendering; gesture actions
409
- // still emit the semantic hand frame for a supplied custom rig.
410
- const hand = face && opts.hand !== false ? createHand(face.svg, face.theme, meta, { dir: opts.handSide }) : null;
411
- const rig = opts.rig ? opts.rig(mount, opts.rigOptions) : createSvgRig(face, hand);
412
-
413
- gaze.onLargeShift = (forced) => idle.blink(true, forced);
414
-
415
- const listeners = {
416
- state: [], speakEnd: [], clipEnd: [], performEnd: [], gestureEnd: [],
417
- };
418
- const emit = (ev, ...a) => listeners[ev] && listeners[ev].forEach((f) => f(...a));
419
- clip.onEnd = (c) => { if (c) emit('clipEnd', c.id); };
420
- speech.onEnd = () => { prosody.closeTurn(); emit('speakEnd'); };
421
- performTrack.onEnd = () => { emit('performEnd'); };
422
-
423
- // --- live state -----------------------------------------------------------
424
- let stateName = 'IDLE';
425
- let emotion = 'neutral';
426
- let emotionAmt = 1;
427
- let gazeName = 'USER';
428
- let gazeCustom = null;
429
- let overrides = null; // demo/debug direct param injection
430
- // Articulation gain. VISEME_SHAPES is tuned for a face at conversational
431
- // size; sharing the screen with live video the same shapes read as
432
- // under-articulated. Scales every viseme away from rest, so shape identities
433
- // and their relative sizes survive and only the excursion changes. Above
434
- // ~1.5 the open vowels saturate against the clamp — the intended ceiling.
435
- let mouthGain = opts.mouthGain ?? 1;
436
- let handSide = opts.handSide === -1 ? 'left' : 'right';
437
- /**
438
- * This avatar's own addressable motions, on top of the core intents.
439
- *
440
- * An avatar is a drawing with a body, and some bodies can do things the wire
441
- * has no portable word for. The nod types the listening research separates —
442
- * a continuer, an assessment, a realisation — are one
443
- * `ACKNOWLEDGE` to a server, because that is all a server can ask of every
444
- * face; the *shapes* are sized in a rig's own units and belong to the rig
445
- * (`packages/avatar/client/three/sequences.ts` is the first table of them).
446
- *
447
- * Deliberately not a registry and not a loader: it is a plain object the
448
- * avatar module passes in, and it can only add. `ACTIONS` wins on a name
449
- * collision, so no asset can quietly change what a core action means.
450
- */
451
- const sequences = opts.sequences || {};
452
- /**
453
- * This avatar's own *rendering* of an id this renderer already has — same id,
454
- * same intent, a shape sized for its body.
455
- *
456
- * These are an intent vocabulary, and `ACK_NOD` is one implementation of
457
- * "the avatar nodded", not a promise about anatomy (interjections.js). The
458
- * shared clips are authored in pose units that mean pixels on a line face;
459
- * on a rig whose unit is a degree the same keys can land outside what the
460
- * gesture *means* — the shared nod renders 15° on tara, where a continuer is
461
- * 3-5° — and scaling it would not fix it, because a continuer is one stroke
462
- * where an agreement is two. So a renderer may re-author the shape.
463
- *
464
- * What it may not do is change the vocabulary: every id here must already be
465
- * one this renderer publishes, so no avatar can add a word through this door
466
- * or make one mean something else. `sequences` is the door for adding.
467
- */
468
- const actionShapes = opts.actions || {};
469
- for (const id of Object.keys(actionShapes)) {
470
- if (!ACTIONS[id]) throw new Error(`actions: ${id} is not one of this renderer's own`);
471
- }
472
- /**
473
- * This avatar's own rendering of a state — the same door as `actions`, for
474
- * the held face instead of the gesture. A rig replaces a state's fields
475
- * whole (`pose`, say), and only for a state the table already has: it can
476
- * re-render the vocabulary, never extend it.
477
- */
478
- const states = { ...STATES };
479
- for (const [id, own] of Object.entries(opts.states || {})) {
480
- if (!STATES[id]) throw new Error(`states: ${id} is not a state`);
481
- states[id] = { ...STATES[id], ...own };
482
- }
483
- let handAction = null;
484
- const handQueue = [];
485
- // Gesture gain, same idea for the clip layer: small gestures under-render
486
- // through the head's τ — see internal-mixer.md § Smoothing.
487
- let gestureGain = opts.gestureGain ?? 1;
488
- // Body-liveness gain. Constraint 8 (this widget shares the screen with a
489
- // live video call) argues for the smallest idle motion that still reads, and
490
- // the amplitudes in idle.js are set where they read. A host that is actually
491
- // re-encoding the avatar — compositing it into an outgoing stream rather
492
- // than rendering it locally as SVG, where the motion costs nothing — turns
493
- // this down instead of the default being a body that does not move.
494
- idle.gain = opts.motionGain ?? 1;
495
- // Speech-rhythm gains. A pose unit is a different angle on every rig — peep's
496
- // head travels 17 px per unit of pitch, a mesh head turns a few degrees — so
497
- // the rig's own module sets these once rather than the library guessing.
498
- // Defaults to the uncalibrated scale; `prosody.js` has what that stands for.
499
- const prosodyHeadGain = opts.prosodyHeadGain ?? UNCALIBRATED_HEAD_GAIN;
500
- const prosodyFaceGain = opts.prosodyFaceGain ?? 1;
501
- // Fixational-saccade gain, the same idea for the eyes: how far a scan
502
- // step, a drift or a line of reading travels. Sized for a line face's
503
- // pupils at 1; a photographic eye needs several times that to move at all.
504
- // Aversions get their own gain: a look-away must read as one from across
505
- // the call, where a fixation step must not, so one number cannot size both.
506
- const saccadeGain = opts.saccadeGain ?? 1;
507
- // How far this face may hold its head off centre, per axis (internal-mixer.md
508
- // § The held-head budget): the rig supplies it; an axis left out is unbudgeted.
509
- const headHold = opts.headHold || {};
510
- // Per-axis gain on the head's *continuous* drive — speech phrasing and idle,
511
- // both sized in pose units for a line drawing. Measuring a 2.5-D face found
512
- // the drive spending a third of the pitch a speaking human uses and nearly
513
- // twice the yaw. In front of 6b deliberately: a gain that skipped the budget
514
- // would be measuring the face's failure rather than the layer's range.
515
- const headGain = typeof opts.headGain === 'number'
516
- ? { headYaw: opts.headGain, headPitch: opts.headGain, headRoll: opts.headGain }
517
- : { headYaw: 1, headPitch: 1, headRoll: 1, ...(opts.headGain || {}) };
518
- gaze.scanGain = saccadeGain;
519
- gaze.avertGain = opts.aversionGain ?? 1;
520
- // A rig that says what its pose units are in degrees gets the eye-head
521
- // system sized for it (gaze.js): its own look targets, how an aversion
522
- // splits between eyes and head, lids that follow the eye both ways, and the
523
- // reflex in step 8b.
524
- const ocu = opts.oculomotor || {};
525
- if (ocu.targets) gaze.targets = { ...GAZE_TARGETS, ...ocu.targets };
526
- if (ocu.avert) gaze.avertSplit = ocu.avert;
527
- if (ocu.angles) gaze.angles = ocu.angles;
528
- if (ocu.lidFollow) gaze.lidFollow = ocu.lidFollow;
529
- if (ocu.head) { gaze.headAccel = ocu.head.accel; gaze.headSpeed = ocu.head.speed; }
530
- // Pupil units of counter-rotation per head unit: the reflex's gain times
531
- // the ratio of what one unit of each is in degrees. The gain may differ by
532
- // axis ({x, y}): a face whose pitch reads weaker than its yaw wants less
533
- // of the eyes' answer to it.
534
- const vorGain = typeof ocu.vor === 'object' ? ocu.vor : { x: ocu.vor, y: ocu.vor };
535
- const vor = ocu.vor && ocu.angles
536
- ? { x: vorGain.x * ocu.angles.head.x / ocu.angles.eye.x,
537
- y: vorGain.y * ocu.angles.head.y / ocu.angles.eye.y }
538
- : null;
539
- // How far the reflex may carry the eye in its socket, in pupil units:
540
- // Guitton & Volle's effective oculomotor range; the rig supplies the reach.
541
- const reach = ocu.range || { x: 1, up: 1, down: 1 };
542
- const reflexX = (px) => clamp(px + vor.x * (aim.x - cur.headYaw), -reach.x, reach.x);
543
- const reflexY = (py) => clamp(py + vor.y * (aim.y - cur.headPitch), -reach.up, reach.down);
544
- // The head the eyes' target was authored against, smoothed at the eye's
545
- // own tau so a shift's compensation moves with the saccade, not ahead of it.
546
- const aim = { x: 0, y: 0 };
547
- let wanderAt = 0;
548
- let driftAt = 0;
549
- let trunkYaw = 0;
550
- let glanceAt = 0;
551
- let glanceUntil = 0;
552
- let lastBack = null;
553
- // THINKING's second look inside a look away: when it moves, where to, and
554
- // how long the brows stay up for it.
555
- let dartAt = 0, dartBrowUntil = 0;
556
- const dart = { x: 0, y: 0 };
557
- // The state whose gaze is showing, and when the current state takes it
558
- // over if that is still pending (GAP_SETTLE). Usually the same state.
559
- let gazeState = 'IDLE';
560
- let settleAt = 0;
561
- // Reading scan: position in the line and the block, see readStep.
562
- let readCol = 0, readCols = 0, readRow = 0, readRows = 0;
563
- let attendUntil = 0;
564
- let speakClock = null;
565
- let speakStart = 0;
566
-
567
- const cur = Object.assign({}, REST);
568
- const target = Object.assign({}, REST);
569
- // The head axes again, carrying only what is held (step 6b).
570
- const hold = { headYaw: 0, headPitch: 0, headRoll: 0 };
571
- // What the rig is handed: `cur` with the reflex applied to the eyes. The
572
- // same object when there is no reflex.
573
- const shown = vor ? Object.assign({}, REST) : cur;
574
-
575
- function applyGaze(blink) {
576
- const g = gazeOverrideByClip || gazeName;
577
- gaze.set(g, gazeOverrideByClip ? null : gazeCustom, blink);
578
- }
579
-
580
- // --- the frame ------------------------------------------------------------
581
- let raf = 0;
582
- let last = 0;
583
- let elapsed = 0;
584
- // `manual` withholds the rAF loop so a tool can drive frames itself.
585
- const manual = !!opts.manual;
586
-
587
- function frame(now) {
588
- raf = requestAnimationFrame(frame);
589
- if (!last) last = now;
590
- // Cap dt so a backgrounded tab doesn't fast-forward the whole rig on return.
591
- const dt = Math.min(0.05, (now - last) / 1000);
592
- last = now;
593
- elapsed += dt;
594
- step(dt, dt * 1000);
595
- }
596
-
597
- function step(dt, dtMs) {
598
- // 0. the performance timeline. Sampled before the pose is built so a verb
599
- // firing this frame shapes this frame.
600
- performTrack.update();
601
-
602
- if (settleAt && elapsed >= settleAt) { settleAt = 0; enterGaze(stateName, {}, true); }
603
- const st = states[stateName] || states.IDLE;
604
- // What the eyes are doing belongs to the state that has the gaze.
605
- const gst = states[gazeState] || st;
606
-
607
- // 1. base pose: rest + emotion + state-specific overlay
608
- for (const c of CHANNELS) target[c] = REST[c];
609
- const ep = emotionPose(emotion, emotionAmt);
610
- for (const k in ep) target[k] = REST[k] + ep[k];
611
- if (st.pose) for (const k in st.pose) target[k] = (target[k] || 0) + st.pose[k];
612
- // The head the state and emotion hold is an attitude, and the eyes are
613
- // authored inside it: the reflex (8b) keeps them on their target against
614
- // everything that moves the head except this.
615
- const poseYaw = target.headYaw, posePitch = target.headPitch;
616
- // What the head *holds*, accumulated alongside the pose as each layer that
617
- // holds one adds to it: the attitude above, the gaze, a phrase's pose and
618
- // the idle posture. Strokes, beats and clip deltas stay out — see 6b.
619
- for (const c of HEAD_AXES) hold[c] = target[c];
620
-
621
- // 2. gaze (absolute: pupils + partial head follow, plus the lid that rides
622
- // with the eye — looking down without it bares sclera and reads as alarm)
623
- const g = gaze.update(elapsed, dt);
624
- for (const k in g) {
625
- if (k === 'trunkYaw' || k === 'aimYaw' || k === 'aimPitch') continue;
626
- target[k] = (k.startsWith('head') ? target[k] : 0) + g[k];
627
- }
628
- for (const c of HEAD_AXES) hold[c] += g[c];
629
- // The lid follows the eye as drawn, reflex and all — last frame's reading
630
- // of it, since the head it corrects for is not smoothed yet.
631
- const lid = gaze.lidBias(vor ? reflexY(g.pupilY) : g.pupilY);
632
- target.lidL += lid;
633
- target.lidR += lid;
634
- if (vor) {
635
- aim.x = approach(aim.x, poseYaw + g.aimYaw, TAU.pupilX, dt);
636
- aim.y = approach(aim.y, posePitch + g.aimPitch, TAU.pupilY, dt);
637
- }
638
-
639
- // 2b. the trunk follows the head. Sampled HERE, after gaze and before the
640
- // clip layer, on purpose: a sustained turn toward the screen recruits
641
- // the trunk, and a nod or a head shake does not — a body that swings
642
- // with every gesture reads as a mannequin on a turntable. The lag is
643
- // not authored anywhere; torsoTurn chases the same target at a slower
644
- // TAU than the head (params.js), so the trunk leaves late and settles
645
- // late for free, at the share TRUNK_FOLLOW names. It follows where the
646
- // head is going and not the looks riding on it, and it holds through a
647
- // glance: checking on the user is a look, not a turn toward them.
648
- if (!glanceUntil) trunkYaw = target.headYaw - g.headYaw + g.trunkYaw;
649
- target.torsoTurn += trunkYaw * TRUNK_FOLLOW;
650
-
651
- // 3. state-driven autonomous behaviour
652
- if (gst.wander && elapsed > wanderAt) {
653
- const w = gst.wander;
654
- // Dwell first, and sometimes for two periods. A dwell of exactly one
655
- // period every time is the metronome the eye reads as mechanical. The
656
- // old code broke that up by accident: a draw landing on the target
657
- // already held left the eyes still for another period. The freeze was a
658
- // bug — the schedule counted a hop that moved nothing — but the dwell
659
- // variety it produced was not, so it is kept deliberately, at the rate
660
- // it used to happen: 1/n, the odds a uniform draw repeats the target.
661
- const linger = Math.random() < 1 / w.targets.length ? 2 : 1;
662
- wanderAt = elapsed + linger * (w.every[0] + Math.random() * (w.every[1] - w.every[0]));
663
- // Then look somewhere the eyes are not. A wander set names a target more
664
- // than once deliberately — SEARCHING_SCREEN's repeats are its revisits,
665
- // the hunt coming back to the middle — but a revisit is arriving
666
- // somewhere again, not never having left it. Same bounded redraw as
667
- // pickBack, so a one-target set still terminates.
668
- let g = pick(w.targets);
669
- for (let i = 0; i < 4 && g === gazeName; i++) g = pick(w.targets);
670
- setGaze(g);
671
- }
672
- // Periodic glance (WORKING's look-up-at-you beat, THINKING's check-in
673
- // between two looks away). The return leg goes to the state's own gaze, or
674
- // to a fresh pick from `back` — a thinker does not look away at the same
675
- // spot every time. Either leg's gaze-evoked blink is the shift's odds
676
- // unless the state says (`blinkTo`, `blinkBack`), because which leg blinks
677
- // is part of what the glance means.
678
- if (gst.glance) {
679
- const gl = gst.glance;
680
- if (glanceUntil && elapsed > glanceUntil) {
681
- glanceUntil = 0;
682
- const away = rand(gl.every);
683
- glanceAt = elapsed + away;
684
- setGaze(gl.back ? pickBack(gl.back, gl.stick) : gst.gaze, null, gl.blinkBack);
685
- // At most one second look per look away, somewhere in its middle: a
686
- // move at the start is the same look landing, and one at the end runs
687
- // into the glance back.
688
- dartAt = gl.dart && Math.random() < gl.dart.p ? elapsed + away * (0.35 + Math.random() * 0.3) : 0;
689
- } else if (!glanceUntil && elapsed > glanceAt) {
690
- glanceUntil = elapsed + rand(gl.hold);
691
- setGaze(gl.to, null, gl.blinkTo);
692
- dartAt = 0;
693
- dart.x = dart.y = 0;
694
- }
695
- if (dartAt && elapsed > dartAt) {
696
- // On from where the eyes are, not back toward the user or across to
697
- // the other side: the thought moves, it does not turn round.
698
- dartAt = 0;
699
- const t = gaze.target;
700
- const a = Math.atan2(t.py, t.px) + (Math.random() < 0.5 ? -1 : 1) * DART_TURN;
701
- const m = gl.dart.mag * saccadeGain;
702
- dart.x = Math.cos(a) * m;
703
- dart.y = Math.sin(a) * m;
704
- dartBrowUntil = elapsed + DART_BROW;
705
- }
706
- const brow = (glanceUntil ? gl.brow || 0 : 0)
707
- + (gl.dart && elapsed < dartBrowUntil ? gl.dart.brow : 0);
708
- target.browRaiseL += brow;
709
- target.browRaiseR += brow;
710
- }
711
- gaze.scanEvery = gst.scan || null;
712
- // A state that reads scans a line; one with a dart holds its second look.
713
- // Neither during a glance: a check-in on the user is steady or it is not
714
- // one.
715
- if (gst.read && !glanceUntil) {
716
- if (elapsed > driftAt) readStep(gst.read);
717
- } else {
718
- gaze.drift.x = glanceUntil ? 0 : dart.x;
719
- gaze.drift.y = glanceUntil ? 0 : dart.y;
720
- }
721
- // Aversion is a property of the state, but it is held off around a turn
722
- // boundary: the floor is handed over under mutual gaze, and an avatar that
723
- // looks away exactly as the user finishes has declined it. `attend` is the
724
- // mixer's one-frame veto — anything that means "the user is checking
725
- // whether I am with them" sets it (see api.attend).
726
- gaze.setAversion(gst.aversion ? AVERSION[gst.aversion] : null);
727
- gaze.hold = attendUntil > elapsed || clip.playing;
728
-
729
- engagement.enabled = !!st.engagement && !clip.playing;
730
- engagement.update(dt);
731
- // Engagement posture: forward lean while the user holds the floor, spent
732
- // only in the states that are *about* the user holding the floor. The
733
- // research (docs/research-biomechanics.md §6.3) puts sustained attentive
734
- // lean at +0.15–0.25; engage glides, and torsoLean's 0.24s tau smooths
735
- // the state gate, so the lean arrives and leaves like weight shifting.
736
- if (st.engagement) {
737
- target.torsoLean += 0.16 * engagement.engage;
738
- // The shoulders come with it. A lean is a whole upper body arriving, and
739
- // in a head-and-shoulders crop the shoulder line is the part of it that
740
- // is actually on screen — §6.1 calls a shoulder rise the most legible
741
- // thing this framing can draw. Without this the listening shoulders were
742
- // the posture shift's ±0.07 and nothing else, so the one channel that
743
- // could show attention sat still through every turn the user took.
744
- // It rides `engage`, so it is contingent on the user's voice rather than
745
- // on the state: quick in, slow out, and gone a few seconds into silence.
746
- target.shoulderL += ENGAGE_SHOULDER * engagement.engage;
747
- target.shoulderR += ENGAGE_SHOULDER * engagement.engage;
748
- }
749
- // Straining leans harder while there is actually a faint voice to strain
750
- // after. engage already tracks "the user is (barely) talking", so this
751
- // costs nothing; with no user signal the static pose carries the state.
752
- else if (stateName === 'CANT_HEAR') target.torsoLean += 0.10 * engagement.engage;
753
-
754
- // 4. mouth. The server's viseme track wins; a clip's mouth track fills the
755
- // gaps. There is deliberately no third leg: with no cues the mouth stays
756
- // shut, and a still mouth under speech is the *visible* symptom of a
757
- // backend that could not align.
758
- const clipOut = clip.update(dtMs);
759
- let mouth = speech.sample();
760
- let mouthOwner = mouth ? 'speech' : null;
761
- if (!mouth && clipOut.ownsMouth && clipOut.mouth) { mouth = clipOut.mouth; mouthOwner = 'clip'; }
762
- if (mouth) {
763
- const shape = mouth.letter !== SILENT
764
- ? shapeFor(mouth.letter, mouth.intensity)
765
- : shapeFor(SILENT, 1);
766
- // Rounding coming up is held against the shape's own (VisemeTrack.sample).
767
- if (mouth.round > shape.mouthRound) shape.mouthRound = mouth.round;
768
- // Gain pivots on the rest shape, not on zero: scaling absolute values would
769
- // drag the closed mouth open, which is the one thing lipsync must never do.
770
- for (const k in shape) {
771
- target[k] = mouthGain === 1
772
- ? shape[k]
773
- : REST_SHAPE[k] + (shape[k] - REST_SHAPE[k]) * mouthGain;
774
- }
775
- // Clip-owned mouths are exempt — a spoken OKAY *is* the warmth episode —
776
- // and only the base is scaled, so a gesture clip can still smile over a
777
- // sentence by authoring corner keys (they add, unscaled, in step 5).
778
- if (mouthOwner !== 'clip') {
779
- target.mouthCornerL *= SPEAK_SMILE_RETAIN;
780
- target.mouthCornerR *= SPEAK_SMILE_RETAIN;
781
- }
782
- }
783
-
784
- // 4b. speech prosody: pause blinks, inbreaths, a head held per phrase and
785
- // moved between them, beats and turn-edge warmth, read off the cue
786
- // track (prosody.js has the research, head.js the hold-and-move). Off while a clip is gesturing, since the clip is
787
- // already the head's and the brows' story. The warmth is withheld
788
- // under an emotion whose corners are down: a smile starting a turn of
789
- // concern is the wrong face, whatever the rhythm says.
790
- const pro = prosody.update(speech, mouthOwner === 'speech' && !clip.playing, dt);
791
- // The warmth rides over a clip, because an acknowledgement's smile is
792
- // *with* its nod: behind the clip gate it arrived as the nod finished, a
793
- // smile at nothing. It is withheld under a clip whose own corners go down
794
- // (a shake, a sorry) for the same reason as under a concerned emotion.
795
- const clipFrowns = !!clipOut.delta
796
- && Math.min(clipOut.delta.mouthCornerL || 0, clipOut.delta.mouthCornerR || 0) < 0;
797
- const warm = clipFrowns || ((EMOTIONS[emotion] || EMOTIONS.neutral).mouthCornerL || 0) < 0
798
- ? 0 : prosodyFaceGain;
799
- target.mouthCornerL += pro.mouthCornerL * warm;
800
- target.mouthCornerR += pro.mouthCornerR * warm;
801
- target.squintL += pro.squintL * warm;
802
- target.squintR += pro.squintR * warm;
803
- if (!clip.playing) {
804
- target.headPitch += pro.headPitch * prosodyHeadGain * headGain.headPitch;
805
- target.headYaw += pro.headYaw * prosodyHeadGain * headGain.headYaw;
806
- target.headRoll += pro.headRoll * prosodyHeadGain * headGain.headRoll;
807
- for (const c of HEAD_AXES) hold[c] += pro.hold[c] * prosodyHeadGain * headGain[c];
808
- // The trunk follows a speech pose the way it follows a gaze turn (2b),
809
- // and for a second reason on a 2.5-D head: a turn the shoulders take
810
- // part of is a turn the neck does not have to stretch for, and the
811
- // stretch is the one thing a video reviewer saw on every pose change.
812
- // Scaled with the yaw it follows: a trunk that kept its old share of a
813
- // wider turn would be the neck stretching again, which is the defect
814
- // this line exists to fix.
815
- target.torsoTurn += pro.trunkYaw * prosodyHeadGain * headGain.headYaw * TRUNK_FOLLOW;
816
- target.browRaiseL += pro.browRaiseL * prosodyFaceGain;
817
- target.browRaiseR += pro.browRaiseR * prosodyFaceGain;
818
- target.browInnerL += pro.browInnerL * prosodyFaceGain;
819
- target.browInnerR += pro.browInnerR * prosodyFaceGain;
820
- target.browAngleL += pro.browAngleL * prosodyFaceGain;
821
- target.browAngleR += pro.browAngleR * prosodyFaceGain;
822
- target.lidL += pro.lidL * prosodyFaceGain;
823
- target.lidR += pro.lidR * prosodyFaceGain;
824
- target.breath += pro.breath * idle.gain;
825
- // The trunk's share of speech rhythm. Scaled by `idle.gain` and not by
826
- // `prosodyHeadGain`, because this is body liveness and that is the knob a
827
- // host turns down when it is re-encoding the avatar into an outgoing
828
- // stream — the same factor `idle.js`'s shoulders and lean already take.
829
- target.shoulderL += pro.shoulderL * idle.gain;
830
- target.shoulderR += pro.shoulderR * idle.gain;
831
- target.torsoLean += pro.torsoLean * idle.gain;
832
- }
833
- if (pro.blink) idle.phraseBlink();
834
-
835
- // 5. gesture deltas (additive, so a nod survives whatever else is happening)
836
- if (clipOut.delta) {
837
- for (const k in clipOut.delta) {
838
- // Never let a clip's mouth keyframes fight the live viseme stream.
839
- if (mouthOwner === 'speech' && MOUTH_LOCK.has(k)) continue;
840
- target[k] = (target[k] || 0) + clipOut.delta[k] * gestureGain;
841
- }
842
- }
843
-
844
- // 6. idle: sway, breath, blink
845
- // The torso's share of the liveness follows whether sound is actually being
846
- // produced, not what state the avatar is nominally in — a SPEAKING state with the
847
- // track finished should already be settling.
848
- idle.talk = approach(idle.talk, mouthOwner ? 1 : 0, 0.25, dt);
849
- idle.setProfile(st.idle);
850
- const il = idle.update(dt);
851
- for (const k in il.add) target[k] = (target[k] || 0) + il.add[k] * (headGain[k] ?? 1);
852
- for (const c of HEAD_AXES) hold[c] += (il.add[c] || 0) * headGain[c];
853
-
854
- // 6b. the held-head budget (internal-mixer.md § The held-head budget).
855
- // Only the excess over `soften` comes off, and only off the hold: a
856
- // nod, a beat and a clip keep every degree they were authored with,
857
- // which is why this is subtracted here rather than applied to the pose.
858
- for (const c of HEAD_AXES) {
859
- if (headHold[c] === undefined) continue;
860
- target[c] -= hold[c] - soften(hold[c], headHold[c]);
861
- }
862
-
863
- // 6c. the body answers a held tilt (SHOULDER_TILT). The trunk takes the
864
- // same share of it that it takes of a turn in 2b — Live2D gives its
865
- // body the same fraction of AngleZ as of AngleX (research-head-rotation
866
- // .md §3) — and the shoulder line tips with the head. Both channels
867
- // smooth slower than the head (TAU), so the body leaves late and
868
- // settles late, and that follow-through is most of what separates a
869
- // neck bending from a hinge.
870
- // Only the *held* roll, and after the budget: a stroke or a clip's roll
871
- // is a gesture riding on the pose, and a body that answers those is 2b's
872
- // mannequin on a turntable. The idle layer's own posture is already
873
- // coupled the other way round, from the weight shift to the head that
874
- // counter-tips on it (idle.js `nextPosture`); this is that arrangement
875
- // read from the head's end, and where both are in play they agree in
876
- // sign — weight onto a side, head over that side, that shoulder up.
877
- const heldRoll = headHold.headRoll === undefined
878
- ? hold.headRoll
879
- : soften(hold.headRoll, headHold.headRoll);
880
- target.torsoTurn += heldRoll * TRUNK_FOLLOW;
881
- target.shoulderR += heldRoll * SHOULDER_TILT;
882
- target.shoulderL -= heldRoll * SHOULDER_TILT;
883
-
884
- // 7. clamp, then blink wins outright over whatever the lids were doing
885
- for (const c of CHANNELS) {
886
- const r = RANGE[c];
887
- target[c] = clamp(target[c], r[0], r[1]);
888
- }
889
- if (il.blink > 0) {
890
- target.lidL = Math.max(target.lidL, il.blink);
891
- target.lidR = Math.max(target.lidR, il.blink);
892
- }
893
-
894
- if (overrides) for (const k in overrides) target[k] = overrides[k];
895
-
896
- // 8. smooth toward the target — this is where co-articulation happens
897
- for (const c of CHANNELS) cur[c] = approach(cur[c], target[c], TAU[c], dt);
898
-
899
- // 8b. The vestibulo-ocular reflex. Here and not in the gaze layer because
900
- // only here is the head that is actually drawn known: prosody, clips
901
- // and idle all land after gaze.
902
- if (vor) {
903
- Object.assign(shown, cur);
904
- if (!overrides || overrides.pupilX === undefined) {
905
- shown.pupilX = clamp(reflexX(cur.pupilX), RANGE.pupilX[0], RANGE.pupilX[1]);
906
- }
907
- if (!overrides || overrides.pupilY === undefined) {
908
- shown.pupilY = clamp(reflexY(cur.pupilY), RANGE.pupilY[0], RANGE.pupilY[1]);
909
- }
910
- }
911
-
912
- // 9. First-class hand control. The semantic frame is generated here, above
913
- // every renderer, so SVG, WebGL, and video rigs receive exactly the same
914
- // gesture/progress information. A handless rig simply ignores `frame.hand`.
915
- const handFrame = updateHandAction(elapsed * 1000);
916
- rig.apply(avatarFrame(shown, handFrame || undefined));
917
- }
918
-
919
- const REST_SHAPE = shapeFor(SILENT, 1);
920
-
921
- // How much of a sustained head turn the trunk takes up. Well under 1: people
922
- // under-rotate the head and then under-rotate the trunk again behind it, and
923
- // at a head-and-shoulders crop the trunk's share is the part you register
924
- // without noticing. A rig may pass its own `trunkFollow`: on a mesh head whose
925
- // neck twists, the trunk's sway is most of what moves the neck's outline, so
926
- // the share that reads as a body on a line face reads there as the neck
927
- // sliding.
928
- const TRUNK_FOLLOW = opts.trunkFollow ?? 0.45;
929
-
930
- // The channels speech owns outright — exactly the params.js mouth group
931
- // (mouth corners stay free: a clip may smile over a sentence).
932
- const MOUTH_LOCK = new Set(GROUPS.mouth);
933
-
934
- /** The shoulders' share of the attentive posture: two thirds of the lean the
935
- * engagement layer spends — the shoulders come up with it, they do not lead it. */
936
- const ENGAGE_SHOULDER = 0.10;
937
-
938
- // What survives of the resting/emotion smile while speech owns the mouth.
939
- // A smile held static through a sentence is discounted as insincere, and
940
- // corners riding every open viseme read as laughing through the words: warmth
941
- // must be episodic (research-perception.md §3). Full warmth returns the moment
942
- // the track ends, which is that onset/offset.
943
- const SPEAK_SMILE_RETAIN = 0.35;
944
-
945
- // Between the user's turn and the reply the server's claim can change
946
- // several times a second — THINKING, a tool's WORKING, THINKING again, a
947
- // grace timer's CANT_HEAR — and every change used to retarget the eyes and
948
- // restart the state's looks: one fast tool call was away, down at the
949
- // screen, back to the user and away again inside a second, a blink at each.
950
- // Among these states the eyes change over only once the new one has held
951
- // this long, and a claim that returns before then moves nothing. The pose
952
- // still changes at once — it is the eyes that make a flicker visible.
953
- // SPEAKING and LISTENING are never held back: those are the floor.
954
- const GAP_STATES = new Set(['THINKING', 'WORKING', 'CANT_HEAR']);
955
- const GAP_SETTLE = 0.5;
956
-
957
- // A dart leaves at up to 40° off the line of the look it is inside, and
958
- // lifts the brows for half a second — a thought arriving, not a stare.
959
- const DART_TURN = 0.7;
960
- const DART_BROW = 0.5;
961
-
962
- /** A look away for the return leg of a glance. `stick` of the time it keeps
963
- * the side the last one took, spot and all — the side a person thinks
964
- * toward is theirs, and it outlasts one thinking pause. Otherwise any look
965
- * but the one just taken: a thinker who goes back to the same patch of
966
- * wall every time *without* meaning to is a loop. */
967
- function pickBack(xs, stick = 0) {
968
- const side = (n) => Math.sign(gaze.targets[n].px);
969
- let g;
970
- if (lastBack && Math.random() < stick) {
971
- const same = xs.filter((n) => side(n) === side(lastBack));
972
- g = pick(same.length ? same : xs);
973
- } else {
974
- g = pick(xs);
975
- for (let i = 0; i < 4 && g === lastBack; i++) g = pick(xs);
976
- }
977
- lastBack = g;
978
- return g;
979
- }
980
-
981
- /**
982
- * One fixation of a reading scan: a step along the line, or at its end a
983
- * sweep back to the start of the next, or at the end of the block a pause
984
- * on the result. That is what busy-at-a-screen looks like from the other
985
- * side of a webcam — an irregular left-to-right march with returns, which
986
- * a random drift about a point is not. Steps are held, like a reader's
987
- * fixations, and the head sits out all of it (DRIFT_HEAD in gaze.js).
988
- */
989
- function readStep(r) {
990
- const n = ([a, b]) => a + ((Math.random() * (b - a + 1)) | 0);
991
- if (readCol < readCols) {
992
- readCol++;
993
- driftAt = elapsed + rand(r.fix);
994
- } else {
995
- readCol = 0;
996
- readCols = n(r.steps);
997
- if (++readRow >= readRows) {
998
- readRow = 0;
999
- readRows = n(r.lines);
1000
- driftAt = elapsed + rand(r.pause);
1001
- } else {
1002
- driftAt = elapsed + rand(r.fix) * 1.4;
1003
- }
1004
- }
1005
- gaze.drift.x = (readCol / readCols - 0.5) * r.width * saccadeGain;
1006
- gaze.drift.y = (readRow - (readRows - 1) / 2) * r.line * saccadeGain;
1007
- }
1008
-
1009
- // --- API ------------------------------------------------------------------
1010
-
1011
- function setState(name, o = {}) {
1012
- if (!states[name]) throw new Error(`unknown state: ${name}`);
1013
- const changed = name !== stateName;
1014
- stateName = name;
1015
- const st = states[name];
1016
- if (o.emotion !== undefined) emotion = o.emotion;
1017
- else if (changed) emotion = st.emotion;
1018
- if (o.intensity !== undefined) emotionAmt = o.intensity;
1019
- idle.setProfile(st.idle);
1020
- // SVG's desaturation filter is a legacy renderer detail. A generic rig
1021
- // receives the same state pose and may express degradation its own way.
1022
- if (face) {
1023
- face.svg.style.filter = st.filter || '';
1024
- face.svg.style.transition = 'filter .5s ease';
1025
- }
1026
- // The eyes wait out a flicker between gap states (GAP_SETTLE), and a
1027
- // repeat of the state still settling leaves it settling.
1028
- const deferrable = !o.gaze && !o.keepGaze && GAP_STATES.has(gazeState) && GAP_STATES.has(name);
1029
- if (deferrable && changed) settleAt = name === gazeState ? 0 : elapsed + GAP_SETTLE;
1030
- else if (!(deferrable && settleAt)) { settleAt = 0; enterGaze(name, o, changed); }
1031
- // The floor has come back to the user: the face receives it (prosody.js
1032
- // `listen`). It is a state change rather than a VAD event on purpose —
1033
- // what is being welcomed is the turn, and the server is the one that knows
1034
- // a turn has changed hands.
1035
- if (changed && name === 'LISTENING') prosody.listen();
1036
- if (changed) emit('state', name);
1037
- return api;
1038
- }
1039
-
1040
- // Whether the eyes were last aimed by somebody who meant it — the public
1041
- // `setGaze`, or a performance's `gaze` verb — rather than by the state's own
1042
- // schedule. It is the one thing that outranks a state's `gaze`, and it is
1043
- // why `speak()` can take the eyes back without overriding a caller.
1044
- let gazeExplicit = false;
1045
-
1046
- /** The state `name` takes the gaze: its target and its schedules. */
1047
- function enterGaze(name, o, blink) {
1048
- gazeState = name;
1049
- const st = states[name];
1050
- const gl = st.glance;
1051
- // A state with an `opening` enters as though its check-in on the user is
1052
- // already under way, and leaves it when that runs out.
1053
- if (!o.keepGaze) { setGaze(o.gaze || (gl && gl.opening ? gl.to : st.gaze)); gazeExplicit = false; }
1054
- // Arm every scheduler fresh, so entering a state never fires a timestamp
1055
- // left over from the last one — the wander in particular, which used to
1056
- // pick a new target on the first frame and override the state's own gaze.
1057
- glanceUntil = gl && gl.opening && !o.gaze && !o.keepGaze ? elapsed + rand(gl.opening) : 0;
1058
- glanceAt = elapsed + (gl ? rand(gl.every) : 0);
1059
- wanderAt = elapsed + (st.wander ? rand(st.wander.every) : 0);
1060
- driftAt = elapsed + (st.read ? rand(st.read.fix) : 0);
1061
- readCol = readCols = readRow = readRows = 0;
1062
- dartAt = dartBrowUntil = 0;
1063
- dart.x = dart.y = 0;
1064
- if (blink) idle.blink(true);
1065
- }
1066
-
1067
- function setEmotion(name, intensity = 1) { emotion = name; emotionAmt = intensity; return api; }
1068
-
1069
- /** @param {string} name @param {{x:number,y:number}} [custom] normalized -1..1
1070
- * @param {boolean} [blink] the mixer's own say over the evoked blink; see GazeLayer.set */
1071
- function setGaze(name, custom, blink) {
1072
- gazeName = GAZE_TARGETS[name] ? name : 'USER';
1073
- gazeCustom = custom || null;
1074
- applyGaze(blink);
1075
- return api;
1076
- }
1077
-
1078
- /**
1079
- * @param {object} o
1080
- * @param {Array<{t:number,v:string,i?:number}>} o.cues
1081
- * @param {HTMLMediaElement} [o.audio] preferred clock source
1082
- * @param {() => number} [o.clock] custom ms clock, if you drive audio yourself
1083
- */
1084
- function speak(o = {}) {
1085
- // Speech owns the mouth in the mixer, but a server action still gets to
1086
- // complete its physical landing on the other channels. Do not cancel it
1087
- // here: a hand or head cannot disappear simply because playout began.
1088
- // A new clock is a new turn. The same clock handed back is the accurate
1089
- // leg rewriting this one (AvatarClient re-speaks a splice), and that must
1090
- // not roll the turn-start look a second time.
1091
- const newTurn = !o.clock || o.clock !== speakClock;
1092
- speakStart = performance.now();
1093
- speakClock = o.clock
1094
- ? o.clock
1095
- : o.audio
1096
- ? () => o.audio.currentTime * 1000
1097
- : () => performance.now() - speakStart;
1098
- speech.start(o.cues || [], speakClock);
1099
- prosody.reset(newTurn);
1100
- // The eyes come back to the user when the audio starts: a reply arriving
1101
- // while THINKING was looking away would otherwise spend its whole turn
1102
- // aimed off the user, because nothing in SPEAKING retargets. `keepGaze` is
1103
- // for the one caller that means it — a performance that aimed the eyes with
1104
- // its own `gaze` verb keeps them, an instruction and not a leftover.
1105
- if (stateName !== 'SPEAKING') setState('SPEAKING', { keepGaze: gazeExplicit });
1106
- if (o.audio && o.audio.paused) o.audio.play().catch(() => {});
1107
- return api;
1108
- }
1109
-
1110
- /**
1111
- * "The user may be checking whether I am with them — hold their eyes."
1112
- *
1113
- * This is the widget's half of the **gaze window**. In face-to-face talk a
1114
- * speaker periodically looks at the listener, mutual gaze is established, the
1115
- * listener responds inside that window, and the speaker looks away again
1116
- * (Bavelas, Coates & Johnson 2002).
1117
- *
1118
- * We cannot see the user, so we cannot observe the window opening. What a
1119
- * caller *can* do is name the moments that co-occur with it — a mid-turn
1120
- * pause, a tag question ("...right?", "you know?"), a completed clause with
1121
- * the turn analyzer's completion probability high, the user answering a
1122
- * question the bot asked. For `ms` the face stops averting and holds the
1123
- * user, which is the prerequisite for any response to be *seen*.
1124
- *
1125
- * @param {number} [ms=1200] how long to hold. Binetti (N=498) puts preferred
1126
- * mutual gaze at 3295 ± 706 ms, so this is a fraction of the ceiling.
1127
- */
1128
- function attend(ms = 1200) {
1129
- attendUntil = Math.max(attendUntil, elapsed + ms / 1000);
1130
- return api;
1131
- }
1132
-
1133
- function pushCues(cues) { speech.push(cues); return api; }
1134
-
1135
- function stopSpeaking() { speech.stop(); return api; }
1136
-
1137
- function interject(id) {
1138
- const c = INTERNAL_CLIPS[id];
1139
- if (!c) throw new Error(`unknown interjection: ${id}`);
1140
- clip.play(c, c.audioEl);
1141
- return api;
1142
- }
1143
-
1144
- /**
1145
- * A hand gesture: the hand at the frame edge, plus the face half that makes it
1146
- * belong to somebody.
1147
- *
1148
- * The face half is not a convenience — a hand rising to the jaw over a head
1149
- * and shoulders sitting perfectly still is a cut-out, not a gesture. Each
1150
- * entry in HAND_GESTURES names the matching semantic face action; firing it here is the
1151
- * library composing two authored things, not the client inventing motion.
1152
- *
1153
- * On an avatar mounted with `hand: false` this internal helper degrades to
1154
- * the face action alone.
1155
- */
1156
- function gesture(id) {
1157
- const def = HAND_GESTURES[id];
1158
- if (!def) throw new Error(`unknown hand gesture: ${id}`);
1159
- if (hand) hand.play(id, elapsed * 1000);
1160
- if (def.face) interject(def.face);
1161
- return api;
1162
- }
1163
-
1164
- /** One self-completing server action. State continues to resolve underneath;
1165
- * face and hand layers queue their next movement so an in-flight physical
1166
- * gesture always gets to land. */
1167
- function action(wireId) {
1168
- // `ACKNOWLEDGE` is the whole backchannel family in one word, and which of
1169
- // them a face makes is a rendering decision the server is not holding the
1170
- // drawing for. The floor decides: a nod while the user still has it is a
1171
- // continuer, "go on"; once they have stopped it is a receipt, and a nod
1172
- // there reads as agreement with whatever they just said. Both are only ever
1173
- // this explicit action — nothing here starts one on its own.
1174
- const id = wireId === 'ACKNOWLEDGE'
1175
- ? (engagement.speaking ? 'ACK_NOD' : 'ACK_RECEIVE')
1176
- : wireId;
1177
- const handDef = HAND_GESTURES[id];
1178
- if (handDef) {
1179
- startHandAction(id, handDef);
1180
- if (handDef.face) {
1181
- const faceClip = actionShapes[handDef.face] || ACTIONS[handDef.face];
1182
- if (faceClip) clip.play(faceClip, faceClip.audioEl, { queue: true });
1183
- }
1184
- return api;
1185
- }
1186
- // Own renderings first, always. An avatar may *add* to what a server can
1187
- // ask for and may never redefine a core intent: `ACKNOWLEDGE` has to mean
1188
- // the same thing on every face or it is not a protocol. It may draw that
1189
- // meaning in its own shape (`actionShapes`), which is a rendering, not a
1190
- // redefinition — the id and the intent are fixed before this line.
1191
- //
1192
- // Unknown is a no-op, not a throw. The wire's action vocabulary is open, so
1193
- // a server asking this face for a motion it does not have is the expected
1194
- // case and not somebody's broken build — the same forward-compat rule an
1195
- // unknown `cmd` gets. This is also where a sequence used to arrive by its
1196
- // own method; one open vocabulary means one door.
1197
- const faceClip = actionShapes[id] || ACTIONS[id] || sequences[id];
1198
- if (!faceClip) return api;
1199
- clip.play(faceClip, faceClip.audioEl, { queue: true });
1200
- // The acknowledgements smile. Only these, and only ever because the
1201
- // server sent one: a smile the renderer timed for itself would be an
1202
- // acknowledgement nobody sent.
1203
- if (id === 'ACK_NOD' || id === 'ACK_RECEIVE') prosody.acknowledge();
1204
- // Now that warmth rides over clips, the reply's opening smile would
1205
- // otherwise carry on through the interrupted face.
1206
- if (id === 'RESPONSE_INTERRUPTED') prosody.cool();
1207
- return api;
1208
- }
1209
-
1210
- function startHandAction(id, def) {
1211
- const gesture = HAND_ACTION_TO_FRAME_GESTURE[id];
1212
- if (!gesture) return;
1213
- if (handAction) {
1214
- if (handAction.id !== id && !handQueue.some((item) => item.id === id)) {
1215
- handQueue.push({ id, def, gesture });
1216
- }
1217
- return;
1218
- }
1219
- handAction = { id, def, gesture, start: elapsed * 1000 };
1220
- }
1221
-
1222
- function updateHandAction(nowMs) {
1223
- if (!handAction) return null;
1224
- const progress = (nowMs - handAction.start) / handAction.def.dur;
1225
- if (progress >= 1) {
1226
- const done = handAction;
1227
- const next = handQueue.shift();
1228
- handAction = next ? { ...next, start: nowMs } : null;
1229
- emit('gestureEnd', done.id);
1230
- return handAction ? { gesture: handAction.gesture, progress: 0, side: handSide } : null;
1231
- }
1232
- return { gesture: handAction.gesture, progress: Math.max(0, progress), side: handSide };
1233
- }
1234
-
1235
- function setUserSpeaking(b) { engagement.setUserSpeaking(b); return api; }
1236
-
1237
- // What one action does when its moment comes. Enum validity is checked here,
1238
- // where the enums live: a bad value warns and is skipped, because one stale
1239
- // verb must never take down the performance around it.
1240
- function dispatchAction(a) {
1241
- try {
1242
- // `state` defaults to keepGaze — a timeline that wants the gaze moved
1243
- // says so with a `gaze` verb at the moment it means, which is how every
1244
- // composed turn in the demo already behaves.
1245
- if (a.do === 'state') setState(a.name, { keepGaze: a.keepGaze !== false });
1246
- else if (a.do === 'emotion') setEmotion(a.name, a.i ?? 1);
1247
- else if (a.do === 'gaze') { setGaze(a.name); gazeExplicit = true; }
1248
- else if (a.do === 'action') action(a.id);
1249
- } catch (e) {
1250
- console.warn(`perform: ${a.do} at ${a.t}ms skipped — ${e.message}`);
1251
- }
1252
- }
1253
-
1254
- let performGen = 0;
1255
-
1256
- /**
1257
- * @param {Array<{t: number, do: string}>} actions
1258
- * @param {{audio?: HTMLMediaElement, clock?: () => number,
1259
- * onAction?: (a: object) => void}} [o]
1260
- * @returns {{stop: () => void}} stop() cancels the *future* of this
1261
- * performance only: an in-flight interjection finishes, a live cue track
1262
- * is untouched, and 'performEnd' does not fire. A handle whose
1263
- * performance was already replaced by a newer perform() is a no-op.
1264
- */
1265
- function perform(actions, o = {}) {
1266
- const start = performance.now();
1267
- const clock = o.clock
1268
- ? o.clock
1269
- : o.audio
1270
- ? () => o.audio.currentTime * 1000
1271
- : () => performance.now() - start;
1272
- performTrack.onAction = (a) => { dispatchAction(a); if (o.onAction) o.onAction(a); };
1273
- const gen = ++performGen;
1274
- performTrack.start(actions, clock);
1275
- return { stop: () => { if (gen === performGen) performTrack.stop(); } };
1276
- }
1277
-
1278
- const api = {
1279
- setState, setEmotion, setGaze: (name, custom) => { gazeExplicit = true; return setGaze(name, custom); }, speak, pushCues, stopSpeaking, attend,
1280
- action, perform,
1281
- /** Which hand the character gestures with: +1 the viewer's right (its own
1282
- * left), -1 the other. Both are anatomically real — the thumb splays away
1283
- * from the body either way — so this is a character choice, not a fix. */
1284
- setHandSide: (d) => { handSide = d === -1 ? 'left' : 'right'; return api; },
1285
- setUserSpeaking,
1286
- /** Articulation gain: 1 is the VISEME_SHAPES table as authored. */
1287
- setMouthGain: (g) => { mouthGain = g; return api; },
1288
- get mouthGain() { return mouthGain; },
1289
- /** Gesture gain: scales every clip delta. 1 is the timelines as authored. */
1290
- setGestureGain: (g) => { gestureGain = g; return api; },
1291
- get gestureGain() { return gestureGain; },
1292
- /** Idle body-motion gain: 1 is the liveness layer as authored, 0 freezes it. */
1293
- setMotionGain: (g) => { idle.gain = g; return api; },
1294
- get motionGain() { return idle.gain; },
1295
- blink: () => { idle.blink(); return api; },
1296
- /** Advance one frame by hand. Only meaningful under `{manual: true}`;
1297
- * fixed-dt stepping is what makes a motion render reproducible. */
1298
- step: (dt) => { elapsed += dt; step(dt, dt * 1000); return api; },
1299
- /** Direct parameter injection — for tuning UIs, not production. */
1300
- setOverrides: (o) => { overrides = o; return api; },
1301
- on: (ev, fn) => { (listeners[ev] || (listeners[ev] = [])).push(fn); return api; },
1302
- get state() { return stateName; },
1303
- get emotion() { return emotion; },
1304
- get gaze() { return gazeName; },
1305
- get speaking() { return speech.playing; },
1306
- get performing() { return performTrack.playing; },
1307
- get clip() { return clip.id; },
1308
- /** Semantic hand gesture in flight, independent of renderer capability. */
1309
- get gesturing() { return handAction ? handAction.id : null; },
1310
- get params() { return shown; },
1311
- get userSpeaking() { return engagement.speaking; },
1312
- // Legacy SVG inspection fields. New AvatarRig implementations should not
1313
- // rely on or provide them; the renderer-neutral contract is apply/destroy.
1314
- svg: face?.svg || null,
1315
- meta: meta || null,
1316
- /** The mounted rig's palette, merged with any `opts.theme` overrides. A
1317
- * host that has to paint anything *around* the widget — a tile margin, a
1318
- * page behind a transparent mount — needs the same colours the drawing
1319
- * used, and guessing them per avatar is how the two drift apart. */
1320
- theme: face?.theme,
1321
- destroy() {
1322
- cancelAnimationFrame(raf);
1323
- rig.destroy();
1324
- },
1325
- };
1326
-
1327
- setState('IDLE');
1328
- if (!manual) raf = requestAnimationFrame(frame);
1329
- return api;
1330
- }
1331
-
1332
- // The parameter space itself. A renderer-neutral rig has to answer "what is
1333
- // this channel's neutral value" before it can map the channel onto whatever it
1334
- // controls — a morph target's influence is `(pose - rest) / (1 - rest)`, and a
1335
- // rig that hard-codes those rests has quietly forked `params.js`.
1336
- export { REST, CHANNELS, RANGE, TAU, clamp, approach, makeParams } from './params.js';
1337
- // The smoothing law itself, for a tool that steps the rig by hand. A
1338
- // filmstrip that reimplemented `approach` would be measuring its own copy of
1339
- // the thing under test — and the smoothing between keyframes is what the face
1340
- // actually does (docs/internal-mixer.md § Smoothing).
1341
- // A pose with no mixer above it: `makeParams(overrides)` fills the rests,
1342
- // `avatarFrame` wraps it, `createSvgRig(face).apply` draws it. That is the whole
1343
- // path an instrument needs to hold a face at one named extreme — no clock, no
1344
- // client, no animation — and the reason it is exported is that a pose sheet
1345
- // that cannot reach it forks the channel rests instead.
1346
- export { avatarFrame, createSvgRig } from './rig.js';
1347
- export { ACTION_IDS, ACTIONS, attachAudio } from './interjections.js';
1348
- // The full authoring catalogue and the player that steps it. Not a server
1349
- // vocabulary — `ACTIONS` is that, and most of these clips are reachable only
1350
- // from inside the mixer. They are exported for the filmstrip instrument,
1351
- // which lays one clip out as frames and therefore has to drive a real
1352
- // `ClipPlayer` rather than re-sample its keys.
1353
- export { INTERNAL_CLIPS } from './interjections.js';
1354
- export { ClipPlayer } from './clips.js';
1355
- export { GAZE_NAMES, GAZE_TARGETS } from './gaze.js';
1356
- export { normalizeActions } from './perform.js';
1357
- export { checkHandFraming } from './hand.js';
1358
- export { EMOTION_NAMES, emotionPose } from './emotions.js';
1359
- // The mouth clock travels with the rest of it. Someone has to turn a cue array
1360
- // plus a clock into "which letter is on screen right now", every renderer needs
1361
- // exactly that, and none of them should write it twice — so it is a plain class
1362
- // to construct, not a contract to implement.
1363
- export {
1364
- VISEME_LETTERS, VISEME_SHAPES, VisemeTrack, shapeFor, JAW_OF_OPEN, SILENT,
1365
- normalizeCues, textToCues,
1366
- ARPABET_TO_VISEME, AZURE_VISEME_TO_LETTER, LEAD_MS,
1367
- } from './visemes.js';
1368
- // No THEME re-export: each face module owns its palette, and `api.theme` is
1369
- // the mounted avatar's. A single barrel THEME was one rig's palette wearing a
1370
- // public name — misleading the moment that rig stopped being the default.