@voqalize/avatar 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +65 -154
  2. package/dist/index.d.ts +37 -26
  3. package/dist/index.js +48 -25
  4. package/dist/pin.d.ts +2 -0
  5. package/dist/pin.js +3 -0
  6. package/dist/react.d.ts +21 -6
  7. package/dist/react.js +24 -6
  8. package/package.json +8 -67
  9. package/LICENSE-APACHE-2.0 +0 -201
  10. package/LICENSE-CC-BY-4.0 +0 -396
  11. package/assets/README.md +0 -60
  12. package/assets/tanvi.glb +0 -0
  13. package/assets/tanya.glb +0 -0
  14. package/assets/tara.glb +0 -0
  15. package/assets/tess.glb +0 -0
  16. package/assets/tushar.glb +0 -0
  17. package/client/Avatar.tsx +0 -57
  18. package/client/AvatarClient.ts +0 -818
  19. package/client/createAvatar.ts +0 -177
  20. package/client/index.ts +0 -44
  21. package/client/internal.ts +0 -132
  22. package/client/playout.ts +0 -95
  23. package/client/react.ts +0 -13
  24. package/client/supports.ts +0 -27
  25. package/client/three/assets.ts +0 -26
  26. package/client/three/budgets.ts +0 -27
  27. package/client/three/character-rig.ts +0 -1746
  28. package/client/three/createCharacter.ts +0 -109
  29. package/client/three/holds.ts +0 -69
  30. package/client/three/internal.ts +0 -55
  31. package/client/three/motion-limits.json +0 -124
  32. package/client/three/sequences.ts +0 -281
  33. package/client/three/tanvi-asset.ts +0 -6
  34. package/client/three/tanvi.ts +0 -30
  35. package/client/three/tanya-asset.ts +0 -6
  36. package/client/three/tanya.ts +0 -29
  37. package/client/three/tara-asset.ts +0 -25
  38. package/client/three/tara.ts +0 -36
  39. package/client/three/tess-asset.ts +0 -6
  40. package/client/three/tess.ts +0 -30
  41. package/client/three/tushar-asset.ts +0 -6
  42. package/client/three/tushar.ts +0 -28
  43. package/client/types.ts +0 -235
  44. package/client/useAvatar.ts +0 -74
  45. package/dist/Avatar.d.ts +0 -44
  46. package/dist/Avatar.d.ts.map +0 -1
  47. package/dist/Avatar.js +0 -7
  48. package/dist/Avatar.js.map +0 -1
  49. package/dist/AvatarClient.d.ts +0 -274
  50. package/dist/AvatarClient.d.ts.map +0 -1
  51. package/dist/AvatarClient.js +0 -712
  52. package/dist/AvatarClient.js.map +0 -1
  53. package/dist/createAvatar.d.ts +0 -135
  54. package/dist/createAvatar.d.ts.map +0 -1
  55. package/dist/createAvatar.js +0 -75
  56. package/dist/createAvatar.js.map +0 -1
  57. package/dist/index.d.ts.map +0 -1
  58. package/dist/index.js.map +0 -1
  59. package/dist/internal.d.ts +0 -34
  60. package/dist/internal.d.ts.map +0 -1
  61. package/dist/internal.js +0 -56
  62. package/dist/internal.js.map +0 -1
  63. package/dist/playout.d.ts +0 -42
  64. package/dist/playout.d.ts.map +0 -1
  65. package/dist/playout.js +0 -85
  66. package/dist/playout.js.map +0 -1
  67. package/dist/react.d.ts.map +0 -1
  68. package/dist/react.js.map +0 -1
  69. package/dist/supports.d.ts +0 -19
  70. package/dist/supports.d.ts.map +0 -1
  71. package/dist/supports.js +0 -25
  72. package/dist/supports.js.map +0 -1
  73. package/dist/three/assets.d.ts +0 -10
  74. package/dist/three/assets.d.ts.map +0 -1
  75. package/dist/three/assets.js +0 -23
  76. package/dist/three/assets.js.map +0 -1
  77. package/dist/three/budgets.d.ts +0 -17
  78. package/dist/three/budgets.d.ts.map +0 -1
  79. package/dist/three/budgets.js +0 -21
  80. package/dist/three/budgets.js.map +0 -1
  81. package/dist/three/character-rig.d.ts +0 -366
  82. package/dist/three/character-rig.d.ts.map +0 -1
  83. package/dist/three/character-rig.js +0 -1644
  84. package/dist/three/character-rig.js.map +0 -1
  85. package/dist/three/createCharacter.d.ts +0 -60
  86. package/dist/three/createCharacter.d.ts.map +0 -1
  87. package/dist/three/createCharacter.js +0 -84
  88. package/dist/three/createCharacter.js.map +0 -1
  89. package/dist/three/holds.d.ts +0 -38
  90. package/dist/three/holds.d.ts.map +0 -1
  91. package/dist/three/holds.js +0 -61
  92. package/dist/three/holds.js.map +0 -1
  93. package/dist/three/internal.d.ts +0 -52
  94. package/dist/three/internal.d.ts.map +0 -1
  95. package/dist/three/internal.js +0 -49
  96. package/dist/three/internal.js.map +0 -1
  97. package/dist/three/motion-limits.json +0 -124
  98. package/dist/three/sequences.d.ts +0 -71
  99. package/dist/three/sequences.d.ts.map +0 -1
  100. package/dist/three/sequences.js +0 -262
  101. package/dist/three/sequences.js.map +0 -1
  102. package/dist/three/tanvi-asset.d.ts +0 -7
  103. package/dist/three/tanvi-asset.d.ts.map +0 -1
  104. package/dist/three/tanvi-asset.js +0 -7
  105. package/dist/three/tanvi-asset.js.map +0 -1
  106. package/dist/three/tanvi.d.ts +0 -24
  107. package/dist/three/tanvi.d.ts.map +0 -1
  108. package/dist/three/tanvi.js +0 -24
  109. package/dist/three/tanvi.js.map +0 -1
  110. package/dist/three/tanya-asset.d.ts +0 -7
  111. package/dist/three/tanya-asset.d.ts.map +0 -1
  112. package/dist/three/tanya-asset.js +0 -7
  113. package/dist/three/tanya-asset.js.map +0 -1
  114. package/dist/three/tanya.d.ts +0 -23
  115. package/dist/three/tanya.d.ts.map +0 -1
  116. package/dist/three/tanya.js +0 -23
  117. package/dist/three/tanya.js.map +0 -1
  118. package/dist/three/tara-asset.d.ts +0 -26
  119. package/dist/three/tara-asset.d.ts.map +0 -1
  120. package/dist/three/tara-asset.js +0 -26
  121. package/dist/three/tara-asset.js.map +0 -1
  122. package/dist/three/tara.d.ts +0 -30
  123. package/dist/three/tara.d.ts.map +0 -1
  124. package/dist/three/tara.js +0 -30
  125. package/dist/three/tara.js.map +0 -1
  126. package/dist/three/tess-asset.d.ts +0 -7
  127. package/dist/three/tess-asset.d.ts.map +0 -1
  128. package/dist/three/tess-asset.js +0 -7
  129. package/dist/three/tess-asset.js.map +0 -1
  130. package/dist/three/tess.d.ts +0 -24
  131. package/dist/three/tess.d.ts.map +0 -1
  132. package/dist/three/tess.js +0 -24
  133. package/dist/three/tess.js.map +0 -1
  134. package/dist/three/tushar-asset.d.ts +0 -7
  135. package/dist/three/tushar-asset.d.ts.map +0 -1
  136. package/dist/three/tushar-asset.js +0 -7
  137. package/dist/three/tushar-asset.js.map +0 -1
  138. package/dist/three/tushar.d.ts +0 -22
  139. package/dist/three/tushar.d.ts.map +0 -1
  140. package/dist/three/tushar.js +0 -22
  141. package/dist/three/tushar.js.map +0 -1
  142. package/dist/types.d.ts +0 -149
  143. package/dist/types.d.ts.map +0 -1
  144. package/dist/types.js +0 -140
  145. package/dist/types.js.map +0 -1
  146. package/dist/useAvatar.d.ts +0 -39
  147. package/dist/useAvatar.d.ts.map +0 -1
  148. package/dist/useAvatar.js +0 -35
  149. package/dist/useAvatar.js.map +0 -1
  150. package/src/avatar.d.ts +0 -436
  151. package/src/avatar.js +0 -1378
  152. package/src/behavior.d.ts +0 -42
  153. package/src/behavior.js +0 -114
  154. package/src/camera.js +0 -29
  155. package/src/clips.js +0 -155
  156. package/src/conformance.js +0 -119
  157. package/src/emotions.js +0 -58
  158. package/src/face-core.js +0 -180
  159. package/src/face-myna.d.ts +0 -7
  160. package/src/face-myna.js +0 -786
  161. package/src/face-peep-control-plane.js +0 -167
  162. package/src/face-peep.d.ts +0 -7
  163. package/src/face-peep.js +0 -925
  164. package/src/face-wren.d.ts +0 -7
  165. package/src/face-wren.js +0 -468
  166. package/src/faces.d.ts +0 -12
  167. package/src/faces.js +0 -53
  168. package/src/gaze.js +0 -504
  169. package/src/hand.js +0 -709
  170. package/src/head.js +0 -134
  171. package/src/idle.js +0 -668
  172. package/src/interjections.js +0 -747
  173. package/src/line-art.js +0 -137
  174. package/src/params.js +0 -180
  175. package/src/perform.js +0 -107
  176. package/src/prosody.js +0 -741
  177. package/src/rig.d.ts +0 -31
  178. package/src/rig.js +0 -32
  179. package/src/speech-timing.js +0 -51
  180. package/src/visemes.js +0 -410
package/src/avatar.js DELETED
@@ -1,1378 +0,0 @@
1
- /**
2
- * The mixer — a programmable talking head.
3
- *
4
- * Not the public surface: that is `createAvatar({ mount, client })` in
5
- * `packages/avatar/client/createAvatar.ts`, and this is what it drives. Everything below is
6
- * addressed by our own runtime, our tooling, and an avatar author who chose to
7
- * build on the SVG renderer (`@voqalize/avatar/internal`, no semver promise).
8
- *
9
- * Per frame the mixer runs a fixed layer order. Earlier layers are overwritten
10
- * by later ones on the channels they touch; the gesture and idle layers are
11
- * additive so they compose rather than fight.
12
- *
13
- * base pose (state + emotion) -> gaze -> visemes -> clip -> idle
14
- */
15
-
16
- import { REST, CHANNELS, TAU, RANGE, GROUPS, clamp, approach } from './params.js';
17
- import { EMOTIONS, emotionPose } from './emotions.js';
18
- import { GazeLayer, GAZE_TARGETS, AVERSION } from './gaze.js';
19
- import { IdleLayer, ListeningEngine } from './idle.js';
20
- import { ClipPlayer } from './clips.js';
21
- import { ACTIONS, INTERNAL_CLIPS } from './interjections.js';
22
- import { VisemeTrack, shapeFor, SILENT } from './visemes.js';
23
- import { LIP_CONTACT_AIM } from './speech-timing.js';
24
- import { SpeechProsody, UNCALIBRATED_HEAD_GAIN } from './prosody.js';
25
- import { HEAD_AXES, soften } from './head.js';
26
- import { PerformTrack } from './perform.js';
27
- import { createHand, HAND_GESTURES, HAND_ACTION_TO_FRAME_GESTURE } from './hand.js';
28
- import { avatarFrame, createSvgRig } from './rig.js';
29
-
30
- const rand = ([a, b]) => a + Math.random() * (b - a);
31
- const pick = (xs) => xs[(Math.random() * xs.length) | 0];
32
-
33
- // Each state's `idle` is a profile for the liveness layer (see DEFAULT_PROFILE
34
- // in idle.js). Blink gaps come from docs/research-biomechanics.md §5: the rate
35
- // alone separates listening (~16/min) from thinking (~25/min) from visually
36
- // busy (~9/min), and it is the cheapest state signal the rig has. The gap is
37
- // the budget for every blink, paired ones included (about one in six), so it
38
- // sits a little longer than 60/rate.
39
- export const STATES = {
40
- // Idle means present and relaxed, not attending. It is also the face before
41
- // a call connects — the first one anyone sees — so it rests on the user and
42
- // leaves for soft, unhurried looks (AVERSION.IDLE) rather than parking the
43
- // eyes somewhere. It used to open on AWAY_THINKING and wander between it,
44
- // AWAY_RIGHT and NOTES from the first frame: a face that loads looking up
45
- // and to the left, at nothing, reads as broken rather than idle. What keeps
46
- // it apart from LISTENING is the longer, lazier looks, the slower scan and
47
- // the missing engagement lean, not refusing to look at the user.
48
- IDLE: { gaze: 'USER', emotion: 'neutral', engagement: false,
49
- aversion: 'IDLE', scan: [0.8, 2.0],
50
- idle: { sway: 0.72, blinkGap: [3.6, 5.2] } },
51
- // `aversion` is why this state does not stare. Continuous eye contact is not
52
- // the attentive pose it looks like — it is a demand for more talk (Rossano)
53
- // and it measures as *tense*, not attentive (Wang & Gratch). See AVERSION in
54
- // gaze.js for the numbers; the mixer holds it off near a turn boundary.
55
- // At 3.2-4.4 s the blink timer measured 20-23/min.
56
- LISTENING: { gaze: 'USER', emotion: 'neutral', engagement: true,
57
- aversion: 'LISTEN',
58
- idle: { sway: 1.0, blinkGap: [3.6, 4.8] },
59
- pose: { browRaiseL: 0.06, browRaiseR: 0.06, lidL: -0.04, lidR: -0.04 } },
60
- // Faster, shallower breath is the measured cognitive-load signature, and the
61
- // occasional dead-still hold is the strongest "working on it" cue a rig this
62
- // simple can make — deliberate stillness, not more motion. The look away is
63
- // Andrist's cognitive aversion (§4.2): one look held ~3.5 s (SD 1.26) that
64
- // drifts a little where it sits, a check-in with the user, then the next.
65
- //
66
- // In a call this state mostly lasts under two seconds — the gap between the
67
- // user finishing and the reply starting — so its opening is what gets seen,
68
- // and the opening used to be wrong. The eyes left the instant the user
69
- // stopped, down and to the left, every turn: a listener who drops their
70
- // eyes the moment you finish reads as ashamed or done with you. Now the
71
- // eyes hold the user for `opening` first (a person takes in the end of
72
- // what was said before they go to think), and a fast reply never looks
73
- // away at all. The look is mostly up-and-aside or level-aside: that is the
74
- // one audiences read as thinking, whatever the measured split, and down
75
- // is kept to a fifth of looks because on a face this real it reads as
76
- // downcast. The pose takes back `thoughtful`'s lid drop: a thinking face is
77
- // alert, and the two together measured past the 0.15 that reads drowsy.
78
- THINKING: { gaze: 'AWAY_SIDE', emotion: 'thoughtful', engagement: false,
79
- // Fixational jumps rare and small: a thinker's eyes rest
80
- // where they land. At the default gap the look jittered
81
- // about its spot every second, and with a drift on top
82
- // the eyes never held anything — searching the wall, not
83
- // thinking.
84
- scan: [1.2, 2.6, 0.35],
85
- idle: { sway: 0.7, blinkGap: [2.6, 3.2], slowBlink: 0.3,
86
- breathRate: 1.18, breathAmp: 0.7,
87
- hold: { every: [4.5, 9.0], dur: [0.8, 1.5] } },
88
- // `stick`: people have a side they look to when they
89
- // think and keep to it (Day; Kinsbourne's lateral eye
90
- // movements) — one who alternates sides on every look is
91
- // scanning the room. `dart`: now and then, once in a
92
- // look, the eyes move on to a second spot with a flick of
93
- // the brows, which is the thought moving; the random
94
- // drift this replaces never stopped and read as roving.
95
- // `blinkTo`: the look back to the user always blinks —
96
- // the re-engagement is the beat to see.
97
- glance: { to: 'USER', opening: [0.35, 0.7], every: [2.2, 4.4], hold: [0.8, 1.3],
98
- back: ['AWAY_THINKING', 'AWAY_THINKING', 'AWAY_THINKING',
99
- 'AWAY_SIDE', 'AWAY_SIDE', 'AWAY_SIDE',
100
- 'AWAY_RIGHT', 'AWAY_RIGHT', 'AWAY_DOWN', 'AWAY_DOWN'],
101
- stick: 0.55, dart: { p: 0.4, mag: 0.08, brow: 0.12 }, blinkTo: true },
102
- // **The chin comes up**: thinking is the one stretch of
103
- // a call where the avatar owes the user feedback and
104
- // has no mouth to give it with, the eyes being off the
105
- // user by design. A head tipping back as the gaze
106
- // leaves is the swing-up of research-biomechanics.md
107
- // § 3.3. It rides the looks that go up, as the head's
108
- // *share* of them: held as a pose it lifted the chin on
109
- // the level and downward looks too, so the user was
110
- // looked at down the nose.
111
- pose: { lidL: -0.10, lidR: -0.10 } },
112
- // Eyes on the user for the whole turn, and no `aversion`: gaze.js has why a
113
- // speaker's measured looks away are not this rig's to render. What moves
114
- // while it talks is the head it holds per phrase and the trunk under it
115
- // (prosody.js), never the eyes leaving the user.
116
- SPEAKING: { gaze: 'USER', emotion: 'neutral',
117
- idle: { sway: 0.55 }, engagement: false },
118
- REVIEWING_SCREEN: { gaze: 'SCREEN_CENTER', emotion: 'thoughtful', engagement: false,
119
- idle: { sway: 0.8, blinkGap: [4.0, 6.5] },
120
- wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_RIGHT', 'SCREEN_TOP', 'SCREEN_WORK'],
121
- every: [1.8, 5.0] } },
122
- // The head cant is the state's signature cue, and it has to clear the roll
123
- // multiplier to exist at all: 0.05 here renders as 0.3° of rotation, which
124
- // is no tilt whatever the number says. 0.30 renders ~1.7° — visible at tile
125
- // size, still gentle.
126
- WAITING_FOR_USER: { gaze: 'USER', emotion: 'encouraging', engagement: true,
127
- idle: { sway: 1.0, blinkGap: [3.1, 4.2] },
128
- pose: { headRoll: 0.30, browRaiseL: 0.16, browRaiseR: 0.12 } },
129
- // Straining to hear. The one state where the amplitude constraint yields,
130
- // because the lean IS the message: torsoLean well past what the engagement
131
- // layer ever spends, head cheated aside on USER_EAR so an ear
132
- // favors the speaker while the eyes hold contact, and a concentration
133
- // squint with knit brows. Stillness does the rest — straining people
134
- // freeze — so holds are frequent and there is no engagement lean: you don't
135
- // nod along to what you can't hear. Server sends it on soft/low-SNR user
136
- // audio, typically followed by SORRY or a "could you repeat" utterance.
137
- CANT_HEAR: {
138
- gaze: 'USER_EAR', emotion: 'neutral', engagement: false,
139
- idle: { sway: 0.5, blinkGap: [4.5, 6.5],
140
- hold: { every: [2.5, 5.5], dur: [1.0, 1.8] } },
141
- // A minimal line face swallows small deltas — the ink moves whole units
142
- // or it doesn't move. These values are set from the contact sheet's
143
- // extremes row, not from what a fleshed rig would need: brows DOWN
144
- // (corrugator effort, not the browInner worry-lift), a real squint, and
145
- // the resting smile pressed flat — nobody smiles while straining to hear.
146
- // The squint is the lower lid's (AU7), with no upper lid dropped on top:
147
- // the two together narrowed a photographic eye from above and read as
148
- // drowsy, and a line face's squint is its lower lid already.
149
- pose: {
150
- torsoLean: 0.70, headPitch: 0.10,
151
- squintL: 0.75, squintR: 0.75,
152
- browRaiseL: -0.45, browRaiseR: -0.45, browInnerL: 0.15, browInnerR: 0.12,
153
- mouthPress: 0.45, mouthCornerL: -0.22, mouthCornerR: -0.22,
154
- },
155
- },
156
- // The user's microphone is closed, and the agent is the one that closed it.
157
- // Authored against CANT_HEAR as its exact inverse, because that contrast is
158
- // the whole read: straining leans *in* to get more of the user, this settles
159
- // *back* — nothing more is coming through and the avatar knows why. The eyes
160
- // stay on the user, because a deliberate hold is not inattention; what
161
- // carries it is the mouth, pressed shut and staying shut, and the slow,
162
- // unhurried blink of waiting rather than working. No engagement lean: you
163
- // cannot nod along to a channel you have muted yourself. No filter, ever —
164
- // DEGRADED and OFFLINE own "something is broken", and this is a decision.
165
- MUTED: {
166
- gaze: 'USER', emotion: 'neutral', engagement: false,
167
- idle: { sway: 0.45, blinkGap: [4.5, 7.0],
168
- hold: { every: [3.0, 6.0], dur: [0.9, 1.6] } },
169
- // Line-face scaled (see CANT_HEAR): peep's resting mouth is drawn smiling,
170
- // so a closed mouth has to be authored clearly past flat to read as closed
171
- // at all. Corners were -0.20 first, which rendered as a straight line and
172
- // at 130 px against LISTENING said nothing — the delta a viewer gets is
173
- // curvature, and flat is the halfway point of it, not the end. -0.34 is as
174
- // far as it goes before the hold starts reading as sulking. No squint and
175
- // no lid drop: one is straining, the other was asleep. browInner carries
176
- // the "one moment" without the worry lift.
177
- pose: {
178
- torsoLean: -0.28, headPitch: 0.04,
179
- browRaiseL: 0.08, browRaiseR: 0.06, browInnerL: 0.28, browInnerR: 0.22,
180
- mouthPress: 0.78, mouthCornerL: -0.34, mouthCornerR: -0.34,
181
- },
182
- },
183
- // --- application state ---------------------------------------------------
184
- // "Momentarily busy on the thing you asked for." No hands in frame, so the
185
- // whole read comes from cheap cues (docs/research-biomechanics.md §6.4,
186
- // recommendation 19): gaze down on one stable target with a reading scan,
187
- // blinks suppressed to task-focus rate (~9/min), shoulders slightly raised
188
- // and *held* with brief micro-freezes, and a glance back up to the user.
189
- // The glance is the tell that the user has not been forgotten; without it,
190
- // busy is just absent, and much more often than that it is fidgeting. There
191
- // is no shoulder rhythm: at 2.2 Hz it ran over the 1.5 Hz ceiling every
192
- // other motion here keeps, and held is what §6.4 describes. The trunk sits
193
- // out the glance — checking on someone is a look, not a turn toward them.
194
- WORKING: {
195
- // OWN_SCREEN: eyes down at the agent's own display, head nearly level.
196
- // This was SCREEN_WORK, which turned the head down and to the left and
197
- // held it there for the whole task, with a random drift too small to see
198
- // on it — a head turned away from the user, staring at a point. `read` is
199
- // what makes it busy instead: fixations stepping along a line, a sweep
200
- // back, the next line, a pause on the result (see readStep). The lids
201
- // take back most of the down look's follow so the eyes stay awake.
202
- gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
203
- scan: [0.9, 2.0, 0.12],
204
- // §6.4's ~9/min is a count of blinks: at the table's own 6-7.5 s gap it
205
- // measured 12-13/min, which is not focus.
206
- idle: { sway: 0.5, blinkGap: [6.8, 8.6], breathRate: 1.05, breathAmp: 0.8,
207
- hold: { every: [5.0, 9.0], dur: [0.6, 1.1] } },
208
- // The look up to check is brows-first and blinkless, the lids leading it
209
- // (their 18 ms against the eye's 32 ms); the blink comes on the way back
210
- // down, where the eyes re-engage with the work. A blink going up hid the
211
- // one moment the user is actually looked at.
212
- glance: { to: 'USER', every: [6, 11], hold: [0.8, 1.2], brow: 0.10,
213
- blinkTo: false, blinkBack: true },
214
- // Fewer, longer fixations than a page reader's. At 3-5 steps a quarter
215
- // second apart the scan measured a shift a second, and over a webcam that
216
- // is a jiggle, not reading.
217
- read: { width: 0.20, steps: [2, 3], fix: [0.35, 0.7], lines: [2, 4], line: 0.05,
218
- pause: [0.9, 1.8] },
219
- // The brows draw down and together — AU4, the one facial action that
220
- // reliably marks effortful concentration. Without it the face over a
221
- // reading scan is blank, and blank over a moving eye reads as bored.
222
- pose: { headPitch: 0.04, lidL: -0.08, lidR: -0.08,
223
- shoulderL: 0.06, shoulderR: 0.06,
224
- browRaiseL: -0.08, browRaiseR: -0.08, browInnerL: -0.10, browInnerR: -0.10 },
225
- },
226
- // The audio channel is broken and the agent is typing in the chat window to
227
- // communicate. The glance looks up and HOLDS 1.2–2 s, expectant, because the
228
- // chat (and the user's face) is now the only channel there is. A touch of
229
- // browInner carries the apology. Relation to DEGRADED is by semantics, not merger:
230
- // DEGRADED says "my feed is broken", TYPING_CHAT says "I'm working around
231
- // it" — a server will typically sequence DEGRADED → TYPING_CHAT.
232
- TYPING_CHAT: {
233
- gaze: 'SCREEN_WORK', emotion: 'neutral', engagement: false,
234
- idle: { sway: 0.6, blinkGap: [5.5, 7.0], breathRate: 1.05,
235
- rhythm: { amp: 0.055, freq: 2.5 } },
236
- glance: { to: 'USER', every: [3.2, 5.5], hold: [1.2, 2.0] },
237
- // Line-face scaled (see CANT_HEAR); the apology has to survive the rig's
238
- // baked resting smile, so the corners go clearly negative.
239
- pose: { headPitch: 0.10, lidL: -0.04, lidR: -0.04,
240
- shoulderL: 0.06, shoulderR: 0.06,
241
- browInnerL: 0.45, browInnerR: 0.38,
242
- mouthPress: 0.50, mouthCornerL: -0.28, mouthCornerR: -0.28 },
243
- },
244
- // Attention genuinely elsewhere. What separates this from WORKING is target
245
- // *stability* (§6.4): busy is one steady off-user target, distracted is
246
- // wandering ones, held long (aversion >3s), with no engagement lean — the
247
- // missing nod is as diagnostic as the look-away. Sway is looser than
248
- // LISTENING because attention is what was holding the body still. The
249
- // widget only looks away; deciding when to snap back is the server's call.
250
- DISTRACTED: {
251
- gaze: 'AWAY_RIGHT', emotion: 'neutral', engagement: false,
252
- idle: { sway: 1.15, blinkGap: [1.8, 4.2] },
253
- // Sideways and up, never steep-down: lateral is where real intimacy/
254
- // distraction aversions live, and a steep down target seals this rig's
255
- // eyes (see WORKING).
256
- wander: { targets: ['AWAY_RIGHT', 'AWAY_THINKING', 'SCREEN_LEFT', 'SCREEN_TOP'],
257
- every: [2.8, 6.8] },
258
- },
259
- // The buying-time move: hunting for a control on screen. Distinct from
260
- // REVIEWING_SCREEN by *hunt* quality — reading dwells (1.8–5 s) become
261
- // search saccades (0.8–2 s) with revisits (targets repeat in the wander
262
- // set), plus the idle layer's flick: the tiny "no, not this one" yaw
263
- // wiggle nobody makes while merely reading. Server semantics: a filler
264
- // while an async activity completes; the server exits it when done.
265
- SEARCHING_SCREEN: {
266
- gaze: 'SCREEN_CENTER', emotion: 'neutral', engagement: false,
267
- idle: { sway: 0.65, blinkGap: [5.0, 6.8], breathRate: 1.05,
268
- flick: { amp: 0.30, every: [3.5, 7.0] } },
269
- // The only screen state that never named its own scan, so it ran the
270
- // default [0.5, 1.6] at full amplitude: a second full-size jump landing on
271
- // top of a wander hop, which is most of what reads as snapping. A hunt's
272
- // refixations are faster and much smaller than a reader's — quick little
273
- // checks around the thing being looked at — so this is the other end of
274
- // the dial from WORKING's [0.9, 2.0, 0.12] rather than a copy of it.
275
- scan: [0.4, 1.1, 0.55],
276
- wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_TOP', 'SCREEN_WORK',
277
- 'SCREEN_RIGHT', 'SCREEN_CENTER', 'SCREEN_LEFT'],
278
- every: [0.8, 2.0] },
279
- // Line-face scaled (see CANT_HEAR). Note the corners: peep's REST mouth
280
- // is drawn smiling, so "not smiling" is a clearly negative net corner,
281
- // not zero.
282
- pose: { squintL: 0.40, squintR: 0.40, mouthPress: 0.65,
283
- mouthCornerL: -0.25, mouthCornerR: -0.25, browRaiseL: -0.26, browRaiseR: -0.20 },
284
- },
285
- // --- floor management ----------------------------------------------------
286
- // Turn-taking is the part of a voice call that goes wrong most often: the
287
- // user either talks over the agent or sits in silence waiting for a signal
288
- // that never comes. These are states rather than clips because the floor is a
289
- // condition and not an event — WANTS_IN in particular has to hold for as long
290
- // as it takes the other person to notice it.
291
- //
292
- // Each of these lifts the shoulders and parts the lips, because that is what an
293
- // inbreath looks like from outside, and an inbreath is the cue humans actually
294
- // use to predict that someone is about to speak. The head comes *up* rather
295
- // than down: a lowered head is deferential and reads as yielding.
296
- TAKING_FLOOR: {
297
- gaze: 'USER', emotion: 'neutral', idle: { sway: 0.6 }, engagement: false,
298
- pose: {
299
- browRaiseL: 0.26, browRaiseR: 0.22, lidL: -0.10, lidR: -0.10,
300
- headPitch: -0.10, torsoLean: 0.22, shoulderL: 0.30, shoulderR: 0.30,
301
- mouthOpen: 0.10, mouthPress: -0.10,
302
- },
303
- },
304
- // The one signal the rig had no way to give at all. An agent needs to be
305
- // able to say "I'd like to come in" without talking over the user, and
306
- // every part of this pose is doing that job: held still (idle is low on
307
- // purpose — stillness is what makes it read as intent rather than as fidget),
308
- // leaning in, lips apart and staying apart.
309
- WANTS_IN: {
310
- gaze: 'USER', emotion: 'neutral', idle: { sway: 0.45 }, engagement: false,
311
- pose: {
312
- browRaiseL: 0.42, browRaiseR: 0.38, lidL: -0.14, lidR: -0.14,
313
- headPitch: -0.14, torsoLean: 0.42, shoulderL: 0.45, shoulderR: 0.45,
314
- mouthOpen: 0.16, mouthWidth: 0.30, mouthPress: -0.14,
315
- },
316
- },
317
- // Interrupted mid-word. The mouth shutting is the whole message, and it has to
318
- // happen faster than anything else on the face — see YIELD_FLOOR, which is
319
- // what actually delivers the snap.
320
- YIELDED: {
321
- gaze: 'USER', emotion: 'neutral', idle: { sway: 0.9 }, engagement: false,
322
- pose: {
323
- browRaiseL: 0.10, browRaiseR: 0.08,
324
- torsoLean: -0.18, shoulderL: -0.12, shoulderR: -0.12,
325
- },
326
- },
327
-
328
- // --- connection ----------------------------------------------------------
329
- // Neither of these may close the eyes. A lid at 0.3 reads sleepy and one at
330
- // 0.95 reads asleep (research-perception.md §6), and on a photographic face
331
- // a disconnected call rendered as someone falling asleep on camera. The SVG
332
- // faces also desaturate (`filter`); a rig that does not is carried by pose
333
- // and gaze alone, so those have to say it.
334
- //
335
- // Our side of the call is struggling: still with the user, a little worried
336
- // about it — inner brows up, mouth pressed — and quieter than listening.
337
- DEGRADED: { gaze: 'USER', emotion: 'neutral', engagement: false,
338
- aversion: 'LISTEN',
339
- idle: { sway: 0.4, blinkGap: [3.4, 5.0] },
340
- pose: { browInnerL: 0.30, browInnerR: 0.26, browRaiseL: -0.04, browRaiseR: -0.04,
341
- mouthPress: 0.20, mouthCornerL: -0.10, mouthCornerR: -0.10 },
342
- filter: 'grayscale(.55) brightness(.82)' },
343
- // The call is gone. What a person does when their call drops: eyes on their
344
- // own screen, waiting, and now and then a look back up to see whether it
345
- // has come back. Still, with long holds — not in conversation — but awake.
346
- OFFLINE: { gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
347
- scan: [1.2, 2.6],
348
- idle: { sway: 0.3, blinkGap: [4.5, 7.0],
349
- hold: { every: [4.0, 8.0], dur: [1.0, 2.0] } },
350
- glance: { to: 'USER', every: [7, 12], hold: [0.8, 1.3] },
351
- pose: { lidL: -0.08, lidR: -0.08, mouthPress: 0.25 },
352
- filter: 'grayscale(1) brightness(.6)' },
353
- };
354
-
355
- export const STATE_NAMES = Object.keys(STATES);
356
-
357
- /**
358
- * The shoulder line's share of a held tilt, per unit of `headRoll`.
359
- *
360
- * Roll is the one head axis with nothing under it: a turn recruits the trunk
361
- * and a nod bends the neck, but a tilt on a 2.5-D head is a rotation about a
362
- * point near the chin and every other pixel stays exactly where it was. That is
363
- * the read reported as "a hinge". Anatomically a tilt is lower-cervical
364
- * (docs/research-head-rotation.md §2.1) — spread down the neck, ending at the
365
- * girdle — so the shoulder line tips a little with it, and that tip is what says
366
- * the neck bent rather than the head swinging off a pin.
367
- *
368
- * Sized to stay an accompaniment: at the roll clamp the line tips about a
369
- * quarter of the trunk's own share of a turn, which lifts a shoulder ~0.11 of
370
- * its shrug — inside the band a weight shift already occupies. A third, the
371
- * figure Live2D gives its body angles, put 6 px of shoulder on an 8 degree tilt
372
- * and read as a shrug arriving with the head.
373
- *
374
- * Exported for the rig instruments.
375
- */
376
- export const SHOULDER_TILT = 0.08;
377
-
378
- export function createAvatar(opts = {}) {
379
- const mount = typeof opts.mount === 'string' ? document.querySelector(opts.mount) : opts.mount;
380
- if (!mount) throw new Error('createAvatar: mount element required');
381
-
382
- // `opts.face` is a Face record — `{ create, meta }`. Passed in rather than
383
- // named: a name needs a table, and a table imports every drawing to answer
384
- // one lookup (`src/faces.js` has that table, for tooling that wants them).
385
- const entry = opts.rig ? null : opts.face;
386
- if (!opts.rig && !entry) {
387
- throw new Error('createAvatar: a `face` (see src/faces.js) or a `rig` is required');
388
- }
389
- const face = entry ? entry.create(mount, opts.theme) : null;
390
- // A renderer-neutral rig needs no SVG descriptor: `meta` is what SVG hosts
391
- // and tools frame the drawing with, never a requirement of the rig contract.
392
- // A face carries its own — a `{ create }` with no META used to be tolerated
393
- // here and the viewBox re-read off the produced svg, which meant a face could
394
- // ship half a descriptor and nothing would say so.
395
- const meta = face ? entry.meta : null;
396
- const gaze = new GazeLayer();
397
- const idle = new IdleLayer();
398
- const speech = new VisemeTrack();
399
- const prosody = new SpeechProsody({ brows: opts.brows });
400
-
401
- let gazeOverrideByClip = null;
402
- const clip = new ClipPlayer({
403
- onGaze: (g) => { gazeOverrideByClip = g; applyGaze(); },
404
- onBlink: () => idle.blink(),
405
- });
406
- const engagement = new ListeningEngine();
407
- const performTrack = new PerformTrack();
408
- // The current SVG hand is a renderer adapter for the first-class `frame.hand`
409
- // control. `hand: false` only disables its SVG rendering; gesture actions
410
- // still emit the semantic hand frame for a supplied custom rig.
411
- const hand = face && opts.hand !== false ? createHand(face.svg, face.theme, meta, { dir: opts.handSide }) : null;
412
- const rig = opts.rig ? opts.rig(mount, opts.rigOptions) : createSvgRig(face, hand);
413
-
414
- gaze.onLargeShift = (forced) => idle.blink(true, forced);
415
-
416
- const listeners = {
417
- state: [], speakEnd: [], clipEnd: [], performEnd: [], gestureEnd: [],
418
- };
419
- const emit = (ev, ...a) => listeners[ev] && listeners[ev].forEach((f) => f(...a));
420
- clip.onEnd = (c) => { if (c) emit('clipEnd', c.id); };
421
- speech.onEnd = () => { prosody.closeTurn(); emit('speakEnd'); };
422
- performTrack.onEnd = () => { emit('performEnd'); };
423
-
424
- // --- live state -----------------------------------------------------------
425
- let stateName = 'IDLE';
426
- let emotion = 'neutral';
427
- let emotionAmt = 1;
428
- let gazeName = 'USER';
429
- let gazeCustom = null;
430
- let overrides = null; // demo/debug direct param injection
431
- // Articulation gain. VISEME_SHAPES is tuned for a face at conversational
432
- // size; sharing the screen with live video the same shapes read as
433
- // under-articulated. Scales every viseme away from rest, so shape identities
434
- // and their relative sizes survive and only the excursion changes. Above
435
- // ~1.5 the open vowels saturate against the clamp — the intended ceiling.
436
- let mouthGain = opts.mouthGain ?? 1;
437
- let handSide = opts.handSide === -1 ? 'left' : 'right';
438
- /**
439
- * This avatar's own addressable motions, on top of the core intents.
440
- *
441
- * An avatar is a drawing with a body, and some bodies can do things the wire
442
- * has no portable word for. The nod types the listening research separates —
443
- * a continuer, an assessment, a realisation — are one
444
- * `ACKNOWLEDGE` to a server, because that is all a server can ask of every
445
- * face; the *shapes* are sized in a rig's own units and belong to the rig
446
- * (`packages/avatar/client/three/sequences.ts` is the first table of them).
447
- *
448
- * Deliberately not a registry and not a loader: it is a plain object the
449
- * avatar module passes in, and it can only add. `ACTIONS` wins on a name
450
- * collision, so no asset can quietly change what a core action means.
451
- */
452
- const sequences = opts.sequences || {};
453
- /**
454
- * This avatar's own *rendering* of an id this renderer already has — same id,
455
- * same intent, a shape sized for its body.
456
- *
457
- * These are an intent vocabulary, and `ACK_NOD` is one implementation of
458
- * "the avatar nodded", not a promise about anatomy (interjections.js). The
459
- * shared clips are authored in pose units that mean pixels on a line face;
460
- * on a rig whose unit is a degree the same keys can land outside what the
461
- * gesture *means* — the shared nod renders 15° on tara, where a continuer is
462
- * 3-5° — and scaling it would not fix it, because a continuer is one stroke
463
- * where an agreement is two. So a renderer may re-author the shape.
464
- *
465
- * What it may not do is change the vocabulary: every id here must already be
466
- * one this renderer publishes, so no avatar can add a word through this door
467
- * or make one mean something else. `sequences` is the door for adding.
468
- */
469
- const actionShapes = opts.actions || {};
470
- for (const id of Object.keys(actionShapes)) {
471
- if (!ACTIONS[id]) throw new Error(`actions: ${id} is not one of this renderer's own`);
472
- }
473
- /**
474
- * This avatar's own rendering of a state — the same door as `actions`, for
475
- * the held face instead of the gesture. A rig replaces a state's fields
476
- * whole (`pose`, say), and only for a state the table already has: it can
477
- * re-render the vocabulary, never extend it.
478
- */
479
- const states = { ...STATES };
480
- for (const [id, own] of Object.entries(opts.states || {})) {
481
- if (!STATES[id]) throw new Error(`states: ${id} is not a state`);
482
- states[id] = { ...STATES[id], ...own };
483
- }
484
- let handAction = null;
485
- const handQueue = [];
486
- // Gesture gain, same idea for the clip layer: small gestures under-render
487
- // through the head's τ — see internal-mixer.md § Smoothing.
488
- let gestureGain = opts.gestureGain ?? 1;
489
- // Body-liveness gain. Constraint 8 (this widget shares the screen with a
490
- // live video call) argues for the smallest idle motion that still reads, and
491
- // the amplitudes in idle.js are set where they read. A host that is actually
492
- // re-encoding the avatar — compositing it into an outgoing stream rather
493
- // than rendering it locally as SVG, where the motion costs nothing — turns
494
- // this down instead of the default being a body that does not move.
495
- idle.gain = opts.motionGain ?? 1;
496
- // Speech-rhythm gains. A pose unit is a different angle on every rig — peep's
497
- // head travels 17 px per unit of pitch, a mesh head turns a few degrees — so
498
- // the rig's own module sets these once rather than the library guessing.
499
- // Defaults to the uncalibrated scale; `prosody.js` has what that stands for.
500
- const prosodyHeadGain = opts.prosodyHeadGain ?? UNCALIBRATED_HEAD_GAIN;
501
- const prosodyFaceGain = opts.prosodyFaceGain ?? 1;
502
- // Fixational-saccade gain, the same idea for the eyes: how far a scan
503
- // step, a drift or a line of reading travels. Sized for a line face's
504
- // pupils at 1; a photographic eye needs several times that to move at all.
505
- // Aversions get their own gain: a look-away must read as one from across
506
- // the call, where a fixation step must not, so one number cannot size both.
507
- const saccadeGain = opts.saccadeGain ?? 1;
508
- // How far this face may hold its head off centre, per axis (internal-mixer.md
509
- // § The held-head budget): the rig supplies it; an axis left out is unbudgeted.
510
- const headHold = opts.headHold || {};
511
- // Per-axis gain on the head's *continuous* drive — speech phrasing and idle,
512
- // both sized in pose units for a line drawing. Measuring a 2.5-D face found
513
- // the drive spending a third of the pitch a speaking human uses and nearly
514
- // twice the yaw. In front of 6b deliberately: a gain that skipped the budget
515
- // would be measuring the face's failure rather than the layer's range.
516
- const headGain = typeof opts.headGain === 'number'
517
- ? { headYaw: opts.headGain, headPitch: opts.headGain, headRoll: opts.headGain }
518
- : { headYaw: 1, headPitch: 1, headRoll: 1, ...(opts.headGain || {}) };
519
- gaze.scanGain = saccadeGain;
520
- gaze.avertGain = opts.aversionGain ?? 1;
521
- // A rig that says what its pose units are in degrees gets the eye-head
522
- // system sized for it (gaze.js): its own look targets, how an aversion
523
- // splits between eyes and head, lids that follow the eye both ways, and the
524
- // reflex in step 8b.
525
- const ocu = opts.oculomotor || {};
526
- if (ocu.targets) gaze.targets = { ...GAZE_TARGETS, ...ocu.targets };
527
- if (ocu.avert) gaze.avertSplit = ocu.avert;
528
- if (ocu.angles) gaze.angles = ocu.angles;
529
- if (ocu.lidFollow) gaze.lidFollow = ocu.lidFollow;
530
- if (ocu.head) { gaze.headAccel = ocu.head.accel; gaze.headSpeed = ocu.head.speed; }
531
- // Pupil units of counter-rotation per head unit: the reflex's gain times
532
- // the ratio of what one unit of each is in degrees. The gain may differ by
533
- // axis ({x, y}): a face whose pitch reads weaker than its yaw wants less
534
- // of the eyes' answer to it.
535
- const vorGain = typeof ocu.vor === 'object' ? ocu.vor : { x: ocu.vor, y: ocu.vor };
536
- const vor = ocu.vor && ocu.angles
537
- ? { x: vorGain.x * ocu.angles.head.x / ocu.angles.eye.x,
538
- y: vorGain.y * ocu.angles.head.y / ocu.angles.eye.y }
539
- : null;
540
- // How far the reflex may carry the eye in its socket, in pupil units:
541
- // Guitton & Volle's effective oculomotor range; the rig supplies the reach.
542
- const reach = ocu.range || { x: 1, up: 1, down: 1 };
543
- const reflexX = (px) => clamp(px + vor.x * (aim.x - cur.headYaw), -reach.x, reach.x);
544
- const reflexY = (py) => clamp(py + vor.y * (aim.y - cur.headPitch), -reach.up, reach.down);
545
- // The head the eyes' target was authored against, smoothed at the eye's
546
- // own tau so a shift's compensation moves with the saccade, not ahead of it.
547
- const aim = { x: 0, y: 0 };
548
- let wanderAt = 0;
549
- let driftAt = 0;
550
- let trunkYaw = 0;
551
- let glanceAt = 0;
552
- let glanceUntil = 0;
553
- let lastBack = null;
554
- // THINKING's second look inside a look away: when it moves, where to, and
555
- // how long the brows stay up for it.
556
- let dartAt = 0, dartBrowUntil = 0;
557
- const dart = { x: 0, y: 0 };
558
- // The state whose gaze is showing, and when the current state takes it
559
- // over if that is still pending (GAP_SETTLE). Usually the same state.
560
- let gazeState = 'IDLE';
561
- let settleAt = 0;
562
- // Reading scan: position in the line and the block, see readStep.
563
- let readCol = 0, readCols = 0, readRow = 0, readRows = 0;
564
- let attendUntil = 0;
565
- let speakClock = null;
566
- let speakStart = 0;
567
-
568
- const cur = Object.assign({}, REST);
569
- const target = Object.assign({}, REST);
570
- // The head axes again, carrying only what is held (step 6b).
571
- const hold = { headYaw: 0, headPitch: 0, headRoll: 0 };
572
- // What the rig is handed: `cur` with the reflex applied to the eyes and the
573
- // lips stopped at shut.
574
- const shown = Object.assign({}, REST);
575
-
576
- function applyGaze(blink) {
577
- const g = gazeOverrideByClip || gazeName;
578
- gaze.set(g, gazeOverrideByClip ? null : gazeCustom, blink);
579
- }
580
-
581
- // --- the frame ------------------------------------------------------------
582
- let raf = 0;
583
- let last = 0;
584
- let elapsed = 0;
585
- // `manual` withholds the rAF loop so a tool can drive frames itself.
586
- const manual = !!opts.manual;
587
-
588
- function frame(now) {
589
- raf = requestAnimationFrame(frame);
590
- if (!last) last = now;
591
- // Cap dt so a backgrounded tab doesn't fast-forward the whole rig on return.
592
- const dt = Math.min(0.05, (now - last) / 1000);
593
- last = now;
594
- elapsed += dt;
595
- step(dt, dt * 1000);
596
- }
597
-
598
- function step(dt, dtMs) {
599
- // 0. the performance timeline. Sampled before the pose is built so a verb
600
- // firing this frame shapes this frame.
601
- performTrack.update();
602
-
603
- if (settleAt && elapsed >= settleAt) { settleAt = 0; enterGaze(stateName, {}, true); }
604
- const st = states[stateName] || states.IDLE;
605
- // What the eyes are doing belongs to the state that has the gaze.
606
- const gst = states[gazeState] || st;
607
-
608
- // 1. base pose: rest + emotion + state-specific overlay
609
- for (const c of CHANNELS) target[c] = REST[c];
610
- const ep = emotionPose(emotion, emotionAmt);
611
- for (const k in ep) target[k] = REST[k] + ep[k];
612
- if (st.pose) for (const k in st.pose) target[k] = (target[k] || 0) + st.pose[k];
613
- // The head the state and emotion hold is an attitude, and the eyes are
614
- // authored inside it: the reflex (8b) keeps them on their target against
615
- // everything that moves the head except this.
616
- const poseYaw = target.headYaw, posePitch = target.headPitch;
617
- // What the head *holds*, accumulated alongside the pose as each layer that
618
- // holds one adds to it: the attitude above, the gaze, a phrase's pose and
619
- // the idle posture. Strokes, beats and clip deltas stay out — see 6b.
620
- for (const c of HEAD_AXES) hold[c] = target[c];
621
-
622
- // 2. gaze (absolute: pupils + partial head follow, plus the lid that rides
623
- // with the eye — looking down without it bares sclera and reads as alarm)
624
- const g = gaze.update(elapsed, dt);
625
- for (const k in g) {
626
- if (k === 'trunkYaw' || k === 'aimYaw' || k === 'aimPitch') continue;
627
- target[k] = (k.startsWith('head') ? target[k] : 0) + g[k];
628
- }
629
- for (const c of HEAD_AXES) hold[c] += g[c];
630
- // The lid follows the eye as drawn, reflex and all — last frame's reading
631
- // of it, since the head it corrects for is not smoothed yet.
632
- const lid = gaze.lidBias(vor ? reflexY(g.pupilY) : g.pupilY);
633
- target.lidL += lid;
634
- target.lidR += lid;
635
- if (vor) {
636
- aim.x = approach(aim.x, poseYaw + g.aimYaw, TAU.pupilX, dt);
637
- aim.y = approach(aim.y, posePitch + g.aimPitch, TAU.pupilY, dt);
638
- }
639
-
640
- // 2b. the trunk follows the head. Sampled HERE, after gaze and before the
641
- // clip layer, on purpose: a sustained turn toward the screen recruits
642
- // the trunk, and a nod or a head shake does not — a body that swings
643
- // with every gesture reads as a mannequin on a turntable. The lag is
644
- // not authored anywhere; torsoTurn chases the same target at a slower
645
- // TAU than the head (params.js), so the trunk leaves late and settles
646
- // late for free, at the share TRUNK_FOLLOW names. It follows where the
647
- // head is going and not the looks riding on it, and it holds through a
648
- // glance: checking on the user is a look, not a turn toward them.
649
- if (!glanceUntil) trunkYaw = target.headYaw - g.headYaw + g.trunkYaw;
650
- target.torsoTurn += trunkYaw * TRUNK_FOLLOW;
651
-
652
- // 3. state-driven autonomous behaviour
653
- if (gst.wander && elapsed > wanderAt) {
654
- const w = gst.wander;
655
- // Dwell first, and sometimes for two periods. A dwell of exactly one
656
- // period every time is the metronome the eye reads as mechanical. The
657
- // old code broke that up by accident: a draw landing on the target
658
- // already held left the eyes still for another period. The freeze was a
659
- // bug — the schedule counted a hop that moved nothing — but the dwell
660
- // variety it produced was not, so it is kept deliberately, at the rate
661
- // it used to happen: 1/n, the odds a uniform draw repeats the target.
662
- const linger = Math.random() < 1 / w.targets.length ? 2 : 1;
663
- wanderAt = elapsed + linger * (w.every[0] + Math.random() * (w.every[1] - w.every[0]));
664
- // Then look somewhere the eyes are not. A wander set names a target more
665
- // than once deliberately — SEARCHING_SCREEN's repeats are its revisits,
666
- // the hunt coming back to the middle — but a revisit is arriving
667
- // somewhere again, not never having left it. Same bounded redraw as
668
- // pickBack, so a one-target set still terminates.
669
- let g = pick(w.targets);
670
- for (let i = 0; i < 4 && g === gazeName; i++) g = pick(w.targets);
671
- setGaze(g);
672
- }
673
- // Periodic glance (WORKING's look-up-at-you beat, THINKING's check-in
674
- // between two looks away). The return leg goes to the state's own gaze, or
675
- // to a fresh pick from `back` — a thinker does not look away at the same
676
- // spot every time. Either leg's gaze-evoked blink is the shift's odds
677
- // unless the state says (`blinkTo`, `blinkBack`), because which leg blinks
678
- // is part of what the glance means.
679
- if (gst.glance) {
680
- const gl = gst.glance;
681
- if (glanceUntil && elapsed > glanceUntil) {
682
- glanceUntil = 0;
683
- const away = rand(gl.every);
684
- glanceAt = elapsed + away;
685
- setGaze(gl.back ? pickBack(gl.back, gl.stick) : gst.gaze, null, gl.blinkBack);
686
- // At most one second look per look away, somewhere in its middle: a
687
- // move at the start is the same look landing, and one at the end runs
688
- // into the glance back.
689
- dartAt = gl.dart && Math.random() < gl.dart.p ? elapsed + away * (0.35 + Math.random() * 0.3) : 0;
690
- } else if (!glanceUntil && elapsed > glanceAt) {
691
- glanceUntil = elapsed + rand(gl.hold);
692
- setGaze(gl.to, null, gl.blinkTo);
693
- dartAt = 0;
694
- dart.x = dart.y = 0;
695
- }
696
- if (dartAt && elapsed > dartAt) {
697
- // On from where the eyes are, not back toward the user or across to
698
- // the other side: the thought moves, it does not turn round.
699
- dartAt = 0;
700
- const t = gaze.target;
701
- const a = Math.atan2(t.py, t.px) + (Math.random() < 0.5 ? -1 : 1) * DART_TURN;
702
- const m = gl.dart.mag * saccadeGain;
703
- dart.x = Math.cos(a) * m;
704
- dart.y = Math.sin(a) * m;
705
- dartBrowUntil = elapsed + DART_BROW;
706
- }
707
- const brow = (glanceUntil ? gl.brow || 0 : 0)
708
- + (gl.dart && elapsed < dartBrowUntil ? gl.dart.brow : 0);
709
- target.browRaiseL += brow;
710
- target.browRaiseR += brow;
711
- }
712
- gaze.scanEvery = gst.scan || null;
713
- // A state that reads scans a line; one with a dart holds its second look.
714
- // Neither during a glance: a check-in on the user is steady or it is not
715
- // one.
716
- if (gst.read && !glanceUntil) {
717
- if (elapsed > driftAt) readStep(gst.read);
718
- } else {
719
- gaze.drift.x = glanceUntil ? 0 : dart.x;
720
- gaze.drift.y = glanceUntil ? 0 : dart.y;
721
- }
722
- // Aversion is a property of the state, but it is held off around a turn
723
- // boundary: the floor is handed over under mutual gaze, and an avatar that
724
- // looks away exactly as the user finishes has declined it. `attend` is the
725
- // mixer's one-frame veto — anything that means "the user is checking
726
- // whether I am with them" sets it (see api.attend).
727
- gaze.setAversion(gst.aversion ? AVERSION[gst.aversion] : null);
728
- gaze.hold = attendUntil > elapsed || clip.playing;
729
-
730
- engagement.enabled = !!st.engagement && !clip.playing;
731
- engagement.update(dt);
732
- // Engagement posture: forward lean while the user holds the floor, spent
733
- // only in the states that are *about* the user holding the floor. The
734
- // research (docs/research-biomechanics.md §6.3) puts sustained attentive
735
- // lean at +0.15–0.25; engage glides, and torsoLean's 0.24s tau smooths
736
- // the state gate, so the lean arrives and leaves like weight shifting.
737
- if (st.engagement) {
738
- target.torsoLean += 0.16 * engagement.engage;
739
- // The shoulders come with it. A lean is a whole upper body arriving, and
740
- // in a head-and-shoulders crop the shoulder line is the part of it that
741
- // is actually on screen — §6.1 calls a shoulder rise the most legible
742
- // thing this framing can draw. Without this the listening shoulders were
743
- // the posture shift's ±0.07 and nothing else, so the one channel that
744
- // could show attention sat still through every turn the user took.
745
- // It rides `engage`, so it is contingent on the user's voice rather than
746
- // on the state: quick in, slow out, and gone a few seconds into silence.
747
- target.shoulderL += ENGAGE_SHOULDER * engagement.engage;
748
- target.shoulderR += ENGAGE_SHOULDER * engagement.engage;
749
- }
750
- // Straining leans harder while there is actually a faint voice to strain
751
- // after. engage already tracks "the user is (barely) talking", so this
752
- // costs nothing; with no user signal the static pose carries the state.
753
- else if (stateName === 'CANT_HEAR') target.torsoLean += 0.10 * engagement.engage;
754
-
755
- // 4. mouth. The server's viseme track wins; a clip's mouth track fills the
756
- // gaps. There is deliberately no third leg: with no cues the mouth stays
757
- // shut, and a still mouth under speech is the *visible* symptom of a
758
- // backend that could not align.
759
- const clipOut = clip.update(dtMs);
760
- let mouth = speech.sample();
761
- let mouthOwner = mouth ? 'speech' : null;
762
- if (!mouth && clipOut.ownsMouth && clipOut.mouth) { mouth = clipOut.mouth; mouthOwner = 'clip'; }
763
- if (mouth) {
764
- const shape = mouth.letter !== SILENT
765
- ? shapeFor(mouth.letter, mouth.intensity)
766
- : shapeFor(SILENT, 1);
767
- // Rounding coming up is held against the shape's own (VisemeTrack.sample).
768
- if (mouth.round > shape.mouthRound) shape.mouthRound = mouth.round;
769
- // Gain pivots on the rest shape, not on zero: scaling absolute values would
770
- // drag the closed mouth open, which is the one thing lipsync must never do.
771
- for (const k in shape) {
772
- target[k] = mouthGain === 1
773
- ? shape[k]
774
- : REST_SHAPE[k] + (shape[k] - REST_SHAPE[k]) * mouthGain;
775
- }
776
- // Clip-owned mouths are exempt — a spoken OKAY *is* the warmth episode —
777
- // and only the base is scaled, so a gesture clip can still smile over a
778
- // sentence by authoring corner keys (they add, unscaled, in step 5).
779
- if (mouthOwner !== 'clip') {
780
- target.mouthCornerL *= SPEAK_SMILE_RETAIN;
781
- target.mouthCornerR *= SPEAK_SMILE_RETAIN;
782
- }
783
- }
784
-
785
- // 4b. speech prosody: pause blinks, inbreaths, a head held per phrase and
786
- // moved between them, beats and turn-edge warmth, read off the cue
787
- // track (prosody.js has the research, head.js the hold-and-move). Off while a clip is gesturing, since the clip is
788
- // already the head's and the brows' story. The warmth is withheld
789
- // under an emotion whose corners are down: a smile starting a turn of
790
- // concern is the wrong face, whatever the rhythm says.
791
- const pro = prosody.update(speech, mouthOwner === 'speech' && !clip.playing, dt);
792
- // The warmth rides over a clip, because an acknowledgement's smile is
793
- // *with* its nod: behind the clip gate it arrived as the nod finished, a
794
- // smile at nothing. It is withheld under a clip whose own corners go down
795
- // (a shake, a sorry) for the same reason as under a concerned emotion.
796
- const clipFrowns = !!clipOut.delta
797
- && Math.min(clipOut.delta.mouthCornerL || 0, clipOut.delta.mouthCornerR || 0) < 0;
798
- const warm = clipFrowns || ((EMOTIONS[emotion] || EMOTIONS.neutral).mouthCornerL || 0) < 0
799
- ? 0 : prosodyFaceGain;
800
- target.mouthCornerL += pro.mouthCornerL * warm;
801
- target.mouthCornerR += pro.mouthCornerR * warm;
802
- target.squintL += pro.squintL * warm;
803
- target.squintR += pro.squintR * warm;
804
- if (!clip.playing) {
805
- target.headPitch += pro.headPitch * prosodyHeadGain * headGain.headPitch;
806
- target.headYaw += pro.headYaw * prosodyHeadGain * headGain.headYaw;
807
- target.headRoll += pro.headRoll * prosodyHeadGain * headGain.headRoll;
808
- for (const c of HEAD_AXES) hold[c] += pro.hold[c] * prosodyHeadGain * headGain[c];
809
- // The trunk follows a speech pose the way it follows a gaze turn (2b),
810
- // and for a second reason on a 2.5-D head: a turn the shoulders take
811
- // part of is a turn the neck does not have to stretch for, and the
812
- // stretch is the one thing a video reviewer saw on every pose change.
813
- // Scaled with the yaw it follows: a trunk that kept its old share of a
814
- // wider turn would be the neck stretching again, which is the defect
815
- // this line exists to fix.
816
- target.torsoTurn += pro.trunkYaw * prosodyHeadGain * headGain.headYaw * TRUNK_FOLLOW;
817
- target.browRaiseL += pro.browRaiseL * prosodyFaceGain;
818
- target.browRaiseR += pro.browRaiseR * prosodyFaceGain;
819
- target.browInnerL += pro.browInnerL * prosodyFaceGain;
820
- target.browInnerR += pro.browInnerR * prosodyFaceGain;
821
- target.browAngleL += pro.browAngleL * prosodyFaceGain;
822
- target.browAngleR += pro.browAngleR * prosodyFaceGain;
823
- target.lidL += pro.lidL * prosodyFaceGain;
824
- target.lidR += pro.lidR * prosodyFaceGain;
825
- target.breath += pro.breath * idle.gain;
826
- // The trunk's share of speech rhythm. Scaled by `idle.gain` and not by
827
- // `prosodyHeadGain`, because this is body liveness and that is the knob a
828
- // host turns down when it is re-encoding the avatar into an outgoing
829
- // stream — the same factor `idle.js`'s shoulders and lean already take.
830
- target.shoulderL += pro.shoulderL * idle.gain;
831
- target.shoulderR += pro.shoulderR * idle.gain;
832
- target.torsoLean += pro.torsoLean * idle.gain;
833
- }
834
- if (pro.blink) idle.phraseBlink();
835
-
836
- // 5. gesture deltas (additive, so a nod survives whatever else is happening)
837
- if (clipOut.delta) {
838
- for (const k in clipOut.delta) {
839
- // Never let a clip's mouth keyframes fight the live viseme stream.
840
- if (mouthOwner === 'speech' && MOUTH_LOCK.has(k)) continue;
841
- target[k] = (target[k] || 0) + clipOut.delta[k] * gestureGain;
842
- }
843
- }
844
-
845
- // 6. idle: sway, breath, blink
846
- // The torso's share of the liveness follows whether sound is actually being
847
- // produced, not what state the avatar is nominally in — a SPEAKING state with the
848
- // track finished should already be settling.
849
- idle.talk = approach(idle.talk, mouthOwner ? 1 : 0, 0.25, dt);
850
- idle.setProfile(st.idle);
851
- const il = idle.update(dt);
852
- for (const k in il.add) target[k] = (target[k] || 0) + il.add[k] * (headGain[k] ?? 1);
853
- for (const c of HEAD_AXES) hold[c] += (il.add[c] || 0) * headGain[c];
854
-
855
- // 6b. the held-head budget (internal-mixer.md § The held-head budget).
856
- // Only the excess over `soften` comes off, and only off the hold: a
857
- // nod, a beat and a clip keep every degree they were authored with,
858
- // which is why this is subtracted here rather than applied to the pose.
859
- for (const c of HEAD_AXES) {
860
- if (headHold[c] === undefined) continue;
861
- target[c] -= hold[c] - soften(hold[c], headHold[c]);
862
- }
863
-
864
- // 6c. the body answers a held tilt (SHOULDER_TILT). The trunk takes the
865
- // same share of it that it takes of a turn in 2b — Live2D gives its
866
- // body the same fraction of AngleZ as of AngleX (research-head-rotation
867
- // .md §3) — and the shoulder line tips with the head. Both channels
868
- // smooth slower than the head (TAU), so the body leaves late and
869
- // settles late, and that follow-through is most of what separates a
870
- // neck bending from a hinge.
871
- // Only the *held* roll, and after the budget: a stroke or a clip's roll
872
- // is a gesture riding on the pose, and a body that answers those is 2b's
873
- // mannequin on a turntable. The idle layer's own posture is already
874
- // coupled the other way round, from the weight shift to the head that
875
- // counter-tips on it (idle.js `nextPosture`); this is that arrangement
876
- // read from the head's end, and where both are in play they agree in
877
- // sign — weight onto a side, head over that side, that shoulder up.
878
- const heldRoll = headHold.headRoll === undefined
879
- ? hold.headRoll
880
- : soften(hold.headRoll, headHold.headRoll);
881
- target.torsoTurn += heldRoll * TRUNK_FOLLOW;
882
- target.shoulderR += heldRoll * SHOULDER_TILT;
883
- target.shoulderL -= heldRoll * SHOULDER_TILT;
884
-
885
- // 7. clamp, then blink wins outright over whatever the lids were doing
886
- for (const c of CHANNELS) {
887
- const r = RANGE[c];
888
- target[c] = clamp(target[c], r[0], r[1]);
889
- }
890
- // 7a. a bilabial aims past shut (`LIP_CONTACT_AIM`), after the clamp
891
- // because it is the one target no pose is allowed to hold: only the
892
- // chase sees it, and step 8b stops what is drawn at shut.
893
- if (mouth && mouth.letter === 'A') target.mouthOpen = -LIP_CONTACT_AIM;
894
- if (il.blink > 0) {
895
- target.lidL = Math.max(target.lidL, il.blink);
896
- target.lidR = Math.max(target.lidR, il.blink);
897
- }
898
-
899
- if (overrides) for (const k in overrides) target[k] = overrides[k];
900
-
901
- // 8. smooth toward the target — this is where co-articulation happens
902
- for (const c of CHANNELS) cur[c] = approach(cur[c], target[c], TAU[c], dt);
903
-
904
- // 8b. What is drawn: the lips stop at shut however far past it they aimed.
905
- Object.assign(shown, cur);
906
- if (shown.mouthOpen < 0) shown.mouthOpen = 0;
907
-
908
- // 8c. The vestibulo-ocular reflex. Here and not in the gaze layer because
909
- // only here is the head that is actually drawn known: prosody, clips
910
- // and idle all land after gaze.
911
- if (vor) {
912
- if (!overrides || overrides.pupilX === undefined) {
913
- shown.pupilX = clamp(reflexX(cur.pupilX), RANGE.pupilX[0], RANGE.pupilX[1]);
914
- }
915
- if (!overrides || overrides.pupilY === undefined) {
916
- shown.pupilY = clamp(reflexY(cur.pupilY), RANGE.pupilY[0], RANGE.pupilY[1]);
917
- }
918
- }
919
-
920
- // 9. First-class hand control. The semantic frame is generated here, above
921
- // every renderer, so SVG, WebGL, and video rigs receive exactly the same
922
- // gesture/progress information. A handless rig simply ignores `frame.hand`.
923
- const handFrame = updateHandAction(elapsed * 1000);
924
- rig.apply(avatarFrame(shown, handFrame || undefined));
925
- }
926
-
927
- const REST_SHAPE = shapeFor(SILENT, 1);
928
-
929
- // How much of a sustained head turn the trunk takes up. Well under 1: people
930
- // under-rotate the head and then under-rotate the trunk again behind it, and
931
- // at a head-and-shoulders crop the trunk's share is the part you register
932
- // without noticing. A rig may pass its own `trunkFollow`: on a mesh head whose
933
- // neck twists, the trunk's sway is most of what moves the neck's outline, so
934
- // the share that reads as a body on a line face reads there as the neck
935
- // sliding.
936
- const TRUNK_FOLLOW = opts.trunkFollow ?? 0.45;
937
-
938
- // The channels speech owns outright — exactly the params.js mouth group
939
- // (mouth corners stay free: a clip may smile over a sentence).
940
- const MOUTH_LOCK = new Set(GROUPS.mouth);
941
-
942
- /** The shoulders' share of the attentive posture: two thirds of the lean the
943
- * engagement layer spends — the shoulders come up with it, they do not lead it. */
944
- const ENGAGE_SHOULDER = 0.10;
945
-
946
- // What survives of the resting/emotion smile while speech owns the mouth.
947
- // A smile held static through a sentence is discounted as insincere, and
948
- // corners riding every open viseme read as laughing through the words: warmth
949
- // must be episodic (research-perception.md §3). Full warmth returns the moment
950
- // the track ends, which is that onset/offset.
951
- const SPEAK_SMILE_RETAIN = 0.35;
952
-
953
- // Between the user's turn and the reply the server's claim can change
954
- // several times a second — THINKING, a tool's WORKING, THINKING again, a
955
- // grace timer's CANT_HEAR — and every change used to retarget the eyes and
956
- // restart the state's looks: one fast tool call was away, down at the
957
- // screen, back to the user and away again inside a second, a blink at each.
958
- // Among these states the eyes change over only once the new one has held
959
- // this long, and a claim that returns before then moves nothing. The pose
960
- // still changes at once — it is the eyes that make a flicker visible.
961
- // SPEAKING and LISTENING are never held back: those are the floor.
962
- const GAP_STATES = new Set(['THINKING', 'WORKING', 'CANT_HEAR']);
963
- const GAP_SETTLE = 0.5;
964
-
965
- // A dart leaves at up to 40° off the line of the look it is inside, and
966
- // lifts the brows for half a second — a thought arriving, not a stare.
967
- const DART_TURN = 0.7;
968
- const DART_BROW = 0.5;
969
-
970
- /** A look away for the return leg of a glance. `stick` of the time it keeps
971
- * the side the last one took, spot and all — the side a person thinks
972
- * toward is theirs, and it outlasts one thinking pause. Otherwise any look
973
- * but the one just taken: a thinker who goes back to the same patch of
974
- * wall every time *without* meaning to is a loop. */
975
- function pickBack(xs, stick = 0) {
976
- const side = (n) => Math.sign(gaze.targets[n].px);
977
- let g;
978
- if (lastBack && Math.random() < stick) {
979
- const same = xs.filter((n) => side(n) === side(lastBack));
980
- g = pick(same.length ? same : xs);
981
- } else {
982
- g = pick(xs);
983
- for (let i = 0; i < 4 && g === lastBack; i++) g = pick(xs);
984
- }
985
- lastBack = g;
986
- return g;
987
- }
988
-
989
- /**
990
- * One fixation of a reading scan: a step along the line, or at its end a
991
- * sweep back to the start of the next, or at the end of the block a pause
992
- * on the result. That is what busy-at-a-screen looks like from the other
993
- * side of a webcam — an irregular left-to-right march with returns, which
994
- * a random drift about a point is not. Steps are held, like a reader's
995
- * fixations, and the head sits out all of it (DRIFT_HEAD in gaze.js).
996
- */
997
- function readStep(r) {
998
- const n = ([a, b]) => a + ((Math.random() * (b - a + 1)) | 0);
999
- if (readCol < readCols) {
1000
- readCol++;
1001
- driftAt = elapsed + rand(r.fix);
1002
- } else {
1003
- readCol = 0;
1004
- readCols = n(r.steps);
1005
- if (++readRow >= readRows) {
1006
- readRow = 0;
1007
- readRows = n(r.lines);
1008
- driftAt = elapsed + rand(r.pause);
1009
- } else {
1010
- driftAt = elapsed + rand(r.fix) * 1.4;
1011
- }
1012
- }
1013
- gaze.drift.x = (readCol / readCols - 0.5) * r.width * saccadeGain;
1014
- gaze.drift.y = (readRow - (readRows - 1) / 2) * r.line * saccadeGain;
1015
- }
1016
-
1017
- // --- API ------------------------------------------------------------------
1018
-
1019
- function setState(name, o = {}) {
1020
- if (!states[name]) throw new Error(`unknown state: ${name}`);
1021
- const changed = name !== stateName;
1022
- stateName = name;
1023
- const st = states[name];
1024
- if (o.emotion !== undefined) emotion = o.emotion;
1025
- else if (changed) emotion = st.emotion;
1026
- if (o.intensity !== undefined) emotionAmt = o.intensity;
1027
- idle.setProfile(st.idle);
1028
- // SVG's desaturation filter is a legacy renderer detail. A generic rig
1029
- // receives the same state pose and may express degradation its own way.
1030
- if (face) {
1031
- face.svg.style.filter = st.filter || '';
1032
- face.svg.style.transition = 'filter .5s ease';
1033
- }
1034
- // The eyes wait out a flicker between gap states (GAP_SETTLE), and a
1035
- // repeat of the state still settling leaves it settling.
1036
- const deferrable = !o.gaze && !o.keepGaze && GAP_STATES.has(gazeState) && GAP_STATES.has(name);
1037
- if (deferrable && changed) settleAt = name === gazeState ? 0 : elapsed + GAP_SETTLE;
1038
- else if (!(deferrable && settleAt)) { settleAt = 0; enterGaze(name, o, changed); }
1039
- // The floor has come back to the user: the face receives it (prosody.js
1040
- // `listen`). It is a state change rather than a VAD event on purpose —
1041
- // what is being welcomed is the turn, and the server is the one that knows
1042
- // a turn has changed hands.
1043
- if (changed && name === 'LISTENING') prosody.listen();
1044
- if (changed) emit('state', name);
1045
- return api;
1046
- }
1047
-
1048
- // Whether the eyes were last aimed by somebody who meant it — the public
1049
- // `setGaze`, or a performance's `gaze` verb — rather than by the state's own
1050
- // schedule. It is the one thing that outranks a state's `gaze`, and it is
1051
- // why `speak()` can take the eyes back without overriding a caller.
1052
- let gazeExplicit = false;
1053
-
1054
- /** The state `name` takes the gaze: its target and its schedules. */
1055
- function enterGaze(name, o, blink) {
1056
- gazeState = name;
1057
- const st = states[name];
1058
- const gl = st.glance;
1059
- // A state with an `opening` enters as though its check-in on the user is
1060
- // already under way, and leaves it when that runs out.
1061
- if (!o.keepGaze) { setGaze(o.gaze || (gl && gl.opening ? gl.to : st.gaze)); gazeExplicit = false; }
1062
- // Arm every scheduler fresh, so entering a state never fires a timestamp
1063
- // left over from the last one — the wander in particular, which used to
1064
- // pick a new target on the first frame and override the state's own gaze.
1065
- glanceUntil = gl && gl.opening && !o.gaze && !o.keepGaze ? elapsed + rand(gl.opening) : 0;
1066
- glanceAt = elapsed + (gl ? rand(gl.every) : 0);
1067
- wanderAt = elapsed + (st.wander ? rand(st.wander.every) : 0);
1068
- driftAt = elapsed + (st.read ? rand(st.read.fix) : 0);
1069
- readCol = readCols = readRow = readRows = 0;
1070
- dartAt = dartBrowUntil = 0;
1071
- dart.x = dart.y = 0;
1072
- if (blink) idle.blink(true);
1073
- }
1074
-
1075
- function setEmotion(name, intensity = 1) { emotion = name; emotionAmt = intensity; return api; }
1076
-
1077
- /** @param {string} name @param {{x:number,y:number}} [custom] normalized -1..1
1078
- * @param {boolean} [blink] the mixer's own say over the evoked blink; see GazeLayer.set */
1079
- function setGaze(name, custom, blink) {
1080
- gazeName = GAZE_TARGETS[name] ? name : 'USER';
1081
- gazeCustom = custom || null;
1082
- applyGaze(blink);
1083
- return api;
1084
- }
1085
-
1086
- /**
1087
- * @param {object} o
1088
- * @param {Array<{t:number,v:string,i?:number}>} o.cues
1089
- * @param {HTMLMediaElement} [o.audio] preferred clock source
1090
- * @param {() => number} [o.clock] custom ms clock, if you drive audio yourself
1091
- */
1092
- function speak(o = {}) {
1093
- // Speech owns the mouth in the mixer, but a server action still gets to
1094
- // complete its physical landing on the other channels. Do not cancel it
1095
- // here: a hand or head cannot disappear simply because playout began.
1096
- // A new clock is a new turn. The same clock handed back is the accurate
1097
- // leg rewriting this one (AvatarClient re-speaks a splice), and that must
1098
- // not roll the turn-start look a second time.
1099
- const newTurn = !o.clock || o.clock !== speakClock;
1100
- speakStart = performance.now();
1101
- speakClock = o.clock
1102
- ? o.clock
1103
- : o.audio
1104
- ? () => o.audio.currentTime * 1000
1105
- : () => performance.now() - speakStart;
1106
- speech.start(o.cues || [], speakClock);
1107
- prosody.reset(newTurn);
1108
- // The eyes come back to the user when the audio starts: a reply arriving
1109
- // while THINKING was looking away would otherwise spend its whole turn
1110
- // aimed off the user, because nothing in SPEAKING retargets. `keepGaze` is
1111
- // for the one caller that means it — a performance that aimed the eyes with
1112
- // its own `gaze` verb keeps them, an instruction and not a leftover.
1113
- if (stateName !== 'SPEAKING') setState('SPEAKING', { keepGaze: gazeExplicit });
1114
- if (o.audio && o.audio.paused) o.audio.play().catch(() => {});
1115
- return api;
1116
- }
1117
-
1118
- /**
1119
- * "The user may be checking whether I am with them — hold their eyes."
1120
- *
1121
- * This is the widget's half of the **gaze window**. In face-to-face talk a
1122
- * speaker periodically looks at the listener, mutual gaze is established, the
1123
- * listener responds inside that window, and the speaker looks away again
1124
- * (Bavelas, Coates & Johnson 2002).
1125
- *
1126
- * We cannot see the user, so we cannot observe the window opening. What a
1127
- * caller *can* do is name the moments that co-occur with it — a mid-turn
1128
- * pause, a tag question ("...right?", "you know?"), a completed clause with
1129
- * the turn analyzer's completion probability high, the user answering a
1130
- * question the bot asked. For `ms` the face stops averting and holds the
1131
- * user, which is the prerequisite for any response to be *seen*.
1132
- *
1133
- * @param {number} [ms=1200] how long to hold. Binetti (N=498) puts preferred
1134
- * mutual gaze at 3295 ± 706 ms, so this is a fraction of the ceiling.
1135
- */
1136
- function attend(ms = 1200) {
1137
- attendUntil = Math.max(attendUntil, elapsed + ms / 1000);
1138
- return api;
1139
- }
1140
-
1141
- function pushCues(cues) { speech.push(cues); return api; }
1142
-
1143
- function stopSpeaking() { speech.stop(); return api; }
1144
-
1145
- function interject(id) {
1146
- const c = INTERNAL_CLIPS[id];
1147
- if (!c) throw new Error(`unknown interjection: ${id}`);
1148
- clip.play(c, c.audioEl);
1149
- return api;
1150
- }
1151
-
1152
- /**
1153
- * A hand gesture: the hand at the frame edge, plus the face half that makes it
1154
- * belong to somebody.
1155
- *
1156
- * The face half is not a convenience — a hand rising to the jaw over a head
1157
- * and shoulders sitting perfectly still is a cut-out, not a gesture. Each
1158
- * entry in HAND_GESTURES names the matching semantic face action; firing it here is the
1159
- * library composing two authored things, not the client inventing motion.
1160
- *
1161
- * On an avatar mounted with `hand: false` this internal helper degrades to
1162
- * the face action alone.
1163
- */
1164
- function gesture(id) {
1165
- const def = HAND_GESTURES[id];
1166
- if (!def) throw new Error(`unknown hand gesture: ${id}`);
1167
- if (hand) hand.play(id, elapsed * 1000);
1168
- if (def.face) interject(def.face);
1169
- return api;
1170
- }
1171
-
1172
- /** One self-completing server action. State continues to resolve underneath;
1173
- * face and hand layers queue their next movement so an in-flight physical
1174
- * gesture always gets to land. */
1175
- function action(wireId) {
1176
- // `ACKNOWLEDGE` is the whole backchannel family in one word, and which of
1177
- // them a face makes is a rendering decision the server is not holding the
1178
- // drawing for. The floor decides: a nod while the user still has it is a
1179
- // continuer, "go on"; once they have stopped it is a receipt, and a nod
1180
- // there reads as agreement with whatever they just said. Both are only ever
1181
- // this explicit action — nothing here starts one on its own.
1182
- const id = wireId === 'ACKNOWLEDGE'
1183
- ? (engagement.speaking ? 'ACK_NOD' : 'ACK_RECEIVE')
1184
- : wireId;
1185
- const handDef = HAND_GESTURES[id];
1186
- if (handDef) {
1187
- startHandAction(id, handDef);
1188
- if (handDef.face) {
1189
- const faceClip = actionShapes[handDef.face] || ACTIONS[handDef.face];
1190
- if (faceClip) clip.play(faceClip, faceClip.audioEl, { queue: true });
1191
- }
1192
- return api;
1193
- }
1194
- // Own renderings first, always. An avatar may *add* to what a server can
1195
- // ask for and may never redefine a core intent: `ACKNOWLEDGE` has to mean
1196
- // the same thing on every face or it is not a protocol. It may draw that
1197
- // meaning in its own shape (`actionShapes`), which is a rendering, not a
1198
- // redefinition — the id and the intent are fixed before this line.
1199
- //
1200
- // Unknown is a no-op, not a throw. The wire's action vocabulary is open, so
1201
- // a server asking this face for a motion it does not have is the expected
1202
- // case and not somebody's broken build — the same forward-compat rule an
1203
- // unknown `cmd` gets. This is also where a sequence used to arrive by its
1204
- // own method; one open vocabulary means one door.
1205
- const faceClip = actionShapes[id] || ACTIONS[id] || sequences[id];
1206
- if (!faceClip) return api;
1207
- clip.play(faceClip, faceClip.audioEl, { queue: true });
1208
- // The acknowledgements smile. Only these, and only ever because the
1209
- // server sent one: a smile the renderer timed for itself would be an
1210
- // acknowledgement nobody sent.
1211
- if (id === 'ACK_NOD' || id === 'ACK_RECEIVE') prosody.acknowledge();
1212
- // Now that warmth rides over clips, the reply's opening smile would
1213
- // otherwise carry on through the interrupted face.
1214
- if (id === 'RESPONSE_INTERRUPTED') prosody.cool();
1215
- return api;
1216
- }
1217
-
1218
- function startHandAction(id, def) {
1219
- const gesture = HAND_ACTION_TO_FRAME_GESTURE[id];
1220
- if (!gesture) return;
1221
- if (handAction) {
1222
- if (handAction.id !== id && !handQueue.some((item) => item.id === id)) {
1223
- handQueue.push({ id, def, gesture });
1224
- }
1225
- return;
1226
- }
1227
- handAction = { id, def, gesture, start: elapsed * 1000 };
1228
- }
1229
-
1230
- function updateHandAction(nowMs) {
1231
- if (!handAction) return null;
1232
- const progress = (nowMs - handAction.start) / handAction.def.dur;
1233
- if (progress >= 1) {
1234
- const done = handAction;
1235
- const next = handQueue.shift();
1236
- handAction = next ? { ...next, start: nowMs } : null;
1237
- emit('gestureEnd', done.id);
1238
- return handAction ? { gesture: handAction.gesture, progress: 0, side: handSide } : null;
1239
- }
1240
- return { gesture: handAction.gesture, progress: Math.max(0, progress), side: handSide };
1241
- }
1242
-
1243
- function setUserSpeaking(b) { engagement.setUserSpeaking(b); return api; }
1244
-
1245
- // What one action does when its moment comes. Enum validity is checked here,
1246
- // where the enums live: a bad value warns and is skipped, because one stale
1247
- // verb must never take down the performance around it.
1248
- function dispatchAction(a) {
1249
- try {
1250
- // `state` defaults to keepGaze — a timeline that wants the gaze moved
1251
- // says so with a `gaze` verb at the moment it means, which is how every
1252
- // composed turn in the demo already behaves.
1253
- if (a.do === 'state') setState(a.name, { keepGaze: a.keepGaze !== false });
1254
- else if (a.do === 'emotion') setEmotion(a.name, a.i ?? 1);
1255
- else if (a.do === 'gaze') { setGaze(a.name); gazeExplicit = true; }
1256
- else if (a.do === 'action') action(a.id);
1257
- } catch (e) {
1258
- console.warn(`perform: ${a.do} at ${a.t}ms skipped — ${e.message}`);
1259
- }
1260
- }
1261
-
1262
- let performGen = 0;
1263
-
1264
- /**
1265
- * @param {Array<{t: number, do: string}>} actions
1266
- * @param {{audio?: HTMLMediaElement, clock?: () => number,
1267
- * onAction?: (a: object) => void}} [o]
1268
- * @returns {{stop: () => void}} stop() cancels the *future* of this
1269
- * performance only: an in-flight interjection finishes, a live cue track
1270
- * is untouched, and 'performEnd' does not fire. A handle whose
1271
- * performance was already replaced by a newer perform() is a no-op.
1272
- */
1273
- function perform(actions, o = {}) {
1274
- const start = performance.now();
1275
- const clock = o.clock
1276
- ? o.clock
1277
- : o.audio
1278
- ? () => o.audio.currentTime * 1000
1279
- : () => performance.now() - start;
1280
- performTrack.onAction = (a) => { dispatchAction(a); if (o.onAction) o.onAction(a); };
1281
- const gen = ++performGen;
1282
- performTrack.start(actions, clock);
1283
- return { stop: () => { if (gen === performGen) performTrack.stop(); } };
1284
- }
1285
-
1286
- const api = {
1287
- setState, setEmotion, setGaze: (name, custom) => { gazeExplicit = true; return setGaze(name, custom); }, speak, pushCues, stopSpeaking, attend,
1288
- action, perform,
1289
- /** Which hand the character gestures with: +1 the viewer's right (its own
1290
- * left), -1 the other. Both are anatomically real — the thumb splays away
1291
- * from the body either way — so this is a character choice, not a fix. */
1292
- setHandSide: (d) => { handSide = d === -1 ? 'left' : 'right'; return api; },
1293
- setUserSpeaking,
1294
- /** Articulation gain: 1 is the VISEME_SHAPES table as authored. */
1295
- setMouthGain: (g) => { mouthGain = g; return api; },
1296
- get mouthGain() { return mouthGain; },
1297
- /** Gesture gain: scales every clip delta. 1 is the timelines as authored. */
1298
- setGestureGain: (g) => { gestureGain = g; return api; },
1299
- get gestureGain() { return gestureGain; },
1300
- /** Idle body-motion gain: 1 is the liveness layer as authored, 0 freezes it. */
1301
- setMotionGain: (g) => { idle.gain = g; return api; },
1302
- get motionGain() { return idle.gain; },
1303
- blink: () => { idle.blink(); return api; },
1304
- /** Advance one frame by hand. Only meaningful under `{manual: true}`;
1305
- * fixed-dt stepping is what makes a motion render reproducible. */
1306
- step: (dt) => { elapsed += dt; step(dt, dt * 1000); return api; },
1307
- /** Direct parameter injection — for tuning UIs, not production. */
1308
- setOverrides: (o) => { overrides = o; return api; },
1309
- on: (ev, fn) => { (listeners[ev] || (listeners[ev] = [])).push(fn); return api; },
1310
- get state() { return stateName; },
1311
- get emotion() { return emotion; },
1312
- get gaze() { return gazeName; },
1313
- get speaking() { return speech.playing; },
1314
- get performing() { return performTrack.playing; },
1315
- get clip() { return clip.id; },
1316
- /** Semantic hand gesture in flight, independent of renderer capability. */
1317
- get gesturing() { return handAction ? handAction.id : null; },
1318
- get params() { return shown; },
1319
- get userSpeaking() { return engagement.speaking; },
1320
- // Legacy SVG inspection fields. New AvatarRig implementations should not
1321
- // rely on or provide them; the renderer-neutral contract is apply/destroy.
1322
- svg: face?.svg || null,
1323
- meta: meta || null,
1324
- /** The mounted rig's palette, merged with any `opts.theme` overrides. A
1325
- * host that has to paint anything *around* the widget — a tile margin, a
1326
- * page behind a transparent mount — needs the same colours the drawing
1327
- * used, and guessing them per avatar is how the two drift apart. */
1328
- theme: face?.theme,
1329
- destroy() {
1330
- cancelAnimationFrame(raf);
1331
- rig.destroy();
1332
- },
1333
- };
1334
-
1335
- setState('IDLE');
1336
- if (!manual) raf = requestAnimationFrame(frame);
1337
- return api;
1338
- }
1339
-
1340
- // The parameter space itself. A renderer-neutral rig has to answer "what is
1341
- // this channel's neutral value" before it can map the channel onto whatever it
1342
- // controls — a morph target's influence is `(pose - rest) / (1 - rest)`, and a
1343
- // rig that hard-codes those rests has quietly forked `params.js`.
1344
- export { REST, CHANNELS, RANGE, TAU, clamp, approach, makeParams } from './params.js';
1345
- // The smoothing law itself, for a tool that steps the rig by hand. A
1346
- // filmstrip that reimplemented `approach` would be measuring its own copy of
1347
- // the thing under test — and the smoothing between keyframes is what the face
1348
- // actually does (docs/internal-mixer.md § Smoothing).
1349
- // A pose with no mixer above it: `makeParams(overrides)` fills the rests,
1350
- // `avatarFrame` wraps it, `createSvgRig(face).apply` draws it. That is the whole
1351
- // path an instrument needs to hold a face at one named extreme — no clock, no
1352
- // client, no animation — and the reason it is exported is that a pose sheet
1353
- // that cannot reach it forks the channel rests instead.
1354
- export { avatarFrame, createSvgRig } from './rig.js';
1355
- export { ACTION_IDS, ACTIONS, attachAudio } from './interjections.js';
1356
- // The full authoring catalogue and the player that steps it. Not a server
1357
- // vocabulary — `ACTIONS` is that, and most of these clips are reachable only
1358
- // from inside the mixer. They are exported for the filmstrip instrument,
1359
- // which lays one clip out as frames and therefore has to drive a real
1360
- // `ClipPlayer` rather than re-sample its keys.
1361
- export { INTERNAL_CLIPS } from './interjections.js';
1362
- export { ClipPlayer } from './clips.js';
1363
- export { GAZE_NAMES, GAZE_TARGETS } from './gaze.js';
1364
- export { normalizeActions } from './perform.js';
1365
- export { checkHandFraming } from './hand.js';
1366
- export { EMOTION_NAMES, emotionPose } from './emotions.js';
1367
- // The mouth clock travels with the rest of it. Someone has to turn a cue array
1368
- // plus a clock into "which letter is on screen right now", every renderer needs
1369
- // exactly that, and none of them should write it twice — so it is a plain class
1370
- // to construct, not a contract to implement.
1371
- export {
1372
- VISEME_LETTERS, VISEME_SHAPES, VisemeTrack, shapeFor, JAW_OF_OPEN, SILENT,
1373
- normalizeCues, textToCues,
1374
- ARPABET_TO_VISEME, AZURE_VISEME_TO_LETTER, LEAD_MS,
1375
- } from './visemes.js';
1376
- // No THEME re-export: each face module owns its palette, and `api.theme` is
1377
- // the mounted avatar's. A single barrel THEME was one rig's palette wearing a
1378
- // public name — misleading the moment that rig stopped being the default.