@voqalize/avatar 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-CC-BY-4.0 +396 -0
- package/README.md +65 -17
- package/assets/README.md +48 -0
- package/assets/tanya.glb +0 -0
- package/assets/tara.glb +0 -0
- package/assets/tushar.glb +0 -0
- package/client/AvatarClient.ts +337 -42
- package/client/arjun.ts +2 -0
- package/client/createAvatar.ts +38 -0
- package/client/index.ts +6 -1
- package/client/internal.ts +31 -3
- package/client/ishita.ts +2 -0
- package/client/kabir.ts +2 -0
- package/client/meera.ts +2 -0
- package/client/naina.ts +2 -0
- package/client/playout.ts +95 -0
- package/client/supports.ts +28 -0
- package/client/three/assets.ts +24 -0
- package/client/three/budgets.ts +27 -0
- package/client/three/holds.ts +64 -0
- package/client/three/internal.ts +54 -0
- package/client/three/motion-limits.json +116 -0
- package/client/three/sequences.ts +281 -0
- package/client/three/tanya.ts +69 -0
- package/client/three/tara-rig.ts +1562 -0
- package/client/three/tara.ts +76 -0
- package/client/three/tushar.ts +65 -0
- package/client/types.ts +109 -20
- package/client/vikram.ts +2 -0
- package/dist/AvatarClient.d.ts +89 -11
- package/dist/AvatarClient.d.ts.map +1 -1
- package/dist/AvatarClient.js +297 -35
- package/dist/AvatarClient.js.map +1 -1
- package/dist/arjun.d.ts +1 -0
- package/dist/arjun.d.ts.map +1 -1
- package/dist/arjun.js +2 -0
- package/dist/arjun.js.map +1 -1
- package/dist/createAvatar.d.ts +36 -0
- package/dist/createAvatar.d.ts.map +1 -1
- package/dist/createAvatar.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +9 -5
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +13 -5
- package/dist/internal.js.map +1 -1
- package/dist/ishita.d.ts +1 -0
- package/dist/ishita.d.ts.map +1 -1
- package/dist/ishita.js +2 -0
- package/dist/ishita.js.map +1 -1
- package/dist/kabir.d.ts +1 -0
- package/dist/kabir.d.ts.map +1 -1
- package/dist/kabir.js +2 -0
- package/dist/kabir.js.map +1 -1
- package/dist/meera.d.ts +1 -0
- package/dist/meera.d.ts.map +1 -1
- package/dist/meera.js +2 -0
- package/dist/meera.js.map +1 -1
- package/dist/naina.d.ts +1 -0
- package/dist/naina.d.ts.map +1 -1
- package/dist/naina.js +2 -0
- package/dist/naina.js.map +1 -1
- package/dist/playout.d.ts +42 -0
- package/dist/playout.d.ts.map +1 -0
- package/dist/playout.js +85 -0
- package/dist/playout.js.map +1 -0
- package/dist/supports.d.ts +20 -0
- package/dist/supports.d.ts.map +1 -0
- package/dist/supports.js +26 -0
- package/dist/supports.js.map +1 -0
- package/dist/three/assets.d.ts +24 -0
- package/dist/three/assets.d.ts.map +1 -0
- package/dist/three/assets.js +22 -0
- package/dist/three/assets.js.map +1 -0
- package/dist/three/budgets.d.ts +17 -0
- package/dist/three/budgets.d.ts.map +1 -0
- package/dist/three/budgets.js +21 -0
- package/dist/three/budgets.js.map +1 -0
- package/dist/three/holds.d.ts +33 -0
- package/dist/three/holds.d.ts.map +1 -0
- package/dist/three/holds.js +56 -0
- package/dist/three/holds.js.map +1 -0
- package/dist/three/internal.d.ts +52 -0
- package/dist/three/internal.d.ts.map +1 -0
- package/dist/three/internal.js +49 -0
- package/dist/three/internal.js.map +1 -0
- package/dist/three/motion-limits.json +116 -0
- package/dist/three/sequences.d.ts +71 -0
- package/dist/three/sequences.d.ts.map +1 -0
- package/dist/three/sequences.js +262 -0
- package/dist/three/sequences.js.map +1 -0
- package/dist/three/tanya.d.ts +35 -0
- package/dist/three/tanya.d.ts.map +1 -0
- package/dist/three/tanya.js +51 -0
- package/dist/three/tanya.js.map +1 -0
- package/dist/three/tara-rig.d.ts +403 -0
- package/dist/three/tara-rig.d.ts.map +1 -0
- package/dist/three/tara-rig.js +1500 -0
- package/dist/three/tara-rig.js.map +1 -0
- package/dist/three/tara.d.ts +35 -0
- package/dist/three/tara.d.ts.map +1 -0
- package/dist/three/tara.js +58 -0
- package/dist/three/tara.js.map +1 -0
- package/dist/three/tushar.d.ts +31 -0
- package/dist/three/tushar.d.ts.map +1 -0
- package/dist/three/tushar.js +47 -0
- package/dist/three/tushar.js.map +1 -0
- package/dist/types.d.ts +65 -10
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +58 -11
- package/dist/types.js.map +1 -1
- package/dist/vikram.d.ts +1 -0
- package/dist/vikram.d.ts.map +1 -1
- package/dist/vikram.js +2 -0
- package/dist/vikram.js.map +1 -1
- package/package.json +34 -6
- package/src/avatar.d.ts +59 -2
- package/src/avatar.js +743 -94
- package/src/behavior.d.ts +28 -12
- package/src/behavior.js +54 -42
- package/src/canvas/create-rig.js +1 -1
- package/src/canvas/src/live.js +2 -2
- package/src/conformance.js +1 -1
- package/src/emotions.js +4 -2
- package/src/gaze.js +334 -73
- package/src/head.js +134 -0
- package/src/idle.js +210 -19
- package/src/interjections.js +70 -26
- package/src/prosody.js +647 -0
- package/src/visemes.js +85 -15
package/src/avatar.js
CHANGED
|
@@ -30,47 +30,120 @@
|
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
import { REST, CHANNELS, TAU, RANGE, GROUPS, clamp, approach } from './params.js';
|
|
33
|
-
import { emotionPose } from './emotions.js';
|
|
33
|
+
import { EMOTIONS, emotionPose } from './emotions.js';
|
|
34
34
|
import { GazeLayer, GAZE_TARGETS, AVERSION } from './gaze.js';
|
|
35
35
|
import { IdleLayer, ListeningEngine } from './idle.js';
|
|
36
36
|
import { ClipPlayer } from './clips.js';
|
|
37
37
|
import { ACTIONS, INTERNAL_CLIPS } from './interjections.js';
|
|
38
38
|
import { VisemeTrack, shapeFor, SILENT } from './visemes.js';
|
|
39
|
+
import { SpeechProsody, UNCALIBRATED_HEAD_GAIN } from './prosody.js';
|
|
40
|
+
import { HEAD_AXES, soften } from './head.js';
|
|
39
41
|
import { PerformTrack } from './perform.js';
|
|
40
42
|
import { createHand, HAND_GESTURES, HAND_ACTION_TO_FRAME_GESTURE } from './hand.js';
|
|
41
43
|
import { avatarFrame, createSvgRig } from './rig.js';
|
|
42
44
|
|
|
45
|
+
const rand = ([a, b]) => a + Math.random() * (b - a);
|
|
46
|
+
const pick = (xs) => xs[(Math.random() * xs.length) | 0];
|
|
47
|
+
|
|
43
48
|
// Each state's `idle` is a profile for the liveness layer (see DEFAULT_PROFILE
|
|
44
49
|
// in idle.js). Blink gaps come from docs/research-biomechanics.md §5: the rate
|
|
45
50
|
// alone separates listening (~16/min) from thinking (~25/min) from visually
|
|
46
|
-
// busy (~9/min), and it is the cheapest state signal the rig has.
|
|
51
|
+
// busy (~9/min), and it is the cheapest state signal the rig has. The gap is
|
|
52
|
+
// the budget for every blink, paired ones included (about one in six), so it
|
|
53
|
+
// sits a little longer than 60/rate.
|
|
47
54
|
export const STATES = {
|
|
48
|
-
// Idle means present
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
55
|
+
// Idle means present and relaxed, not attending. It is also the face before
|
|
56
|
+
// a call connects — the first one anyone sees — so it rests on the user and
|
|
57
|
+
// leaves for soft, unhurried looks (AVERSION.IDLE) rather than parking the
|
|
58
|
+
// eyes somewhere. It used to open on AWAY_THINKING and wander between it,
|
|
59
|
+
// AWAY_RIGHT and NOTES from the first frame: a face that loads looking up
|
|
60
|
+
// and to the left, at nothing, reads as broken rather than idle. What keeps
|
|
61
|
+
// it apart from LISTENING is the longer, lazier looks, the slower scan and
|
|
62
|
+
// the missing engagement lean, not refusing to look at the user.
|
|
63
|
+
IDLE: { gaze: 'USER', emotion: 'neutral', engagement: false,
|
|
64
|
+
aversion: 'IDLE', scan: [0.8, 2.0],
|
|
65
|
+
idle: { sway: 0.72, blinkGap: [3.6, 5.2] } },
|
|
54
66
|
// `aversion` is why this state does not stare. Continuous eye contact is not
|
|
55
67
|
// the attentive pose it looks like — it is a demand for more talk (Rossano)
|
|
56
68
|
// and it measures as *tense*, not attentive (Wang & Gratch). See AVERSION in
|
|
57
69
|
// gaze.js for the numbers; the mixer holds it off near a turn boundary.
|
|
70
|
+
// The blink timer runs slower than the ~17/min it lands on because a sixth
|
|
71
|
+
// of timed blinks come as a pair; at 3.2-4.4 s it measured 20-23/min.
|
|
58
72
|
LISTENING: { gaze: 'USER', emotion: 'neutral', engagement: true,
|
|
59
73
|
aversion: 'LISTEN',
|
|
60
|
-
idle: { sway: 1.0, blinkGap: [3.
|
|
74
|
+
idle: { sway: 1.0, blinkGap: [3.6, 4.8] },
|
|
61
75
|
pose: { browRaiseL: 0.06, browRaiseR: 0.06, lidL: -0.04, lidR: -0.04 } },
|
|
62
76
|
// Faster, shallower breath is the measured cognitive-load signature, and the
|
|
63
77
|
// occasional dead-still hold is the strongest "working on it" cue a rig this
|
|
64
|
-
// simple can make — deliberate stillness, not more motion. The
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
|
|
78
|
+
// simple can make — deliberate stillness, not more motion. The look away is
|
|
79
|
+
// Andrist's cognitive aversion (§4.2): one look held ~3.5 s (SD 1.26) that
|
|
80
|
+
// drifts a little where it sits, a check-in with the user, then the next.
|
|
81
|
+
//
|
|
82
|
+
// In a call this state mostly lasts under two seconds — the gap between the
|
|
83
|
+
// user finishing and the reply starting — so its opening is what gets seen,
|
|
84
|
+
// and the opening used to be wrong. The eyes left the instant the user
|
|
85
|
+
// stopped, down and to the left, every turn: a listener who drops their
|
|
86
|
+
// eyes the moment you finish reads as ashamed or done with you. Now the
|
|
87
|
+
// eyes hold the user for `opening` first (a person takes in the end of
|
|
88
|
+
// what was said before they go to think), and a fast reply never looks
|
|
89
|
+
// away at all. The look is mostly up-and-aside or level-aside: that is the
|
|
90
|
+
// one audiences read as thinking, whatever the measured split, and down
|
|
91
|
+
// is kept to a fifth of looks because on a face this real it reads as
|
|
92
|
+
// downcast. The pose takes back `thoughtful`'s lid drop: a thinking face is
|
|
93
|
+
// alert, and the two together measured past the 0.15 that reads drowsy.
|
|
94
|
+
// The handoff to SPEAKING is in setState: a reply that starts mid-look
|
|
95
|
+
// finishes the look instead of snapping back.
|
|
96
|
+
THINKING: { gaze: 'AWAY_SIDE', emotion: 'thoughtful', engagement: false,
|
|
97
|
+
// Fixational jumps rare and small: a thinker's eyes rest
|
|
98
|
+
// where they land. At the default gap the look jittered
|
|
99
|
+
// about its spot every second, and with a drift on top
|
|
100
|
+
// the eyes never held anything — searching the wall, not
|
|
101
|
+
// thinking.
|
|
102
|
+
scan: [1.2, 2.6, 0.35],
|
|
103
|
+
idle: { sway: 0.7, blinkGap: [2.6, 3.2], slowBlink: 0.3,
|
|
104
|
+
breathRate: 1.18, breathAmp: 0.7,
|
|
70
105
|
hold: { every: [4.5, 9.0], dur: [0.8, 1.5] } },
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
106
|
+
// `stick`: people have a side they look to when they
|
|
107
|
+
// think and keep to it (Day; Kinsbourne's lateral eye
|
|
108
|
+
// movements) — one who alternates sides on every look is
|
|
109
|
+
// scanning the room. `dart`: now and then, once in a
|
|
110
|
+
// look, the eyes move on to a second spot with a flick of
|
|
111
|
+
// the brows, which is the thought moving; the random
|
|
112
|
+
// drift this replaces never stopped and read as roving.
|
|
113
|
+
// `blinkTo`: the look back to the user always blinks —
|
|
114
|
+
// the re-engagement is the beat to see.
|
|
115
|
+
glance: { to: 'USER', opening: [0.35, 0.7], every: [2.2, 4.4], hold: [0.8, 1.3],
|
|
116
|
+
back: ['AWAY_THINKING', 'AWAY_THINKING', 'AWAY_THINKING',
|
|
117
|
+
'AWAY_SIDE', 'AWAY_SIDE', 'AWAY_SIDE',
|
|
118
|
+
'AWAY_RIGHT', 'AWAY_RIGHT', 'AWAY_DOWN', 'AWAY_DOWN'],
|
|
119
|
+
stick: 0.55, dart: { p: 0.4, mag: 0.08, brow: 0.12 }, blinkTo: true },
|
|
120
|
+
// **The chin comes up, and that is the state's loudest
|
|
121
|
+
// signal below the eyes.** Thinking is the one stretch
|
|
122
|
+
// of a call where the avatar owes the user visible
|
|
123
|
+
// feedback and has no mouth to give it with: the eyes
|
|
124
|
+
// are off the user by design, so without the head there
|
|
125
|
+
// is nothing left moving that says *working on it*
|
|
126
|
+
// rather than *gone*. A head that tips back as the gaze
|
|
127
|
+
// leaves is also what the research calls a swing-up —
|
|
128
|
+
// "nodding with swinging up is regarded to reflect a
|
|
129
|
+
// cognitive shift in the listener"
|
|
130
|
+
// (research-biomechanics.md § 3.3) — so the same
|
|
131
|
+
// gesture that means "ah, I see" at the end of a nod
|
|
132
|
+
// means "let me think" at the start of a pause.
|
|
133
|
+
//
|
|
134
|
+
// It rides the looks that go up, as the head's share of
|
|
135
|
+
// them, and not the state. Held as a pose it lifted the
|
|
136
|
+
// chin on the level and downward looks too and on every
|
|
137
|
+
// check-in, so the user was looked at down the nose, and
|
|
138
|
+
// eyes drawn inside a head tipped back aimed every look
|
|
139
|
+
// meant to be level at the ceiling.
|
|
140
|
+
pose: { lidL: -0.10, lidR: -0.10 } },
|
|
141
|
+
// Eyes on the user, with the speaker's own looks away (AVERSION.SPEAK):
|
|
142
|
+
// a planning look at the start of most turns, a short one every ~5 s at a
|
|
143
|
+
// phrase boundary, and none as the turn ends. A speaker who never looks
|
|
144
|
+
// away is staring the listener down.
|
|
145
|
+
SPEAKING: { gaze: 'USER', emotion: 'neutral', aversion: 'SPEAK',
|
|
146
|
+
idle: { sway: 0.55 }, engagement: false },
|
|
74
147
|
REVIEWING_SCREEN: { gaze: 'SCREEN_CENTER', emotion: 'thoughtful', engagement: false,
|
|
75
148
|
idle: { sway: 0.8, blinkGap: [4.0, 6.5] },
|
|
76
149
|
wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_RIGHT', 'SCREEN_TOP', 'SCREEN_WORK'],
|
|
@@ -99,9 +172,12 @@ export const STATES = {
|
|
|
99
172
|
// extremes row, not from what a fleshed rig would need: brows DOWN
|
|
100
173
|
// (corrugator effort, not the browInner worry-lift), a real squint, and
|
|
101
174
|
// the resting smile pressed flat — nobody smiles while straining to hear.
|
|
175
|
+
// The squint is the lower lid's (AU7), with no upper lid dropped on top:
|
|
176
|
+
// the two together narrowed a photographic eye from above and read as
|
|
177
|
+
// drowsy, and a line face's squint is its lower lid already.
|
|
102
178
|
pose: {
|
|
103
179
|
torsoLean: 0.70, headPitch: 0.10,
|
|
104
|
-
|
|
180
|
+
squintL: 0.75, squintR: 0.75,
|
|
105
181
|
browRaiseL: -0.45, browRaiseR: -0.45, browInnerL: 0.15, browInnerR: 0.12,
|
|
106
182
|
mouthPress: 0.45, mouthCornerL: -0.22, mouthCornerR: -0.22,
|
|
107
183
|
},
|
|
@@ -135,23 +211,47 @@ export const STATES = {
|
|
|
135
211
|
},
|
|
136
212
|
// --- application state ---------------------------------------------------
|
|
137
213
|
// "Momentarily busy on the thing you asked for." No hands in frame, so the
|
|
138
|
-
// whole read comes from four cheap cues (docs/research-biomechanics.md §6.4
|
|
139
|
-
// gaze
|
|
140
|
-
// (~9/min), shoulders slightly raised
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
//
|
|
214
|
+
// whole read comes from four cheap cues (docs/research-biomechanics.md §6.4,
|
|
215
|
+
// recommendation 19): gaze down on one stable target with a reading scan,
|
|
216
|
+
// blinks suppressed to task-focus rate (~9/min), shoulders slightly raised
|
|
217
|
+
// and *held* with brief micro-freezes, and a glance back up to the user.
|
|
218
|
+
// The glance is the tell that the user has not been forgotten; without it,
|
|
219
|
+
// busy is just absent, and much more often than that it is fidgeting. There
|
|
220
|
+
// is no shoulder rhythm: at 2.2 Hz it ran over the 1.5 Hz ceiling every
|
|
221
|
+
// other motion here keeps, and held is what §6.4 describes. The trunk sits
|
|
222
|
+
// out the glance — checking on someone is a look, not a turn toward them.
|
|
144
223
|
WORKING: {
|
|
145
|
-
//
|
|
146
|
-
//
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
224
|
+
// OWN_SCREEN: eyes down at the agent's own display, head nearly level.
|
|
225
|
+
// This was SCREEN_WORK, which turned the head down and to the left and
|
|
226
|
+
// held it there for the whole task, with a random drift too small to see
|
|
227
|
+
// on it — a head turned away from the user, staring at a point. `read` is
|
|
228
|
+
// what makes it busy instead: fixations stepping along a line, a sweep
|
|
229
|
+
// back, the next line, a pause on the result (see readStep). The lids
|
|
230
|
+
// take back most of the down look's follow so the eyes stay awake.
|
|
231
|
+
gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
|
|
232
|
+
scan: [0.9, 2.0, 0.12],
|
|
233
|
+
// §6.4's ~9/min is a count of blinks, and a sixth of timed blinks here
|
|
234
|
+
// come as a pair, so the timer runs slower than the table's 6-7.5 s gap
|
|
235
|
+
// to land on it; at 6-7.5 s it measured 12-13/min, which is not focus.
|
|
236
|
+
idle: { sway: 0.5, blinkGap: [6.8, 8.6], breathRate: 1.05, breathAmp: 0.8,
|
|
237
|
+
hold: { every: [5.0, 9.0], dur: [0.6, 1.1] } },
|
|
238
|
+
// The look up to check is brows-first and blinkless, the lids leading it
|
|
239
|
+
// (their 18 ms against the eye's 32 ms); the blink comes on the way back
|
|
240
|
+
// down, where the eyes re-engage with the work. A blink going up hid the
|
|
241
|
+
// one moment the user is actually looked at.
|
|
242
|
+
glance: { to: 'USER', every: [6, 11], hold: [0.8, 1.2], brow: 0.10,
|
|
243
|
+
blinkTo: false, blinkBack: true },
|
|
244
|
+
// Fewer, longer fixations than a page reader's. At 3-5 steps a quarter
|
|
245
|
+
// second apart the scan measured a shift a second, and over a webcam that
|
|
246
|
+
// is a jiggle, not reading.
|
|
247
|
+
read: { width: 0.20, steps: [2, 3], fix: [0.35, 0.7], lines: [2, 4], line: 0.05,
|
|
248
|
+
pause: [0.9, 1.8] },
|
|
249
|
+
// The brows draw down and together — AU4, the one facial action that
|
|
250
|
+
// reliably marks effortful concentration. Without it the face over a
|
|
251
|
+
// reading scan is blank, and blank over a moving eye reads as bored.
|
|
252
|
+
pose: { headPitch: 0.04, lidL: -0.08, lidR: -0.08,
|
|
253
|
+
shoulderL: 0.06, shoulderR: 0.06,
|
|
254
|
+
browRaiseL: -0.08, browRaiseR: -0.08, browInnerL: -0.10, browInnerR: -0.10 },
|
|
155
255
|
},
|
|
156
256
|
// The audio channel is broken and the agent is typing in the chat window to
|
|
157
257
|
// communicate — TYPING's mechanics turned *communicative*. The glance is
|
|
@@ -198,6 +298,13 @@ export const STATES = {
|
|
|
198
298
|
gaze: 'SCREEN_CENTER', emotion: 'neutral', engagement: false,
|
|
199
299
|
idle: { sway: 0.65, blinkGap: [5.0, 6.8], breathRate: 1.05,
|
|
200
300
|
flick: { amp: 0.30, every: [3.5, 7.0] } },
|
|
301
|
+
// The only screen state that never named its own scan, so it ran the
|
|
302
|
+
// default [0.5, 1.6] at full amplitude: a second full-size jump landing on
|
|
303
|
+
// top of a wander hop, which is most of what reads as snapping. A hunt's
|
|
304
|
+
// refixations are faster and much smaller than a reader's — quick little
|
|
305
|
+
// checks around the thing being looked at — so this is the other end of
|
|
306
|
+
// the dial from WORKING's [0.9, 2.0, 0.12] rather than a copy of it.
|
|
307
|
+
scan: [0.4, 1.1, 0.55],
|
|
201
308
|
wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_TOP', 'SCREEN_WORK',
|
|
202
309
|
'SCREEN_RIGHT', 'SCREEN_CENTER', 'SCREEN_LEFT'],
|
|
203
310
|
every: [0.8, 2.0] },
|
|
@@ -250,17 +357,58 @@ export const STATES = {
|
|
|
250
357
|
},
|
|
251
358
|
},
|
|
252
359
|
|
|
360
|
+
// --- connection ----------------------------------------------------------
|
|
361
|
+
// Neither of these may close the eyes. A lid at 0.3 reads sleepy and one at
|
|
362
|
+
// 0.95 reads asleep (research-perception.md §6), and on a photographic face
|
|
363
|
+
// a disconnected call rendered as someone falling asleep on camera. The SVG
|
|
364
|
+
// faces also desaturate (`filter`); a rig that does not is carried by pose
|
|
365
|
+
// and gaze alone, so those have to say it.
|
|
366
|
+
//
|
|
367
|
+
// Our side of the call is struggling: still with the user, a little worried
|
|
368
|
+
// about it — inner brows up, mouth pressed — and quieter than listening.
|
|
253
369
|
DEGRADED: { gaze: 'USER', emotion: 'neutral', engagement: false,
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
370
|
+
aversion: 'LISTEN',
|
|
371
|
+
idle: { sway: 0.4, blinkGap: [3.4, 5.0] },
|
|
372
|
+
pose: { browInnerL: 0.30, browInnerR: 0.26, browRaiseL: -0.04, browRaiseR: -0.04,
|
|
373
|
+
mouthPress: 0.20, mouthCornerL: -0.10, mouthCornerR: -0.10 },
|
|
374
|
+
filter: 'grayscale(.55) brightness(.82)' },
|
|
375
|
+
// The call is gone. What a person does when their call drops: eyes on their
|
|
376
|
+
// own screen, waiting, and now and then a look back up to see whether it
|
|
377
|
+
// has come back. Still, with long holds — not in conversation — but awake.
|
|
378
|
+
OFFLINE: { gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
|
|
379
|
+
scan: [1.2, 2.6],
|
|
380
|
+
idle: { sway: 0.3, blinkGap: [4.5, 7.0],
|
|
381
|
+
hold: { every: [4.0, 8.0], dur: [1.0, 2.0] } },
|
|
382
|
+
glance: { to: 'USER', every: [7, 12], hold: [0.8, 1.3] },
|
|
383
|
+
pose: { lidL: -0.08, lidR: -0.08, mouthPress: 0.25 },
|
|
259
384
|
filter: 'grayscale(1) brightness(.6)' },
|
|
260
385
|
};
|
|
261
386
|
|
|
262
387
|
export const STATE_NAMES = Object.keys(STATES);
|
|
263
388
|
|
|
389
|
+
/**
|
|
390
|
+
* The shoulder line's share of a held tilt, per unit of `headRoll`.
|
|
391
|
+
*
|
|
392
|
+
* Roll is the one head axis with nothing under it: a turn recruits the trunk
|
|
393
|
+
* and a nod bends the neck, but a tilt on a 2.5-D head is a rotation about a
|
|
394
|
+
* point near the chin and every other pixel stays exactly where it was. That is
|
|
395
|
+
* the read reported as "a hinge". Anatomically a tilt is lower-cervical
|
|
396
|
+
* (docs/research-head-rotation.md §2.1) — spread down the neck, ending at the
|
|
397
|
+
* girdle — so the shoulder line tips a little with it, and that tip is what says
|
|
398
|
+
* the neck bent rather than the head swinging off a pin.
|
|
399
|
+
*
|
|
400
|
+
* Sized to stay an accompaniment: at the roll clamp the line tips about a
|
|
401
|
+
* quarter of the trunk's own share of a turn, which lifts a shoulder ~0.11 of
|
|
402
|
+
* its shrug — inside the band a weight shift already occupies. A third, the
|
|
403
|
+
* figure Live2D gives its body angles, put 6 px of shoulder on an 8 degree tilt
|
|
404
|
+
* and read as a shrug arriving with the head.
|
|
405
|
+
*
|
|
406
|
+
* Exported for a page that drives a rig by hand and wants the body the shipping
|
|
407
|
+
* mixer would have put under the tilt; the rig instruments in the working tree
|
|
408
|
+
* are its only callers.
|
|
409
|
+
*/
|
|
410
|
+
export const SHOULDER_TILT = 0.08;
|
|
411
|
+
|
|
264
412
|
export function createAvatar(opts = {}) {
|
|
265
413
|
const mount = typeof opts.mount === 'string' ? document.querySelector(opts.mount) : opts.mount;
|
|
266
414
|
if (!mount) throw new Error('createAvatar: mount element required');
|
|
@@ -284,6 +432,7 @@ export function createAvatar(opts = {}) {
|
|
|
284
432
|
const gaze = new GazeLayer();
|
|
285
433
|
const idle = new IdleLayer();
|
|
286
434
|
const speech = new VisemeTrack();
|
|
435
|
+
const prosody = new SpeechProsody({ brows: opts.brows });
|
|
287
436
|
|
|
288
437
|
let gazeOverrideByClip = null;
|
|
289
438
|
const clip = new ClipPlayer({
|
|
@@ -301,14 +450,14 @@ export function createAvatar(opts = {}) {
|
|
|
301
450
|
// coordinates or the hand layer's private geometry.
|
|
302
451
|
const rig = opts.rig ? opts.rig(mount, opts.rigOptions) : createSvgRig(face, hand);
|
|
303
452
|
|
|
304
|
-
gaze.onLargeShift = () => idle.blink();
|
|
453
|
+
gaze.onLargeShift = (forced) => idle.blink(false, true, forced);
|
|
305
454
|
|
|
306
455
|
const listeners = {
|
|
307
456
|
state: [], speakEnd: [], clipEnd: [], performEnd: [], gestureEnd: [],
|
|
308
457
|
};
|
|
309
458
|
const emit = (ev, ...a) => listeners[ev] && listeners[ev].forEach((f) => f(...a));
|
|
310
459
|
clip.onEnd = (c) => { if (c) emit('clipEnd', c.id); };
|
|
311
|
-
speech.onEnd = () => { emit('speakEnd'); };
|
|
460
|
+
speech.onEnd = () => { prosody.closeTurn(); emit('speakEnd'); };
|
|
312
461
|
performTrack.onEnd = () => { emit('performEnd'); };
|
|
313
462
|
|
|
314
463
|
// --- live state -----------------------------------------------------------
|
|
@@ -328,6 +477,56 @@ export function createAvatar(opts = {}) {
|
|
|
328
477
|
// channel clamp, which is the intended ceiling rather than a bug.
|
|
329
478
|
let mouthGain = opts.mouthGain ?? 1;
|
|
330
479
|
let handSide = opts.handSide === -1 ? 'left' : 'right';
|
|
480
|
+
/**
|
|
481
|
+
* This avatar's own addressable motions, on top of the two core intents.
|
|
482
|
+
*
|
|
483
|
+
* An avatar is a drawing with a body, and some bodies can do things the wire
|
|
484
|
+
* has no portable word for. The three nod types the listening research
|
|
485
|
+
* separates — a continuer, an assessment, a realisation — are one
|
|
486
|
+
* `ACKNOWLEDGE` to a server, because that is all a server can ask of every
|
|
487
|
+
* face; the *shapes* are sized in a rig's own units and belong to the rig
|
|
488
|
+
* (`packages/avatar/client/three/sequences.ts` is the first table of them).
|
|
489
|
+
*
|
|
490
|
+
* Deliberately not a registry and not a loader: it is a plain object the
|
|
491
|
+
* avatar module passes in, and it can only add. `ACTIONS` wins on a name
|
|
492
|
+
* collision, so no asset can quietly change what a core action means.
|
|
493
|
+
*/
|
|
494
|
+
const sequences = opts.sequences || {};
|
|
495
|
+
/**
|
|
496
|
+
* This avatar's own *rendering* of an id this renderer already has — same id,
|
|
497
|
+
* same intent, a shape sized for its body.
|
|
498
|
+
*
|
|
499
|
+
* These are an intent vocabulary, and `ACK_NOD` is one implementation of
|
|
500
|
+
* "the avatar nodded", not a promise about anatomy (interjections.js). The
|
|
501
|
+
* shared clips are authored in pose units that mean pixels on a line face;
|
|
502
|
+
* on a rig whose unit is a degree the same keys can land outside what the
|
|
503
|
+
* gesture *means* — the shared nod renders 15° on tara, where a continuer is
|
|
504
|
+
* 3-5° — and scaling it would not fix it, because a continuer is one stroke
|
|
505
|
+
* where an agreement is two. So a renderer may re-author the shape.
|
|
506
|
+
*
|
|
507
|
+
* What it may not do is change the vocabulary: every id here must already be
|
|
508
|
+
* one this renderer publishes, so no avatar can add a word through this door
|
|
509
|
+
* or make one mean something else. `sequences` is the door for adding.
|
|
510
|
+
*/
|
|
511
|
+
const actionShapes = opts.actions || {};
|
|
512
|
+
for (const id of Object.keys(actionShapes)) {
|
|
513
|
+
if (!ACTIONS[id]) throw new Error(`actions: ${id} is not one of this renderer's own`);
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* This avatar's own rendering of a state — the same door as `actions`, for
|
|
517
|
+
* the held face instead of the gesture. A state's pose is authored where a
|
|
518
|
+
* line face reads, and the cue that carries it there can say something else
|
|
519
|
+
* on a photograph: WORKING's knit brows are what keep peep's reading face
|
|
520
|
+
* from going blank, and on a real eye — already hooded by a lid following it
|
|
521
|
+
* down to the screen — they finish a squint that reads as strain. A rig
|
|
522
|
+
* replaces a state's fields whole (`pose`, say), and only for a state the
|
|
523
|
+
* table already has: it can re-render the vocabulary, never extend it.
|
|
524
|
+
*/
|
|
525
|
+
const states = { ...STATES };
|
|
526
|
+
for (const [id, own] of Object.entries(opts.states || {})) {
|
|
527
|
+
if (!STATES[id]) throw new Error(`states: ${id} is not a state`);
|
|
528
|
+
states[id] = { ...STATES[id], ...own };
|
|
529
|
+
}
|
|
331
530
|
let handAction = null;
|
|
332
531
|
const handQueue = [];
|
|
333
532
|
// Gesture gain, same idea for the clip layer. A nod is ballistic — NOD_SMALL
|
|
@@ -343,20 +542,114 @@ export function createAvatar(opts = {}) {
|
|
|
343
542
|
// than rendering it locally as SVG, where the motion costs nothing — turns
|
|
344
543
|
// this down instead of the default being a body that does not move.
|
|
345
544
|
idle.gain = opts.motionGain ?? 1;
|
|
545
|
+
// Speech-rhythm gains. A pose unit is a different angle on every rig — peep's
|
|
546
|
+
// head travels 17 px per unit of pitch, a mesh head turns a few degrees — so
|
|
547
|
+
// the rig's own module sets these once rather than the library guessing.
|
|
548
|
+
// Defaults to the uncalibrated scale; `prosody.js` has what that stands for.
|
|
549
|
+
const prosodyHeadGain = opts.prosodyHeadGain ?? UNCALIBRATED_HEAD_GAIN;
|
|
550
|
+
const prosodyFaceGain = opts.prosodyFaceGain ?? 1;
|
|
551
|
+
// Fixational-saccade gain, the same idea for the eyes: how far a scan
|
|
552
|
+
// step, a drift or a line of reading travels. Sized for a line face's
|
|
553
|
+
// pupils at 1; a photographic eye needs several times that to move at all.
|
|
554
|
+
// Aversions get their own gain: a look-away must read as one from across
|
|
555
|
+
// the call, where a fixation step must not, so one number cannot size both.
|
|
556
|
+
const saccadeGain = opts.saccadeGain ?? 1;
|
|
557
|
+
// How far this face may hold its head off centre, per axis, in pose units.
|
|
558
|
+
// A pose unit is an angle on a mesh head and a pixel count on a drawing, and
|
|
559
|
+
// what a 2.5-D face can hold before the photograph gives it away is a
|
|
560
|
+
// measurement of that face and of nothing else — so the number is the rig's
|
|
561
|
+
// to supply and is nowhere in this library. An axis left out is unbudgeted,
|
|
562
|
+
// which is every SVG and Canvas face and is the behaviour they have always
|
|
563
|
+
// had.
|
|
564
|
+
const headHold = opts.headHold || {};
|
|
565
|
+
// Per-axis gain on the head's *continuous* drive, in front of the budget.
|
|
566
|
+
//
|
|
567
|
+
// The two layers that hold a head somewhere — speech phrasing and idle —
|
|
568
|
+
// are sized in pose units for a line drawing, and a 2.5-D face measured its
|
|
569
|
+
// own limits afterwards. Measuring found the drive spending well under them:
|
|
570
|
+
// a third of the pitch a speaking human uses and nearly twice the yaw, on a
|
|
571
|
+
// face whose pitch budget has room for all of it. That is one number per
|
|
572
|
+
// axis, not a rewrite of either layer, and it belongs here rather than in
|
|
573
|
+
// the rig because it scales a *drive* — a rig scale would multiply the nods
|
|
574
|
+
// and beats too, and those are authored at amplitudes that already read.
|
|
575
|
+
//
|
|
576
|
+
// Before 6b deliberately: the budget is what stops a scaled-up drive from
|
|
577
|
+
// leaving the envelope, so a gain that skipped it would be measuring the
|
|
578
|
+
// face's failure rather than the layer's range. A number, or one per axis.
|
|
579
|
+
const headGain = typeof opts.headGain === 'number'
|
|
580
|
+
? { headYaw: opts.headGain, headPitch: opts.headGain, headRoll: opts.headGain }
|
|
581
|
+
: { headYaw: 1, headPitch: 1, headRoll: 1, ...(opts.headGain || {}) };
|
|
582
|
+
gaze.scanGain = saccadeGain;
|
|
583
|
+
gaze.avertGain = opts.aversionGain ?? 1;
|
|
584
|
+
// A rig that says what its pose units are in degrees gets the eye-head
|
|
585
|
+
// system sized for it (gaze.js): its own look targets, how an aversion
|
|
586
|
+
// splits between eyes and head, lids that follow the eye both ways, and the
|
|
587
|
+
// reflex in step 8b. A face that does not say keeps the line-face behaviour
|
|
588
|
+
// exactly.
|
|
589
|
+
const ocu = opts.oculomotor || {};
|
|
590
|
+
if (ocu.targets) gaze.targets = { ...GAZE_TARGETS, ...ocu.targets };
|
|
591
|
+
if (ocu.avert) gaze.avertSplit = ocu.avert;
|
|
592
|
+
if (ocu.angles) gaze.angles = ocu.angles;
|
|
593
|
+
if (ocu.lidFollow) gaze.lidFollow = ocu.lidFollow;
|
|
594
|
+
if (ocu.head) { gaze.headAccel = ocu.head.accel; gaze.headSpeed = ocu.head.speed; }
|
|
595
|
+
// Pupil units of counter-rotation per head unit: the reflex's gain times
|
|
596
|
+
// the ratio of what one unit of each is in degrees. The gain may differ by
|
|
597
|
+
// axis ({x, y}): a face whose pitch reads weaker than its yaw wants less
|
|
598
|
+
// of the eyes' answer to it.
|
|
599
|
+
const vorGain = typeof ocu.vor === 'object' ? ocu.vor : { x: ocu.vor, y: ocu.vor };
|
|
600
|
+
const vor = ocu.vor && ocu.angles
|
|
601
|
+
? { x: vorGain.x * ocu.angles.head.x / ocu.angles.eye.x,
|
|
602
|
+
y: vorGain.y * ocu.angles.head.y / ocu.angles.eye.y }
|
|
603
|
+
: null;
|
|
604
|
+
// How far the reflex may carry the eye in its socket, in pupil units. A
|
|
605
|
+
// person looking 8° up does not roll the eyes to the lid and wait for the
|
|
606
|
+
// head: the eye stops well short of its mechanical limit and the head makes
|
|
607
|
+
// up the rest (Guitton & Volle's effective oculomotor range). On a
|
|
608
|
+
// photographic eye an iris pinned under the upper lid with white showing
|
|
609
|
+
// beneath it reads as an eye-roll, not a thought. Down is looser — the lid
|
|
610
|
+
// follows the eye down and hides the sclera there.
|
|
611
|
+
const reach = ocu.range || { x: 1, up: 1, down: 1 };
|
|
612
|
+
const reflexX = (px) => clamp(px + vor.x * (aim.x - cur.headYaw), -reach.x, reach.x);
|
|
613
|
+
const reflexY = (py) => clamp(py + vor.y * (aim.y - cur.headPitch), -reach.up, reach.down);
|
|
614
|
+
// The head the eyes' target was authored against, smoothed at the eye's
|
|
615
|
+
// own tau so a shift's compensation moves with the saccade, not ahead of it.
|
|
616
|
+
const aim = { x: 0, y: 0 };
|
|
346
617
|
let wanderAt = 0;
|
|
347
|
-
let
|
|
618
|
+
let driftAt = 0;
|
|
619
|
+
let trunkYaw = 0;
|
|
620
|
+
let turnStartPending = false;
|
|
348
621
|
let glanceAt = 0;
|
|
349
622
|
let glanceUntil = 0;
|
|
623
|
+
let lastBack = null;
|
|
624
|
+
// THINKING's second look inside a look away: when it moves, where to, and
|
|
625
|
+
// how long the brows stay up for it.
|
|
626
|
+
let dartAt = 0, dartBrowUntil = 0;
|
|
627
|
+
const dart = { x: 0, y: 0 };
|
|
628
|
+
// THINKING -> SPEAKING mid-look: when the eyes go back to the user, and
|
|
629
|
+
// when that handoff happened (it stands in for the turn-start look).
|
|
630
|
+
let returnAt = 0;
|
|
631
|
+
let carriedAt = -Infinity;
|
|
632
|
+
// The state whose gaze is showing, and when the current state takes it
|
|
633
|
+
// over if that is still pending (GAP_SETTLE). Usually the same state.
|
|
634
|
+
let gazeState = 'IDLE';
|
|
635
|
+
let settleAt = 0;
|
|
636
|
+
// Reading scan: position in the line and the block, see readStep.
|
|
637
|
+
let readCol = 0, readCols = 0, readRow = 0, readRows = 0;
|
|
350
638
|
let attendUntil = 0;
|
|
351
639
|
let speakClock = null;
|
|
352
640
|
let speakStart = 0;
|
|
353
641
|
|
|
354
642
|
const cur = Object.assign({}, REST);
|
|
355
643
|
const target = Object.assign({}, REST);
|
|
644
|
+
// The three head axes again, carrying only what is held (step 6b).
|
|
645
|
+
const hold = { headYaw: 0, headPitch: 0, headRoll: 0 };
|
|
646
|
+
// What the rig is handed: `cur` with the reflex applied to the eyes. The
|
|
647
|
+
// same object when there is no reflex.
|
|
648
|
+
const shown = vor ? Object.assign({}, REST) : cur;
|
|
356
649
|
|
|
357
|
-
function applyGaze() {
|
|
650
|
+
function applyGaze(blink) {
|
|
358
651
|
const g = gazeOverrideByClip || gazeName;
|
|
359
|
-
gaze.set(g, gazeOverrideByClip ? null : gazeCustom);
|
|
652
|
+
gaze.set(g, gazeOverrideByClip ? null : gazeCustom, blink);
|
|
360
653
|
}
|
|
361
654
|
|
|
362
655
|
// --- the frame ------------------------------------------------------------
|
|
@@ -366,7 +659,8 @@ export function createAvatar(opts = {}) {
|
|
|
366
659
|
// `manual` withholds the rAF loop so a tool can drive frames itself. The
|
|
367
660
|
// baseline pages could already step a ClipPlayer by hand, but nothing could
|
|
368
661
|
// step the *mixer* — which is where idle, gaze and engagement actually
|
|
369
|
-
// compose — so motion had no reproducible render.
|
|
662
|
+
// compose — so motion had no reproducible render. The headless motion map in
|
|
663
|
+
// the working tree is what steps it.
|
|
370
664
|
const manual = !!opts.manual;
|
|
371
665
|
|
|
372
666
|
function frame(now) {
|
|
@@ -384,23 +678,42 @@ export function createAvatar(opts = {}) {
|
|
|
384
678
|
// firing this frame shapes this frame.
|
|
385
679
|
performTrack.update();
|
|
386
680
|
|
|
387
|
-
|
|
681
|
+
if (settleAt && elapsed >= settleAt) { settleAt = 0; enterGaze(stateName, {}, true); }
|
|
682
|
+
const st = states[stateName] || states.IDLE;
|
|
683
|
+
// What the eyes are doing belongs to the state that has the gaze.
|
|
684
|
+
const gst = states[gazeState] || st;
|
|
388
685
|
|
|
389
686
|
// 1. base pose: rest + emotion + state-specific overlay
|
|
390
687
|
for (const c of CHANNELS) target[c] = REST[c];
|
|
391
688
|
const ep = emotionPose(emotion, emotionAmt);
|
|
392
689
|
for (const k in ep) target[k] = REST[k] + ep[k];
|
|
393
690
|
if (st.pose) for (const k in st.pose) target[k] = (target[k] || 0) + st.pose[k];
|
|
691
|
+
// The head the state and emotion hold is an attitude, and the eyes are
|
|
692
|
+
// authored inside it: the reflex (8b) keeps them on their target against
|
|
693
|
+
// everything that moves the head except this.
|
|
694
|
+
const poseYaw = target.headYaw, posePitch = target.headPitch;
|
|
695
|
+
// What the head *holds*, accumulated alongside the pose as each layer that
|
|
696
|
+
// holds one adds to it: the attitude above, the gaze, a phrase's pose and
|
|
697
|
+
// the idle posture. Strokes, beats and clip deltas stay out — see 6b.
|
|
698
|
+
for (const c of HEAD_AXES) hold[c] = target[c];
|
|
394
699
|
|
|
395
700
|
// 2. gaze (absolute: pupils + partial head follow, plus the lid that rides
|
|
396
701
|
// with the eye — looking down without it bares sclera and reads as alarm)
|
|
397
702
|
const g = gaze.update(elapsed, dt);
|
|
398
703
|
for (const k in g) {
|
|
399
|
-
if (k === '
|
|
704
|
+
if (k === 'trunkYaw' || k === 'aimYaw' || k === 'aimPitch') continue;
|
|
400
705
|
target[k] = (k.startsWith('head') ? target[k] : 0) + g[k];
|
|
401
706
|
}
|
|
402
|
-
|
|
403
|
-
|
|
707
|
+
for (const c of HEAD_AXES) hold[c] += g[c];
|
|
708
|
+
// The lid follows the eye as drawn, reflex and all — last frame's reading
|
|
709
|
+
// of it, since the head it corrects for is not smoothed yet.
|
|
710
|
+
const lid = gaze.lidBias(vor ? reflexY(g.pupilY) : g.pupilY);
|
|
711
|
+
target.lidL += lid;
|
|
712
|
+
target.lidR += lid;
|
|
713
|
+
if (vor) {
|
|
714
|
+
aim.x = approach(aim.x, poseYaw + g.aimYaw, TAU.pupilX, dt);
|
|
715
|
+
aim.y = approach(aim.y, posePitch + g.aimPitch, TAU.pupilY, dt);
|
|
716
|
+
}
|
|
404
717
|
|
|
405
718
|
// 2b. the trunk follows the head. Sampled HERE, after gaze and before the
|
|
406
719
|
// clip layer, on purpose: a sustained turn toward the screen recruits
|
|
@@ -408,40 +721,111 @@ export function createAvatar(opts = {}) {
|
|
|
408
721
|
// with every gesture reads as a mannequin on a turntable. The lag is
|
|
409
722
|
// not authored anywhere; torsoTurn simply chases the same target at
|
|
410
723
|
// nearly 3x the head's time constant (TAU in params.js), so the trunk
|
|
411
|
-
// leaves late and settles late for free.
|
|
412
|
-
|
|
724
|
+
// leaves late and settles late for free. It follows where the head is
|
|
725
|
+
// going and not the looks riding on it, and it holds through a glance:
|
|
726
|
+
// checking on the user is a look, not a turn toward them.
|
|
727
|
+
if (!glanceUntil) trunkYaw = target.headYaw - g.headYaw + g.trunkYaw;
|
|
728
|
+
target.torsoTurn += trunkYaw * TRUNK_FOLLOW;
|
|
413
729
|
|
|
414
730
|
// 3. state-driven autonomous behaviour
|
|
415
|
-
if (
|
|
416
|
-
const w =
|
|
417
|
-
|
|
418
|
-
|
|
731
|
+
if (gst.wander && elapsed > wanderAt) {
|
|
732
|
+
const w = gst.wander;
|
|
733
|
+
// Dwell first, and sometimes for two periods. A dwell of exactly one
|
|
734
|
+
// period every time is the metronome the eye reads as mechanical. The
|
|
735
|
+
// old code broke that up by accident: a draw landing on the target
|
|
736
|
+
// already held left the eyes still for another period. The freeze was a
|
|
737
|
+
// bug — the schedule counted a hop that moved nothing — but the dwell
|
|
738
|
+
// variety it produced was not, so it is kept deliberately, at the rate
|
|
739
|
+
// it used to happen: 1/n, the odds a uniform draw repeats the target.
|
|
740
|
+
const linger = Math.random() < 1 / w.targets.length ? 2 : 1;
|
|
741
|
+
wanderAt = elapsed + linger * (w.every[0] + Math.random() * (w.every[1] - w.every[0]));
|
|
742
|
+
// Then look somewhere the eyes are not. A wander set names a target more
|
|
743
|
+
// than once deliberately — SEARCHING_SCREEN's repeats are its revisits,
|
|
744
|
+
// the hunt coming back to the middle — but a revisit is arriving
|
|
745
|
+
// somewhere again, not never having left it. Same bounded redraw as
|
|
746
|
+
// pickBack, so a one-target set still terminates.
|
|
747
|
+
let g = pick(w.targets);
|
|
748
|
+
for (let i = 0; i < 4 && g === gazeName; i++) g = pick(w.targets);
|
|
749
|
+
setGaze(g);
|
|
419
750
|
}
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
751
|
+
// Periodic glance (WORKING's look-up-at-you beat, THINKING's check-in
|
|
752
|
+
// between two looks away). The return leg goes to the state's own gaze, or
|
|
753
|
+
// to a fresh pick from `back` — a thinker does not look away at the same
|
|
754
|
+
// spot every time. Either leg's gaze-evoked blink is the shift's odds
|
|
755
|
+
// unless the state says (`blinkTo`, `blinkBack`), because which leg blinks
|
|
756
|
+
// is part of what the glance means.
|
|
757
|
+
if (returnAt && elapsed > returnAt) {
|
|
758
|
+
returnAt = 0;
|
|
759
|
+
setGaze(gst.gaze);
|
|
423
760
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
// both legs for free, which is exactly the blink a real glance carries.
|
|
427
|
-
if (st.glance) {
|
|
428
|
-
const gl = st.glance;
|
|
761
|
+
if (gst.glance && !returnAt) {
|
|
762
|
+
const gl = gst.glance;
|
|
429
763
|
if (glanceUntil && elapsed > glanceUntil) {
|
|
430
764
|
glanceUntil = 0;
|
|
431
|
-
|
|
432
|
-
|
|
765
|
+
const away = rand(gl.every);
|
|
766
|
+
glanceAt = elapsed + away;
|
|
767
|
+
setGaze(gl.back ? pickBack(gl.back, gl.stick) : gst.gaze, null, gl.blinkBack);
|
|
768
|
+
// At most one second look per look away, somewhere in its middle: a
|
|
769
|
+
// move at the start is the same look landing, and one at the end runs
|
|
770
|
+
// into the glance back.
|
|
771
|
+
dartAt = gl.dart && Math.random() < gl.dart.p ? elapsed + away * (0.35 + Math.random() * 0.3) : 0;
|
|
433
772
|
} else if (!glanceUntil && elapsed > glanceAt) {
|
|
434
|
-
glanceUntil = elapsed +
|
|
435
|
-
setGaze(gl.to);
|
|
773
|
+
glanceUntil = elapsed + rand(gl.hold);
|
|
774
|
+
setGaze(gl.to, null, gl.blinkTo);
|
|
775
|
+
dartAt = 0;
|
|
776
|
+
dart.x = dart.y = 0;
|
|
777
|
+
}
|
|
778
|
+
if (dartAt && elapsed > dartAt) {
|
|
779
|
+
// On from where the eyes are, not back toward the user or across to
|
|
780
|
+
// the other side: the thought moves, it does not turn round.
|
|
781
|
+
dartAt = 0;
|
|
782
|
+
const t = gaze.target;
|
|
783
|
+
const a = Math.atan2(t.py, t.px) + (Math.random() < 0.5 ? -1 : 1) * DART_TURN;
|
|
784
|
+
const m = gl.dart.mag * saccadeGain;
|
|
785
|
+
dart.x = Math.cos(a) * m;
|
|
786
|
+
dart.y = Math.sin(a) * m;
|
|
787
|
+
dartBrowUntil = elapsed + DART_BROW;
|
|
436
788
|
}
|
|
789
|
+
const brow = (glanceUntil ? gl.brow || 0 : 0)
|
|
790
|
+
+ (gl.dart && elapsed < dartBrowUntil ? gl.dart.brow : 0);
|
|
791
|
+
target.browRaiseL += brow;
|
|
792
|
+
target.browRaiseR += brow;
|
|
793
|
+
}
|
|
794
|
+
gaze.scanEvery = gst.scan || null;
|
|
795
|
+
// A state that reads scans a line; one with a dart holds its second look.
|
|
796
|
+
// Neither during a glance: a check-in on the user is steady or it is not
|
|
797
|
+
// one.
|
|
798
|
+
if (gst.read && !glanceUntil) {
|
|
799
|
+
if (elapsed > driftAt) readStep(gst.read);
|
|
800
|
+
} else {
|
|
801
|
+
gaze.drift.x = glanceUntil ? 0 : dart.x;
|
|
802
|
+
gaze.drift.y = glanceUntil ? 0 : dart.y;
|
|
437
803
|
}
|
|
438
804
|
// Aversion is a property of the state, but it is held off around a turn
|
|
439
805
|
// boundary: the floor is handed over under mutual gaze, and an avatar that
|
|
440
806
|
// looks away exactly as the user finishes has declined it. `attend` is the
|
|
441
807
|
// mixer's one-frame veto — anything that means "the user is checking
|
|
442
808
|
// whether I am with them" sets it (see api.attend).
|
|
443
|
-
gaze.setAversion(
|
|
444
|
-
gaze
|
|
809
|
+
gaze.setAversion(gst.aversion ? AVERSION[gst.aversion] : null);
|
|
810
|
+
// A speaking turn ends under mutual gaze: no look in the last 2.4 s of the
|
|
811
|
+
// cue track as far as it has arrived, nor once it has run out (§4.2). The
|
|
812
|
+
// mid-turn looks wait for a silent cue, which is a phrase boundary. Both
|
|
813
|
+
// read last frame's sample, a frame late and harmlessly so.
|
|
814
|
+
let floorReturn = false;
|
|
815
|
+
gaze.gate = true;
|
|
816
|
+
if (gst.aversion === 'SPEAK') {
|
|
817
|
+
const cues = speech.cues;
|
|
818
|
+
floorReturn = !speech.playing || !cues.length
|
|
819
|
+
|| cues[cues.length - 1].t - speech.now < FLOOR_RETURN_MS;
|
|
820
|
+
const cue = cues[speech.index];
|
|
821
|
+
gaze.gate = !!cue && cue.v === SILENT;
|
|
822
|
+
}
|
|
823
|
+
gaze.hold = attendUntil > elapsed || clip.playing || floorReturn;
|
|
824
|
+
if (turnStartPending) {
|
|
825
|
+
turnStartPending = false;
|
|
826
|
+
// A reply that began mid-think has had its planning look already.
|
|
827
|
+
if (elapsed - carriedAt > 1.5) gaze.startTurn();
|
|
828
|
+
}
|
|
445
829
|
|
|
446
830
|
engagement.enabled = !!st.engagement && !clip.playing;
|
|
447
831
|
engagement.update(dt);
|
|
@@ -450,7 +834,19 @@ export function createAvatar(opts = {}) {
|
|
|
450
834
|
// research (docs/research-biomechanics.md §6.3) puts sustained attentive
|
|
451
835
|
// lean at +0.15–0.25; engage glides, and torsoLean's 0.24s tau smooths
|
|
452
836
|
// the state gate, so the lean arrives and leaves like weight shifting.
|
|
453
|
-
if (st.engagement)
|
|
837
|
+
if (st.engagement) {
|
|
838
|
+
target.torsoLean += 0.16 * engagement.engage;
|
|
839
|
+
// The shoulders come with it. A lean is a whole upper body arriving, and
|
|
840
|
+
// in a head-and-shoulders crop the shoulder line is the part of it that
|
|
841
|
+
// is actually on screen — §6.1 calls a shoulder rise the most legible
|
|
842
|
+
// thing this framing can draw. Without this the listening shoulders were
|
|
843
|
+
// the posture shift's ±0.07 and nothing else, so the one channel that
|
|
844
|
+
// could show attention sat still through every turn the user took.
|
|
845
|
+
// It rides `engage`, so it is contingent on the user's voice rather than
|
|
846
|
+
// on the state: quick in, slow out, and gone a few seconds into silence.
|
|
847
|
+
target.shoulderL += ENGAGE_SHOULDER * engagement.engage;
|
|
848
|
+
target.shoulderR += ENGAGE_SHOULDER * engagement.engage;
|
|
849
|
+
}
|
|
454
850
|
// Straining leans harder while there is actually a faint voice to strain
|
|
455
851
|
// after. engage already tracks "the user is (barely) talking", so this
|
|
456
852
|
// costs nothing; with no user signal the static pose carries the state.
|
|
@@ -489,6 +885,57 @@ export function createAvatar(opts = {}) {
|
|
|
489
885
|
}
|
|
490
886
|
}
|
|
491
887
|
|
|
888
|
+
// 4b. speech prosody: pause blinks, inbreaths, a head held per phrase and
|
|
889
|
+
// moved between them, beats and turn-edge warmth, read off the cue
|
|
890
|
+
// track (prosody.js has the research, head.js the hold-and-move). Off while a clip is gesturing, since the clip is
|
|
891
|
+
// already the head's and the brows' story. The warmth is withheld
|
|
892
|
+
// under an emotion whose corners are down: a smile starting a turn of
|
|
893
|
+
// concern is the wrong face, whatever the rhythm says.
|
|
894
|
+
const pro = prosody.update(speech, mouthOwner === 'speech' && !clip.playing, dt);
|
|
895
|
+
// The warmth rides over a clip, because an acknowledgement's smile is
|
|
896
|
+
// *with* its nod: behind the clip gate it arrived as the nod finished, a
|
|
897
|
+
// smile at nothing. It is withheld under a clip whose own corners go down
|
|
898
|
+
// (a shake, a sorry) for the same reason as under a concerned emotion.
|
|
899
|
+
const clipFrowns = !!clipOut.delta
|
|
900
|
+
&& Math.min(clipOut.delta.mouthCornerL || 0, clipOut.delta.mouthCornerR || 0) < 0;
|
|
901
|
+
const warm = clipFrowns || ((EMOTIONS[emotion] || EMOTIONS.neutral).mouthCornerL || 0) < 0
|
|
902
|
+
? 0 : prosodyFaceGain;
|
|
903
|
+
target.mouthCornerL += pro.mouthCornerL * warm;
|
|
904
|
+
target.mouthCornerR += pro.mouthCornerR * warm;
|
|
905
|
+
target.squintL += pro.squintL * warm;
|
|
906
|
+
target.squintR += pro.squintR * warm;
|
|
907
|
+
if (!clip.playing) {
|
|
908
|
+
target.headPitch += pro.headPitch * prosodyHeadGain * headGain.headPitch;
|
|
909
|
+
target.headYaw += pro.headYaw * prosodyHeadGain * headGain.headYaw;
|
|
910
|
+
target.headRoll += pro.headRoll * prosodyHeadGain * headGain.headRoll;
|
|
911
|
+
for (const c of HEAD_AXES) hold[c] += pro.hold[c] * prosodyHeadGain * headGain[c];
|
|
912
|
+
// The trunk follows a speech pose the way it follows a gaze turn (2b),
|
|
913
|
+
// and for a second reason on a 2.5-D head: a turn the shoulders take
|
|
914
|
+
// part of is a turn the neck does not have to stretch for, and the
|
|
915
|
+
// stretch is the one thing a video reviewer saw on every pose change.
|
|
916
|
+
// Scaled with the yaw it follows: a trunk that kept its old share of a
|
|
917
|
+
// wider turn would be the neck stretching again, which is the defect
|
|
918
|
+
// this line exists to fix.
|
|
919
|
+
target.torsoTurn += pro.trunkYaw * prosodyHeadGain * headGain.headYaw * TRUNK_FOLLOW;
|
|
920
|
+
target.browRaiseL += pro.browRaiseL * prosodyFaceGain;
|
|
921
|
+
target.browRaiseR += pro.browRaiseR * prosodyFaceGain;
|
|
922
|
+
target.browInnerL += pro.browInnerL * prosodyFaceGain;
|
|
923
|
+
target.browInnerR += pro.browInnerR * prosodyFaceGain;
|
|
924
|
+
target.browAngleL += pro.browAngleL * prosodyFaceGain;
|
|
925
|
+
target.browAngleR += pro.browAngleR * prosodyFaceGain;
|
|
926
|
+
target.lidL += pro.lidL * prosodyFaceGain;
|
|
927
|
+
target.lidR += pro.lidR * prosodyFaceGain;
|
|
928
|
+
target.breath += pro.breath * idle.gain;
|
|
929
|
+
// The trunk's share of speech rhythm. Scaled by `idle.gain` and not by
|
|
930
|
+
// `prosodyHeadGain`, because this is body liveness and that is the knob a
|
|
931
|
+
// host turns down when it is re-encoding the avatar into an outgoing
|
|
932
|
+
// stream — the same factor `idle.js`'s shoulders and lean already take.
|
|
933
|
+
target.shoulderL += pro.shoulderL * idle.gain;
|
|
934
|
+
target.shoulderR += pro.shoulderR * idle.gain;
|
|
935
|
+
target.torsoLean += pro.torsoLean * idle.gain;
|
|
936
|
+
}
|
|
937
|
+
if (pro.blink) idle.phraseBlink();
|
|
938
|
+
|
|
492
939
|
// 5. gesture deltas (additive, so a nod survives whatever else is happening)
|
|
493
940
|
if (clipOut.delta) {
|
|
494
941
|
for (const k in clipOut.delta) {
|
|
@@ -505,7 +952,40 @@ export function createAvatar(opts = {}) {
|
|
|
505
952
|
idle.talk = approach(idle.talk, mouthOwner ? 1 : 0, 0.25, dt);
|
|
506
953
|
idle.setProfile(st.idle);
|
|
507
954
|
const il = idle.update(dt);
|
|
508
|
-
for (const k in il.add) target[k] = (target[k] || 0) + il.add[k];
|
|
955
|
+
for (const k in il.add) target[k] = (target[k] || 0) + il.add[k] * (headGain[k] ?? 1);
|
|
956
|
+
for (const c of HEAD_AXES) hold[c] += (il.add[c] || 0) * headGain[c];
|
|
957
|
+
|
|
958
|
+
// 6b. the held-head budget. Every layer above holds its own small pose and
|
|
959
|
+
// they are independent, so now and then they all point the same way and
|
|
960
|
+
// the head arrives somewhere no one layer asked for and the face cannot
|
|
961
|
+
// go. Only the excess over `soften` comes off, and only off the hold:
|
|
962
|
+
// a nod, a beat and a clip keep every degree they were authored with,
|
|
963
|
+
// which is why this is subtracted here rather than applied to the pose.
|
|
964
|
+
for (const c of HEAD_AXES) {
|
|
965
|
+
if (headHold[c] === undefined) continue;
|
|
966
|
+
target[c] -= hold[c] - soften(hold[c], headHold[c]);
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
// 6c. the body answers a held tilt (SHOULDER_TILT). The trunk takes the
|
|
970
|
+
// same share of it that it takes of a turn in 2b — Live2D gives its
|
|
971
|
+
// body the same fraction of AngleZ as of AngleX (research-head-rotation
|
|
972
|
+
// .md §3) — and the shoulder line tips with the head. Both channels are
|
|
973
|
+
// slower than the head (TAU: 0.44 and 0.19 against 0.16), so the body
|
|
974
|
+
// leaves late and settles late, and that follow-through is most of what
|
|
975
|
+
// separates a neck bending from a hinge.
|
|
976
|
+
// Only the *held* roll, and after the budget: a stroke or a clip's roll
|
|
977
|
+
// is a gesture riding on the pose, and a body that answers those is 2b's
|
|
978
|
+
// mannequin on a turntable. The idle layer's own posture is already
|
|
979
|
+
// coupled the other way round, from the weight shift to the head that
|
|
980
|
+
// counter-tips on it (idle.js `nextPosture`); this is that arrangement
|
|
981
|
+
// read from the head's end, and where both are in play they agree in
|
|
982
|
+
// sign — weight onto a side, head over that side, that shoulder up.
|
|
983
|
+
const heldRoll = headHold.headRoll === undefined
|
|
984
|
+
? hold.headRoll
|
|
985
|
+
: soften(hold.headRoll, headHold.headRoll);
|
|
986
|
+
target.torsoTurn += heldRoll * TRUNK_FOLLOW;
|
|
987
|
+
target.shoulderR += heldRoll * SHOULDER_TILT;
|
|
988
|
+
target.shoulderL -= heldRoll * SHOULDER_TILT;
|
|
509
989
|
|
|
510
990
|
// 7. clamp, then blink wins outright over whatever the lids were doing
|
|
511
991
|
for (const c of CHANNELS) {
|
|
@@ -522,11 +1002,32 @@ export function createAvatar(opts = {}) {
|
|
|
522
1002
|
// 8. smooth toward the target — this is where co-articulation happens
|
|
523
1003
|
for (const c of CHANNELS) cur[c] = approach(cur[c], target[c], TAU[c], dt);
|
|
524
1004
|
|
|
1005
|
+
// 8b. The vestibulo-ocular reflex. Eyes held still in a moving head look
|
|
1006
|
+
// wherever the head points, so without this every nod, speech pose and
|
|
1007
|
+
// sway was also a small look somewhere else — and on a face whose head
|
|
1008
|
+
// turns further than its eyes, a look at a different spot each time.
|
|
1009
|
+
// Real eyes counter-rotate against the head within ~10 ms and stay on
|
|
1010
|
+
// what they look at. Here and not in the gaze layer because only here
|
|
1011
|
+
// is the head that is actually drawn known: prosody, clips and idle all
|
|
1012
|
+
// land after gaze. It also gives a large shift its real shape for free:
|
|
1013
|
+
// the eyes jump to the target, past where they will sit, and roll back
|
|
1014
|
+
// in the head as it arrives under them. A pupil a tuning UI overrides is
|
|
1015
|
+
// left where it was put.
|
|
1016
|
+
if (vor) {
|
|
1017
|
+
Object.assign(shown, cur);
|
|
1018
|
+
if (!overrides || overrides.pupilX === undefined) {
|
|
1019
|
+
shown.pupilX = clamp(reflexX(cur.pupilX), RANGE.pupilX[0], RANGE.pupilX[1]);
|
|
1020
|
+
}
|
|
1021
|
+
if (!overrides || overrides.pupilY === undefined) {
|
|
1022
|
+
shown.pupilY = clamp(reflexY(cur.pupilY), RANGE.pupilY[0], RANGE.pupilY[1]);
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
|
|
525
1026
|
// 9. First-class hand control. The semantic frame is generated here, above
|
|
526
1027
|
// every renderer, so SVG, WebGL, and video rigs receive exactly the same
|
|
527
1028
|
// gesture/progress information. A handless rig simply ignores `frame.hand`.
|
|
528
1029
|
const handFrame = updateHandAction(elapsed * 1000);
|
|
529
|
-
rig.apply(avatarFrame(
|
|
1030
|
+
rig.apply(avatarFrame(shown, handFrame || undefined));
|
|
530
1031
|
}
|
|
531
1032
|
|
|
532
1033
|
const REST_SHAPE = shapeFor(SILENT, 1);
|
|
@@ -534,8 +1035,16 @@ export function createAvatar(opts = {}) {
|
|
|
534
1035
|
// How much of a sustained head turn the trunk takes up. Well under 1: people
|
|
535
1036
|
// under-rotate the head and then under-rotate the trunk again behind it, and
|
|
536
1037
|
// at a head-and-shoulders crop the trunk's share is the part you register
|
|
537
|
-
// without noticing.
|
|
538
|
-
|
|
1038
|
+
// without noticing. A rig may pass its own `trunkFollow`: on a mesh head whose
|
|
1039
|
+
// neck twists, the trunk's sway is most of what moves the neck's outline, so
|
|
1040
|
+
// the share that reads as a body on a line face reads there as the neck
|
|
1041
|
+
// sliding.
|
|
1042
|
+
const TRUNK_FOLLOW = opts.trunkFollow ?? 0.45;
|
|
1043
|
+
|
|
1044
|
+
// Speakers are back on the listener's eyes this long before they stop
|
|
1045
|
+
// (Andrist, docs/research-biomechanics.md §4.2): the floor is handed over
|
|
1046
|
+
// under mutual gaze, and a look away there declines it.
|
|
1047
|
+
const FLOOR_RETURN_MS = 2400;
|
|
539
1048
|
|
|
540
1049
|
// The channels speech owns outright — exactly the params.js mouth group
|
|
541
1050
|
// (mouth corners stay free: a clip may smile over a sentence).
|
|
@@ -545,42 +1054,144 @@ export function createAvatar(opts = {}) {
|
|
|
545
1054
|
// ~a third keeps the face warm without the corners fighting the visemes;
|
|
546
1055
|
// full warmth returns the moment the track ends, which is exactly the
|
|
547
1056
|
// episodic onset/offset a credible smile needs (research-perception.md §3).
|
|
548
|
-
|
|
1057
|
+
/** The shoulders' share of the attentive posture, against `torsoLean`'s 0.16.
|
|
1058
|
+
* Two thirds of the lean: the shoulders come up with it, they do not lead it. */
|
|
1059
|
+
const ENGAGE_SHOULDER = 0.10;
|
|
1060
|
+
const SPEAK_SMILE_RETAIN = 0.35;
|
|
1061
|
+
|
|
1062
|
+
// How long a reply that starts mid-think keeps the look before it comes
|
|
1063
|
+
// back. Speakers look away to find the words and back to deliver them
|
|
1064
|
+
// (§4.2); a reply that snapped to the user on its first syllable and then,
|
|
1065
|
+
// three times in four, left again for the turn-start look did both halves
|
|
1066
|
+
// twice.
|
|
1067
|
+
const THINK_CARRY = [0.25, 0.6];
|
|
1068
|
+
|
|
1069
|
+
// Between the user's turn and the reply the server's claim can change
|
|
1070
|
+
// several times a second — THINKING, a tool's WORKING, THINKING again, a
|
|
1071
|
+
// grace timer's CANT_HEAR — and every change used to retarget the eyes and
|
|
1072
|
+
// restart the state's looks: one fast tool call was away, down at the
|
|
1073
|
+
// screen, back to the user and away again inside a second, a blink at each.
|
|
1074
|
+
// Among these states the eyes change over only once the new one has held
|
|
1075
|
+
// this long, and a claim that returns before then moves nothing. The pose
|
|
1076
|
+
// still changes at once — it is the eyes that make a flicker visible.
|
|
1077
|
+
// SPEAKING and LISTENING are never held back: those are the floor.
|
|
1078
|
+
const GAP_STATES = new Set(['THINKING', 'WORKING', 'CANT_HEAR']);
|
|
1079
|
+
const GAP_SETTLE = 0.5;
|
|
1080
|
+
|
|
1081
|
+
// A dart leaves at up to 40° off the line of the look it is inside, and
|
|
1082
|
+
// lifts the brows for half a second — a thought arriving, not a stare.
|
|
1083
|
+
const DART_TURN = 0.7;
|
|
1084
|
+
const DART_BROW = 0.5;
|
|
1085
|
+
|
|
1086
|
+
/** A look away for the return leg of a glance. `stick` of the time it keeps
|
|
1087
|
+
* the side the last one took, spot and all — the side a person thinks
|
|
1088
|
+
* toward is theirs, and it outlasts one thinking pause. Otherwise any look
|
|
1089
|
+
* but the one just taken: a thinker who goes back to the same patch of
|
|
1090
|
+
* wall every time *without* meaning to is a loop. */
|
|
1091
|
+
function pickBack(xs, stick = 0) {
|
|
1092
|
+
const side = (n) => Math.sign(gaze.targets[n].px);
|
|
1093
|
+
let g;
|
|
1094
|
+
if (lastBack && Math.random() < stick) {
|
|
1095
|
+
const same = xs.filter((n) => side(n) === side(lastBack));
|
|
1096
|
+
g = pick(same.length ? same : xs);
|
|
1097
|
+
} else {
|
|
1098
|
+
g = pick(xs);
|
|
1099
|
+
for (let i = 0; i < 4 && g === lastBack; i++) g = pick(xs);
|
|
1100
|
+
}
|
|
1101
|
+
lastBack = g;
|
|
1102
|
+
return g;
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* One fixation of a reading scan: a step along the line, or at its end a
|
|
1107
|
+
* sweep back to the start of the next, or at the end of the block a pause
|
|
1108
|
+
* on the result. That is what busy-at-a-screen looks like from the other
|
|
1109
|
+
* side of a webcam — an irregular left-to-right march with returns, which
|
|
1110
|
+
* a random drift about a point is not. Steps are held, like a reader's
|
|
1111
|
+
* fixations, and the head sits out all of it (DRIFT_HEAD in gaze.js).
|
|
1112
|
+
*/
|
|
1113
|
+
function readStep(r) {
|
|
1114
|
+
const n = ([a, b]) => a + ((Math.random() * (b - a + 1)) | 0);
|
|
1115
|
+
if (readCol < readCols) {
|
|
1116
|
+
readCol++;
|
|
1117
|
+
driftAt = elapsed + rand(r.fix);
|
|
1118
|
+
} else {
|
|
1119
|
+
readCol = 0;
|
|
1120
|
+
readCols = n(r.steps);
|
|
1121
|
+
if (++readRow >= readRows) {
|
|
1122
|
+
readRow = 0;
|
|
1123
|
+
readRows = n(r.lines);
|
|
1124
|
+
driftAt = elapsed + rand(r.pause);
|
|
1125
|
+
} else {
|
|
1126
|
+
driftAt = elapsed + rand(r.fix) * 1.4;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
gaze.drift.x = (readCol / readCols - 0.5) * r.width * saccadeGain;
|
|
1130
|
+
gaze.drift.y = (readRow - (readRows - 1) / 2) * r.line * saccadeGain;
|
|
1131
|
+
}
|
|
549
1132
|
|
|
550
1133
|
// --- API ------------------------------------------------------------------
|
|
551
1134
|
|
|
552
1135
|
function setState(name, o = {}) {
|
|
553
|
-
if (!
|
|
1136
|
+
if (!states[name]) throw new Error(`unknown state: ${name}`);
|
|
554
1137
|
const changed = name !== stateName;
|
|
555
1138
|
stateName = name;
|
|
556
|
-
const st =
|
|
1139
|
+
const st = states[name];
|
|
557
1140
|
if (o.emotion !== undefined) emotion = o.emotion;
|
|
558
1141
|
else if (changed) emotion = st.emotion;
|
|
559
1142
|
if (o.intensity !== undefined) emotionAmt = o.intensity;
|
|
560
|
-
|
|
561
|
-
if (!o.keepGaze) setGaze(o.gaze || st.gaze);
|
|
562
1143
|
idle.setProfile(st.idle);
|
|
563
|
-
// Arm the glance scheduler fresh so entering a glancing state doesn't
|
|
564
|
-
// fire a stale timestamp immediately.
|
|
565
|
-
glanceUntil = 0;
|
|
566
|
-
glanceAt = elapsed + (st.glance ? st.glance.every[0] + Math.random() * (st.glance.every[1] - st.glance.every[0]) : 0);
|
|
567
1144
|
// SVG's desaturation filter is a legacy renderer detail. A generic rig
|
|
568
1145
|
// receives the same state pose and may express degradation its own way.
|
|
569
1146
|
if (face) {
|
|
570
1147
|
face.svg.style.filter = st.filter || '';
|
|
571
1148
|
face.svg.style.transition = 'filter .5s ease';
|
|
572
1149
|
}
|
|
573
|
-
|
|
1150
|
+
// The eyes wait out a flicker between gap states (GAP_SETTLE), and a
|
|
1151
|
+
// repeat of the state still settling leaves it settling.
|
|
1152
|
+
const deferrable = !o.gaze && !o.keepGaze && GAP_STATES.has(gazeState) && GAP_STATES.has(name);
|
|
1153
|
+
if (deferrable && changed) settleAt = name === gazeState ? 0 : elapsed + GAP_SETTLE;
|
|
1154
|
+
else if (!(deferrable && settleAt)) { settleAt = 0; enterGaze(name, o, changed); }
|
|
1155
|
+
if (changed) emit('state', name);
|
|
574
1156
|
return api;
|
|
575
1157
|
}
|
|
576
1158
|
|
|
1159
|
+
/** The state `name` takes the gaze: its target and its schedules. */
|
|
1160
|
+
function enterGaze(name, o, blink) {
|
|
1161
|
+
const prev = gazeState;
|
|
1162
|
+
gazeState = name;
|
|
1163
|
+
const st = states[name];
|
|
1164
|
+
const gl = st.glance;
|
|
1165
|
+
// A reply that starts while THINKING is looking away finishes that look
|
|
1166
|
+
// first, and the look stands in for the turn-start one.
|
|
1167
|
+
const carry = prev === 'THINKING' && name === 'SPEAKING'
|
|
1168
|
+
&& !o.gaze && !o.keepGaze && gazeName !== st.gaze && !glanceUntil;
|
|
1169
|
+
returnAt = carry ? elapsed + rand(THINK_CARRY) : 0;
|
|
1170
|
+
if (carry) carriedAt = elapsed;
|
|
1171
|
+
// A state with an `opening` enters as though its check-in on the user is
|
|
1172
|
+
// already under way, and leaves it when that runs out.
|
|
1173
|
+
if (!o.keepGaze && !carry) setGaze(o.gaze || (gl && gl.opening ? gl.to : st.gaze));
|
|
1174
|
+
// Arm every scheduler fresh, so entering a state never fires a timestamp
|
|
1175
|
+
// left over from the last one — the wander in particular, which used to
|
|
1176
|
+
// pick a new target on the first frame and override the state's own gaze.
|
|
1177
|
+
glanceUntil = gl && gl.opening && !o.gaze && !o.keepGaze ? elapsed + rand(gl.opening) : 0;
|
|
1178
|
+
glanceAt = elapsed + (gl ? rand(gl.every) : 0);
|
|
1179
|
+
wanderAt = elapsed + (st.wander ? rand(st.wander.every) : 0);
|
|
1180
|
+
driftAt = elapsed + (st.read ? rand(st.read.fix) : 0);
|
|
1181
|
+
readCol = readCols = readRow = readRows = 0;
|
|
1182
|
+
dartAt = dartBrowUntil = 0;
|
|
1183
|
+
dart.x = dart.y = 0;
|
|
1184
|
+
if (blink) idle.blink(false, true);
|
|
1185
|
+
}
|
|
1186
|
+
|
|
577
1187
|
function setEmotion(name, intensity = 1) { emotion = name; emotionAmt = intensity; return api; }
|
|
578
1188
|
|
|
579
|
-
/** @param {string} name @param {{x:number,y:number}} [custom] normalized -1..1
|
|
580
|
-
|
|
1189
|
+
/** @param {string} name @param {{x:number,y:number}} [custom] normalized -1..1
|
|
1190
|
+
* @param {boolean} [blink] the mixer's own say over the evoked blink; see GazeLayer.set */
|
|
1191
|
+
function setGaze(name, custom, blink) {
|
|
581
1192
|
gazeName = GAZE_TARGETS[name] ? name : 'USER';
|
|
582
1193
|
gazeCustom = custom || null;
|
|
583
|
-
applyGaze();
|
|
1194
|
+
applyGaze(blink);
|
|
584
1195
|
return api;
|
|
585
1196
|
}
|
|
586
1197
|
|
|
@@ -594,6 +1205,10 @@ export function createAvatar(opts = {}) {
|
|
|
594
1205
|
// Speech owns the mouth in the mixer, but a server action still gets to
|
|
595
1206
|
// complete its physical landing on the other channels. Do not cancel it
|
|
596
1207
|
// here: a hand or head cannot disappear simply because playout began.
|
|
1208
|
+
// A new clock is a new turn. The same clock handed back is the accurate
|
|
1209
|
+
// leg rewriting this one (AvatarClient re-speaks a splice), and that must
|
|
1210
|
+
// not roll the turn-start look a second time.
|
|
1211
|
+
const newTurn = !o.clock || o.clock !== speakClock;
|
|
597
1212
|
speakStart = performance.now();
|
|
598
1213
|
speakClock = o.clock
|
|
599
1214
|
? o.clock
|
|
@@ -601,6 +1216,8 @@ export function createAvatar(opts = {}) {
|
|
|
601
1216
|
? () => o.audio.currentTime * 1000
|
|
602
1217
|
: () => performance.now() - speakStart;
|
|
603
1218
|
speech.start(o.cues || [], speakClock);
|
|
1219
|
+
prosody.reset(newTurn);
|
|
1220
|
+
if (newTurn) turnStartPending = true;
|
|
604
1221
|
if (stateName !== 'SPEAKING') setState('SPEAKING', { keepGaze: true });
|
|
605
1222
|
if (o.audio && o.audio.paused) o.audio.play().catch(() => {});
|
|
606
1223
|
return api;
|
|
@@ -674,19 +1291,46 @@ export function createAvatar(opts = {}) {
|
|
|
674
1291
|
/** One self-completing server action. State continues to resolve underneath;
|
|
675
1292
|
* face and hand layers queue their next movement so an in-flight physical
|
|
676
1293
|
* gesture always gets to land. */
|
|
677
|
-
function action(
|
|
1294
|
+
function action(wireId) {
|
|
1295
|
+
// `ACKNOWLEDGE` is the whole backchannel family in one word, and which of
|
|
1296
|
+
// them a face makes is a rendering decision the server is not holding the
|
|
1297
|
+
// drawing for. The floor decides: a nod while the user still has it is a
|
|
1298
|
+
// continuer, "go on"; once they have stopped it is a receipt, and a nod
|
|
1299
|
+
// there reads as agreement with whatever they just said. Both are only ever
|
|
1300
|
+
// this explicit action — nothing here starts one on its own.
|
|
1301
|
+
const id = wireId === 'ACKNOWLEDGE'
|
|
1302
|
+
? (engagement.speaking ? 'ACK_NOD' : 'ACK_RECEIVE')
|
|
1303
|
+
: wireId;
|
|
678
1304
|
const handDef = HAND_GESTURES[id];
|
|
679
1305
|
if (handDef) {
|
|
680
1306
|
startHandAction(id, handDef);
|
|
681
1307
|
if (handDef.face) {
|
|
682
|
-
const faceClip = ACTIONS[handDef.face];
|
|
1308
|
+
const faceClip = actionShapes[handDef.face] || ACTIONS[handDef.face];
|
|
683
1309
|
if (faceClip) clip.play(faceClip, faceClip.audioEl, { queue: true });
|
|
684
1310
|
}
|
|
685
1311
|
return api;
|
|
686
1312
|
}
|
|
687
|
-
|
|
688
|
-
|
|
1313
|
+
// Own renderings first, always. An avatar may *add* to what a server can
|
|
1314
|
+
// ask for and may never redefine a core intent: `ACKNOWLEDGE` has to mean
|
|
1315
|
+
// the same thing on every face or it is not a protocol. It may draw that
|
|
1316
|
+
// meaning in its own shape (`actionShapes`), which is a rendering, not a
|
|
1317
|
+
// redefinition — the id and the intent are fixed before this line.
|
|
1318
|
+
//
|
|
1319
|
+
// Unknown is a no-op, not a throw. The wire's action vocabulary is open, so
|
|
1320
|
+
// a server asking this face for a motion it does not have is the expected
|
|
1321
|
+
// case and not somebody's broken build — the same forward-compat rule an
|
|
1322
|
+
// unknown `cmd` gets. This is also where a sequence used to arrive by its
|
|
1323
|
+
// own method; one open vocabulary means one door.
|
|
1324
|
+
const faceClip = actionShapes[id] || ACTIONS[id] || sequences[id];
|
|
1325
|
+
if (!faceClip) return api;
|
|
689
1326
|
clip.play(faceClip, faceClip.audioEl, { queue: true });
|
|
1327
|
+
// The two acknowledgements smile. Only these, and only ever because the
|
|
1328
|
+
// server sent one: a smile the renderer timed for itself would be an
|
|
1329
|
+
// acknowledgement nobody sent.
|
|
1330
|
+
if (id === 'ACK_NOD' || id === 'ACK_RECEIVE') prosody.acknowledge();
|
|
1331
|
+
// Now that warmth rides over clips, the reply's opening smile would
|
|
1332
|
+
// otherwise carry on through the interrupted face.
|
|
1333
|
+
if (id === 'RESPONSE_INTERRUPTED') prosody.cool();
|
|
690
1334
|
return api;
|
|
691
1335
|
}
|
|
692
1336
|
|
|
@@ -773,7 +1417,7 @@ export function createAvatar(opts = {}) {
|
|
|
773
1417
|
}
|
|
774
1418
|
|
|
775
1419
|
const api = {
|
|
776
|
-
setState, setEmotion, setGaze, speak, pushCues, stopSpeaking, attend,
|
|
1420
|
+
setState, setEmotion, setGaze: (name, custom) => setGaze(name, custom), speak, pushCues, stopSpeaking, attend,
|
|
777
1421
|
action, perform,
|
|
778
1422
|
/** Which hand the character gestures with: +1 the viewer's right (its own
|
|
779
1423
|
* left), -1 the other. Both are anatomically real — the thumb splays away
|
|
@@ -804,7 +1448,7 @@ export function createAvatar(opts = {}) {
|
|
|
804
1448
|
get clip() { return clip.id; },
|
|
805
1449
|
/** Semantic hand gesture in flight, independent of renderer capability. */
|
|
806
1450
|
get gesturing() { return handAction ? handAction.id : null; },
|
|
807
|
-
get params() { return
|
|
1451
|
+
get params() { return shown; },
|
|
808
1452
|
get userSpeaking() { return engagement.speaking; },
|
|
809
1453
|
// Legacy SVG inspection fields. New AvatarRig implementations should not
|
|
810
1454
|
// rely on or provide them; the renderer-neutral contract is apply/destroy.
|
|
@@ -826,6 +1470,11 @@ export function createAvatar(opts = {}) {
|
|
|
826
1470
|
return api;
|
|
827
1471
|
}
|
|
828
1472
|
|
|
1473
|
+
// The parameter space itself. A renderer-neutral rig has to answer "what is
|
|
1474
|
+
// this channel's neutral value" before it can map the channel onto whatever it
|
|
1475
|
+
// controls — a morph target's influence is `(pose - rest) / (1 - rest)`, and a
|
|
1476
|
+
// rig that hard-codes those rests has quietly forked `params.js`.
|
|
1477
|
+
export { REST, CHANNELS, RANGE } from './params.js';
|
|
829
1478
|
export { ACTION_IDS, ACTIONS, attachAudio } from './interjections.js';
|
|
830
1479
|
export { GAZE_NAMES, GAZE_TARGETS } from './gaze.js';
|
|
831
1480
|
export { normalizeActions } from './perform.js';
|