@voqalize/avatar 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -154
- package/dist/index.d.ts +37 -26
- package/dist/index.js +48 -25
- package/dist/pin.d.ts +2 -0
- package/dist/pin.js +3 -0
- package/dist/react.d.ts +21 -6
- package/dist/react.js +24 -6
- package/package.json +8 -67
- package/LICENSE-APACHE-2.0 +0 -201
- package/LICENSE-CC-BY-4.0 +0 -396
- package/assets/README.md +0 -60
- package/assets/tanvi.glb +0 -0
- package/assets/tanya.glb +0 -0
- package/assets/tara.glb +0 -0
- package/assets/tess.glb +0 -0
- package/assets/tushar.glb +0 -0
- package/client/Avatar.tsx +0 -57
- package/client/AvatarClient.ts +0 -818
- package/client/createAvatar.ts +0 -177
- package/client/index.ts +0 -44
- package/client/internal.ts +0 -132
- package/client/playout.ts +0 -95
- package/client/react.ts +0 -13
- package/client/supports.ts +0 -27
- package/client/three/assets.ts +0 -26
- package/client/three/budgets.ts +0 -27
- package/client/three/character-rig.ts +0 -1746
- package/client/three/createCharacter.ts +0 -109
- package/client/three/holds.ts +0 -69
- package/client/three/internal.ts +0 -55
- package/client/three/motion-limits.json +0 -124
- package/client/three/sequences.ts +0 -281
- package/client/three/tanvi-asset.ts +0 -6
- package/client/three/tanvi.ts +0 -30
- package/client/three/tanya-asset.ts +0 -6
- package/client/three/tanya.ts +0 -29
- package/client/three/tara-asset.ts +0 -25
- package/client/three/tara.ts +0 -36
- package/client/three/tess-asset.ts +0 -6
- package/client/three/tess.ts +0 -30
- package/client/three/tushar-asset.ts +0 -6
- package/client/three/tushar.ts +0 -28
- package/client/types.ts +0 -235
- package/client/useAvatar.ts +0 -74
- package/dist/Avatar.d.ts +0 -44
- package/dist/Avatar.d.ts.map +0 -1
- package/dist/Avatar.js +0 -7
- package/dist/Avatar.js.map +0 -1
- package/dist/AvatarClient.d.ts +0 -274
- package/dist/AvatarClient.d.ts.map +0 -1
- package/dist/AvatarClient.js +0 -712
- package/dist/AvatarClient.js.map +0 -1
- package/dist/createAvatar.d.ts +0 -135
- package/dist/createAvatar.d.ts.map +0 -1
- package/dist/createAvatar.js +0 -75
- package/dist/createAvatar.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/internal.d.ts +0 -34
- package/dist/internal.d.ts.map +0 -1
- package/dist/internal.js +0 -56
- package/dist/internal.js.map +0 -1
- package/dist/playout.d.ts +0 -42
- package/dist/playout.d.ts.map +0 -1
- package/dist/playout.js +0 -85
- package/dist/playout.js.map +0 -1
- package/dist/react.d.ts.map +0 -1
- package/dist/react.js.map +0 -1
- package/dist/supports.d.ts +0 -19
- package/dist/supports.d.ts.map +0 -1
- package/dist/supports.js +0 -25
- package/dist/supports.js.map +0 -1
- package/dist/three/assets.d.ts +0 -10
- package/dist/three/assets.d.ts.map +0 -1
- package/dist/three/assets.js +0 -23
- package/dist/three/assets.js.map +0 -1
- package/dist/three/budgets.d.ts +0 -17
- package/dist/three/budgets.d.ts.map +0 -1
- package/dist/three/budgets.js +0 -21
- package/dist/three/budgets.js.map +0 -1
- package/dist/three/character-rig.d.ts +0 -366
- package/dist/three/character-rig.d.ts.map +0 -1
- package/dist/three/character-rig.js +0 -1644
- package/dist/three/character-rig.js.map +0 -1
- package/dist/three/createCharacter.d.ts +0 -60
- package/dist/three/createCharacter.d.ts.map +0 -1
- package/dist/three/createCharacter.js +0 -84
- package/dist/three/createCharacter.js.map +0 -1
- package/dist/three/holds.d.ts +0 -38
- package/dist/three/holds.d.ts.map +0 -1
- package/dist/three/holds.js +0 -61
- package/dist/three/holds.js.map +0 -1
- package/dist/three/internal.d.ts +0 -52
- package/dist/three/internal.d.ts.map +0 -1
- package/dist/three/internal.js +0 -49
- package/dist/three/internal.js.map +0 -1
- package/dist/three/motion-limits.json +0 -124
- package/dist/three/sequences.d.ts +0 -71
- package/dist/three/sequences.d.ts.map +0 -1
- package/dist/three/sequences.js +0 -262
- package/dist/three/sequences.js.map +0 -1
- package/dist/three/tanvi-asset.d.ts +0 -7
- package/dist/three/tanvi-asset.d.ts.map +0 -1
- package/dist/three/tanvi-asset.js +0 -7
- package/dist/three/tanvi-asset.js.map +0 -1
- package/dist/three/tanvi.d.ts +0 -24
- package/dist/three/tanvi.d.ts.map +0 -1
- package/dist/three/tanvi.js +0 -24
- package/dist/three/tanvi.js.map +0 -1
- package/dist/three/tanya-asset.d.ts +0 -7
- package/dist/three/tanya-asset.d.ts.map +0 -1
- package/dist/three/tanya-asset.js +0 -7
- package/dist/three/tanya-asset.js.map +0 -1
- package/dist/three/tanya.d.ts +0 -23
- package/dist/three/tanya.d.ts.map +0 -1
- package/dist/three/tanya.js +0 -23
- package/dist/three/tanya.js.map +0 -1
- package/dist/three/tara-asset.d.ts +0 -26
- package/dist/three/tara-asset.d.ts.map +0 -1
- package/dist/three/tara-asset.js +0 -26
- package/dist/three/tara-asset.js.map +0 -1
- package/dist/three/tara.d.ts +0 -30
- package/dist/three/tara.d.ts.map +0 -1
- package/dist/three/tara.js +0 -30
- package/dist/three/tara.js.map +0 -1
- package/dist/three/tess-asset.d.ts +0 -7
- package/dist/three/tess-asset.d.ts.map +0 -1
- package/dist/three/tess-asset.js +0 -7
- package/dist/three/tess-asset.js.map +0 -1
- package/dist/three/tess.d.ts +0 -24
- package/dist/three/tess.d.ts.map +0 -1
- package/dist/three/tess.js +0 -24
- package/dist/three/tess.js.map +0 -1
- package/dist/three/tushar-asset.d.ts +0 -7
- package/dist/three/tushar-asset.d.ts.map +0 -1
- package/dist/three/tushar-asset.js +0 -7
- package/dist/three/tushar-asset.js.map +0 -1
- package/dist/three/tushar.d.ts +0 -22
- package/dist/three/tushar.d.ts.map +0 -1
- package/dist/three/tushar.js +0 -22
- package/dist/three/tushar.js.map +0 -1
- package/dist/types.d.ts +0 -149
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -140
- package/dist/types.js.map +0 -1
- package/dist/useAvatar.d.ts +0 -39
- package/dist/useAvatar.d.ts.map +0 -1
- package/dist/useAvatar.js +0 -35
- package/dist/useAvatar.js.map +0 -1
- package/src/avatar.d.ts +0 -436
- package/src/avatar.js +0 -1378
- package/src/behavior.d.ts +0 -42
- package/src/behavior.js +0 -114
- package/src/camera.js +0 -29
- package/src/clips.js +0 -155
- package/src/conformance.js +0 -119
- package/src/emotions.js +0 -58
- package/src/face-core.js +0 -180
- package/src/face-myna.d.ts +0 -7
- package/src/face-myna.js +0 -786
- package/src/face-peep-control-plane.js +0 -167
- package/src/face-peep.d.ts +0 -7
- package/src/face-peep.js +0 -925
- package/src/face-wren.d.ts +0 -7
- package/src/face-wren.js +0 -468
- package/src/faces.d.ts +0 -12
- package/src/faces.js +0 -53
- package/src/gaze.js +0 -504
- package/src/hand.js +0 -709
- package/src/head.js +0 -134
- package/src/idle.js +0 -668
- package/src/interjections.js +0 -747
- package/src/line-art.js +0 -137
- package/src/params.js +0 -180
- package/src/perform.js +0 -107
- package/src/prosody.js +0 -741
- package/src/rig.d.ts +0 -31
- package/src/rig.js +0 -32
- package/src/speech-timing.js +0 -51
- package/src/visemes.js +0 -410
package/src/avatar.js
DELETED
|
@@ -1,1378 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The mixer — a programmable talking head.
|
|
3
|
-
*
|
|
4
|
-
* Not the public surface: that is `createAvatar({ mount, client })` in
|
|
5
|
-
* `packages/avatar/client/createAvatar.ts`, and this is what it drives. Everything below is
|
|
6
|
-
* addressed by our own runtime, our tooling, and an avatar author who chose to
|
|
7
|
-
* build on the SVG renderer (`@voqalize/avatar/internal`, no semver promise).
|
|
8
|
-
*
|
|
9
|
-
* Per frame the mixer runs a fixed layer order. Earlier layers are overwritten
|
|
10
|
-
* by later ones on the channels they touch; the gesture and idle layers are
|
|
11
|
-
* additive so they compose rather than fight.
|
|
12
|
-
*
|
|
13
|
-
* base pose (state + emotion) -> gaze -> visemes -> clip -> idle
|
|
14
|
-
*/
|
|
15
|
-
|
|
16
|
-
import { REST, CHANNELS, TAU, RANGE, GROUPS, clamp, approach } from './params.js';
|
|
17
|
-
import { EMOTIONS, emotionPose } from './emotions.js';
|
|
18
|
-
import { GazeLayer, GAZE_TARGETS, AVERSION } from './gaze.js';
|
|
19
|
-
import { IdleLayer, ListeningEngine } from './idle.js';
|
|
20
|
-
import { ClipPlayer } from './clips.js';
|
|
21
|
-
import { ACTIONS, INTERNAL_CLIPS } from './interjections.js';
|
|
22
|
-
import { VisemeTrack, shapeFor, SILENT } from './visemes.js';
|
|
23
|
-
import { LIP_CONTACT_AIM } from './speech-timing.js';
|
|
24
|
-
import { SpeechProsody, UNCALIBRATED_HEAD_GAIN } from './prosody.js';
|
|
25
|
-
import { HEAD_AXES, soften } from './head.js';
|
|
26
|
-
import { PerformTrack } from './perform.js';
|
|
27
|
-
import { createHand, HAND_GESTURES, HAND_ACTION_TO_FRAME_GESTURE } from './hand.js';
|
|
28
|
-
import { avatarFrame, createSvgRig } from './rig.js';
|
|
29
|
-
|
|
30
|
-
const rand = ([a, b]) => a + Math.random() * (b - a);
|
|
31
|
-
const pick = (xs) => xs[(Math.random() * xs.length) | 0];
|
|
32
|
-
|
|
33
|
-
// Each state's `idle` is a profile for the liveness layer (see DEFAULT_PROFILE
|
|
34
|
-
// in idle.js). Blink gaps come from docs/research-biomechanics.md §5: the rate
|
|
35
|
-
// alone separates listening (~16/min) from thinking (~25/min) from visually
|
|
36
|
-
// busy (~9/min), and it is the cheapest state signal the rig has. The gap is
|
|
37
|
-
// the budget for every blink, paired ones included (about one in six), so it
|
|
38
|
-
// sits a little longer than 60/rate.
|
|
39
|
-
export const STATES = {
|
|
40
|
-
// Idle means present and relaxed, not attending. It is also the face before
|
|
41
|
-
// a call connects — the first one anyone sees — so it rests on the user and
|
|
42
|
-
// leaves for soft, unhurried looks (AVERSION.IDLE) rather than parking the
|
|
43
|
-
// eyes somewhere. It used to open on AWAY_THINKING and wander between it,
|
|
44
|
-
// AWAY_RIGHT and NOTES from the first frame: a face that loads looking up
|
|
45
|
-
// and to the left, at nothing, reads as broken rather than idle. What keeps
|
|
46
|
-
// it apart from LISTENING is the longer, lazier looks, the slower scan and
|
|
47
|
-
// the missing engagement lean, not refusing to look at the user.
|
|
48
|
-
IDLE: { gaze: 'USER', emotion: 'neutral', engagement: false,
|
|
49
|
-
aversion: 'IDLE', scan: [0.8, 2.0],
|
|
50
|
-
idle: { sway: 0.72, blinkGap: [3.6, 5.2] } },
|
|
51
|
-
// `aversion` is why this state does not stare. Continuous eye contact is not
|
|
52
|
-
// the attentive pose it looks like — it is a demand for more talk (Rossano)
|
|
53
|
-
// and it measures as *tense*, not attentive (Wang & Gratch). See AVERSION in
|
|
54
|
-
// gaze.js for the numbers; the mixer holds it off near a turn boundary.
|
|
55
|
-
// At 3.2-4.4 s the blink timer measured 20-23/min.
|
|
56
|
-
LISTENING: { gaze: 'USER', emotion: 'neutral', engagement: true,
|
|
57
|
-
aversion: 'LISTEN',
|
|
58
|
-
idle: { sway: 1.0, blinkGap: [3.6, 4.8] },
|
|
59
|
-
pose: { browRaiseL: 0.06, browRaiseR: 0.06, lidL: -0.04, lidR: -0.04 } },
|
|
60
|
-
// Faster, shallower breath is the measured cognitive-load signature, and the
|
|
61
|
-
// occasional dead-still hold is the strongest "working on it" cue a rig this
|
|
62
|
-
// simple can make — deliberate stillness, not more motion. The look away is
|
|
63
|
-
// Andrist's cognitive aversion (§4.2): one look held ~3.5 s (SD 1.26) that
|
|
64
|
-
// drifts a little where it sits, a check-in with the user, then the next.
|
|
65
|
-
//
|
|
66
|
-
// In a call this state mostly lasts under two seconds — the gap between the
|
|
67
|
-
// user finishing and the reply starting — so its opening is what gets seen,
|
|
68
|
-
// and the opening used to be wrong. The eyes left the instant the user
|
|
69
|
-
// stopped, down and to the left, every turn: a listener who drops their
|
|
70
|
-
// eyes the moment you finish reads as ashamed or done with you. Now the
|
|
71
|
-
// eyes hold the user for `opening` first (a person takes in the end of
|
|
72
|
-
// what was said before they go to think), and a fast reply never looks
|
|
73
|
-
// away at all. The look is mostly up-and-aside or level-aside: that is the
|
|
74
|
-
// one audiences read as thinking, whatever the measured split, and down
|
|
75
|
-
// is kept to a fifth of looks because on a face this real it reads as
|
|
76
|
-
// downcast. The pose takes back `thoughtful`'s lid drop: a thinking face is
|
|
77
|
-
// alert, and the two together measured past the 0.15 that reads drowsy.
|
|
78
|
-
THINKING: { gaze: 'AWAY_SIDE', emotion: 'thoughtful', engagement: false,
|
|
79
|
-
// Fixational jumps rare and small: a thinker's eyes rest
|
|
80
|
-
// where they land. At the default gap the look jittered
|
|
81
|
-
// about its spot every second, and with a drift on top
|
|
82
|
-
// the eyes never held anything — searching the wall, not
|
|
83
|
-
// thinking.
|
|
84
|
-
scan: [1.2, 2.6, 0.35],
|
|
85
|
-
idle: { sway: 0.7, blinkGap: [2.6, 3.2], slowBlink: 0.3,
|
|
86
|
-
breathRate: 1.18, breathAmp: 0.7,
|
|
87
|
-
hold: { every: [4.5, 9.0], dur: [0.8, 1.5] } },
|
|
88
|
-
// `stick`: people have a side they look to when they
|
|
89
|
-
// think and keep to it (Day; Kinsbourne's lateral eye
|
|
90
|
-
// movements) — one who alternates sides on every look is
|
|
91
|
-
// scanning the room. `dart`: now and then, once in a
|
|
92
|
-
// look, the eyes move on to a second spot with a flick of
|
|
93
|
-
// the brows, which is the thought moving; the random
|
|
94
|
-
// drift this replaces never stopped and read as roving.
|
|
95
|
-
// `blinkTo`: the look back to the user always blinks —
|
|
96
|
-
// the re-engagement is the beat to see.
|
|
97
|
-
glance: { to: 'USER', opening: [0.35, 0.7], every: [2.2, 4.4], hold: [0.8, 1.3],
|
|
98
|
-
back: ['AWAY_THINKING', 'AWAY_THINKING', 'AWAY_THINKING',
|
|
99
|
-
'AWAY_SIDE', 'AWAY_SIDE', 'AWAY_SIDE',
|
|
100
|
-
'AWAY_RIGHT', 'AWAY_RIGHT', 'AWAY_DOWN', 'AWAY_DOWN'],
|
|
101
|
-
stick: 0.55, dart: { p: 0.4, mag: 0.08, brow: 0.12 }, blinkTo: true },
|
|
102
|
-
// **The chin comes up**: thinking is the one stretch of
|
|
103
|
-
// a call where the avatar owes the user feedback and
|
|
104
|
-
// has no mouth to give it with, the eyes being off the
|
|
105
|
-
// user by design. A head tipping back as the gaze
|
|
106
|
-
// leaves is the swing-up of research-biomechanics.md
|
|
107
|
-
// § 3.3. It rides the looks that go up, as the head's
|
|
108
|
-
// *share* of them: held as a pose it lifted the chin on
|
|
109
|
-
// the level and downward looks too, so the user was
|
|
110
|
-
// looked at down the nose.
|
|
111
|
-
pose: { lidL: -0.10, lidR: -0.10 } },
|
|
112
|
-
// Eyes on the user for the whole turn, and no `aversion`: gaze.js has why a
|
|
113
|
-
// speaker's measured looks away are not this rig's to render. What moves
|
|
114
|
-
// while it talks is the head it holds per phrase and the trunk under it
|
|
115
|
-
// (prosody.js), never the eyes leaving the user.
|
|
116
|
-
SPEAKING: { gaze: 'USER', emotion: 'neutral',
|
|
117
|
-
idle: { sway: 0.55 }, engagement: false },
|
|
118
|
-
REVIEWING_SCREEN: { gaze: 'SCREEN_CENTER', emotion: 'thoughtful', engagement: false,
|
|
119
|
-
idle: { sway: 0.8, blinkGap: [4.0, 6.5] },
|
|
120
|
-
wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_RIGHT', 'SCREEN_TOP', 'SCREEN_WORK'],
|
|
121
|
-
every: [1.8, 5.0] } },
|
|
122
|
-
// The head cant is the state's signature cue, and it has to clear the roll
|
|
123
|
-
// multiplier to exist at all: 0.05 here renders as 0.3° of rotation, which
|
|
124
|
-
// is no tilt whatever the number says. 0.30 renders ~1.7° — visible at tile
|
|
125
|
-
// size, still gentle.
|
|
126
|
-
WAITING_FOR_USER: { gaze: 'USER', emotion: 'encouraging', engagement: true,
|
|
127
|
-
idle: { sway: 1.0, blinkGap: [3.1, 4.2] },
|
|
128
|
-
pose: { headRoll: 0.30, browRaiseL: 0.16, browRaiseR: 0.12 } },
|
|
129
|
-
// Straining to hear. The one state where the amplitude constraint yields,
|
|
130
|
-
// because the lean IS the message: torsoLean well past what the engagement
|
|
131
|
-
// layer ever spends, head cheated aside on USER_EAR so an ear
|
|
132
|
-
// favors the speaker while the eyes hold contact, and a concentration
|
|
133
|
-
// squint with knit brows. Stillness does the rest — straining people
|
|
134
|
-
// freeze — so holds are frequent and there is no engagement lean: you don't
|
|
135
|
-
// nod along to what you can't hear. Server sends it on soft/low-SNR user
|
|
136
|
-
// audio, typically followed by SORRY or a "could you repeat" utterance.
|
|
137
|
-
CANT_HEAR: {
|
|
138
|
-
gaze: 'USER_EAR', emotion: 'neutral', engagement: false,
|
|
139
|
-
idle: { sway: 0.5, blinkGap: [4.5, 6.5],
|
|
140
|
-
hold: { every: [2.5, 5.5], dur: [1.0, 1.8] } },
|
|
141
|
-
// A minimal line face swallows small deltas — the ink moves whole units
|
|
142
|
-
// or it doesn't move. These values are set from the contact sheet's
|
|
143
|
-
// extremes row, not from what a fleshed rig would need: brows DOWN
|
|
144
|
-
// (corrugator effort, not the browInner worry-lift), a real squint, and
|
|
145
|
-
// the resting smile pressed flat — nobody smiles while straining to hear.
|
|
146
|
-
// The squint is the lower lid's (AU7), with no upper lid dropped on top:
|
|
147
|
-
// the two together narrowed a photographic eye from above and read as
|
|
148
|
-
// drowsy, and a line face's squint is its lower lid already.
|
|
149
|
-
pose: {
|
|
150
|
-
torsoLean: 0.70, headPitch: 0.10,
|
|
151
|
-
squintL: 0.75, squintR: 0.75,
|
|
152
|
-
browRaiseL: -0.45, browRaiseR: -0.45, browInnerL: 0.15, browInnerR: 0.12,
|
|
153
|
-
mouthPress: 0.45, mouthCornerL: -0.22, mouthCornerR: -0.22,
|
|
154
|
-
},
|
|
155
|
-
},
|
|
156
|
-
// The user's microphone is closed, and the agent is the one that closed it.
|
|
157
|
-
// Authored against CANT_HEAR as its exact inverse, because that contrast is
|
|
158
|
-
// the whole read: straining leans *in* to get more of the user, this settles
|
|
159
|
-
// *back* — nothing more is coming through and the avatar knows why. The eyes
|
|
160
|
-
// stay on the user, because a deliberate hold is not inattention; what
|
|
161
|
-
// carries it is the mouth, pressed shut and staying shut, and the slow,
|
|
162
|
-
// unhurried blink of waiting rather than working. No engagement lean: you
|
|
163
|
-
// cannot nod along to a channel you have muted yourself. No filter, ever —
|
|
164
|
-
// DEGRADED and OFFLINE own "something is broken", and this is a decision.
|
|
165
|
-
MUTED: {
|
|
166
|
-
gaze: 'USER', emotion: 'neutral', engagement: false,
|
|
167
|
-
idle: { sway: 0.45, blinkGap: [4.5, 7.0],
|
|
168
|
-
hold: { every: [3.0, 6.0], dur: [0.9, 1.6] } },
|
|
169
|
-
// Line-face scaled (see CANT_HEAR): peep's resting mouth is drawn smiling,
|
|
170
|
-
// so a closed mouth has to be authored clearly past flat to read as closed
|
|
171
|
-
// at all. Corners were -0.20 first, which rendered as a straight line and
|
|
172
|
-
// at 130 px against LISTENING said nothing — the delta a viewer gets is
|
|
173
|
-
// curvature, and flat is the halfway point of it, not the end. -0.34 is as
|
|
174
|
-
// far as it goes before the hold starts reading as sulking. No squint and
|
|
175
|
-
// no lid drop: one is straining, the other was asleep. browInner carries
|
|
176
|
-
// the "one moment" without the worry lift.
|
|
177
|
-
pose: {
|
|
178
|
-
torsoLean: -0.28, headPitch: 0.04,
|
|
179
|
-
browRaiseL: 0.08, browRaiseR: 0.06, browInnerL: 0.28, browInnerR: 0.22,
|
|
180
|
-
mouthPress: 0.78, mouthCornerL: -0.34, mouthCornerR: -0.34,
|
|
181
|
-
},
|
|
182
|
-
},
|
|
183
|
-
// --- application state ---------------------------------------------------
|
|
184
|
-
// "Momentarily busy on the thing you asked for." No hands in frame, so the
|
|
185
|
-
// whole read comes from cheap cues (docs/research-biomechanics.md §6.4,
|
|
186
|
-
// recommendation 19): gaze down on one stable target with a reading scan,
|
|
187
|
-
// blinks suppressed to task-focus rate (~9/min), shoulders slightly raised
|
|
188
|
-
// and *held* with brief micro-freezes, and a glance back up to the user.
|
|
189
|
-
// The glance is the tell that the user has not been forgotten; without it,
|
|
190
|
-
// busy is just absent, and much more often than that it is fidgeting. There
|
|
191
|
-
// is no shoulder rhythm: at 2.2 Hz it ran over the 1.5 Hz ceiling every
|
|
192
|
-
// other motion here keeps, and held is what §6.4 describes. The trunk sits
|
|
193
|
-
// out the glance — checking on someone is a look, not a turn toward them.
|
|
194
|
-
WORKING: {
|
|
195
|
-
// OWN_SCREEN: eyes down at the agent's own display, head nearly level.
|
|
196
|
-
// This was SCREEN_WORK, which turned the head down and to the left and
|
|
197
|
-
// held it there for the whole task, with a random drift too small to see
|
|
198
|
-
// on it — a head turned away from the user, staring at a point. `read` is
|
|
199
|
-
// what makes it busy instead: fixations stepping along a line, a sweep
|
|
200
|
-
// back, the next line, a pause on the result (see readStep). The lids
|
|
201
|
-
// take back most of the down look's follow so the eyes stay awake.
|
|
202
|
-
gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
|
|
203
|
-
scan: [0.9, 2.0, 0.12],
|
|
204
|
-
// §6.4's ~9/min is a count of blinks: at the table's own 6-7.5 s gap it
|
|
205
|
-
// measured 12-13/min, which is not focus.
|
|
206
|
-
idle: { sway: 0.5, blinkGap: [6.8, 8.6], breathRate: 1.05, breathAmp: 0.8,
|
|
207
|
-
hold: { every: [5.0, 9.0], dur: [0.6, 1.1] } },
|
|
208
|
-
// The look up to check is brows-first and blinkless, the lids leading it
|
|
209
|
-
// (their 18 ms against the eye's 32 ms); the blink comes on the way back
|
|
210
|
-
// down, where the eyes re-engage with the work. A blink going up hid the
|
|
211
|
-
// one moment the user is actually looked at.
|
|
212
|
-
glance: { to: 'USER', every: [6, 11], hold: [0.8, 1.2], brow: 0.10,
|
|
213
|
-
blinkTo: false, blinkBack: true },
|
|
214
|
-
// Fewer, longer fixations than a page reader's. At 3-5 steps a quarter
|
|
215
|
-
// second apart the scan measured a shift a second, and over a webcam that
|
|
216
|
-
// is a jiggle, not reading.
|
|
217
|
-
read: { width: 0.20, steps: [2, 3], fix: [0.35, 0.7], lines: [2, 4], line: 0.05,
|
|
218
|
-
pause: [0.9, 1.8] },
|
|
219
|
-
// The brows draw down and together — AU4, the one facial action that
|
|
220
|
-
// reliably marks effortful concentration. Without it the face over a
|
|
221
|
-
// reading scan is blank, and blank over a moving eye reads as bored.
|
|
222
|
-
pose: { headPitch: 0.04, lidL: -0.08, lidR: -0.08,
|
|
223
|
-
shoulderL: 0.06, shoulderR: 0.06,
|
|
224
|
-
browRaiseL: -0.08, browRaiseR: -0.08, browInnerL: -0.10, browInnerR: -0.10 },
|
|
225
|
-
},
|
|
226
|
-
// The audio channel is broken and the agent is typing in the chat window to
|
|
227
|
-
// communicate. The glance looks up and HOLDS 1.2–2 s, expectant, because the
|
|
228
|
-
// chat (and the user's face) is now the only channel there is. A touch of
|
|
229
|
-
// browInner carries the apology. Relation to DEGRADED is by semantics, not merger:
|
|
230
|
-
// DEGRADED says "my feed is broken", TYPING_CHAT says "I'm working around
|
|
231
|
-
// it" — a server will typically sequence DEGRADED → TYPING_CHAT.
|
|
232
|
-
TYPING_CHAT: {
|
|
233
|
-
gaze: 'SCREEN_WORK', emotion: 'neutral', engagement: false,
|
|
234
|
-
idle: { sway: 0.6, blinkGap: [5.5, 7.0], breathRate: 1.05,
|
|
235
|
-
rhythm: { amp: 0.055, freq: 2.5 } },
|
|
236
|
-
glance: { to: 'USER', every: [3.2, 5.5], hold: [1.2, 2.0] },
|
|
237
|
-
// Line-face scaled (see CANT_HEAR); the apology has to survive the rig's
|
|
238
|
-
// baked resting smile, so the corners go clearly negative.
|
|
239
|
-
pose: { headPitch: 0.10, lidL: -0.04, lidR: -0.04,
|
|
240
|
-
shoulderL: 0.06, shoulderR: 0.06,
|
|
241
|
-
browInnerL: 0.45, browInnerR: 0.38,
|
|
242
|
-
mouthPress: 0.50, mouthCornerL: -0.28, mouthCornerR: -0.28 },
|
|
243
|
-
},
|
|
244
|
-
// Attention genuinely elsewhere. What separates this from WORKING is target
|
|
245
|
-
// *stability* (§6.4): busy is one steady off-user target, distracted is
|
|
246
|
-
// wandering ones, held long (aversion >3s), with no engagement lean — the
|
|
247
|
-
// missing nod is as diagnostic as the look-away. Sway is looser than
|
|
248
|
-
// LISTENING because attention is what was holding the body still. The
|
|
249
|
-
// widget only looks away; deciding when to snap back is the server's call.
|
|
250
|
-
DISTRACTED: {
|
|
251
|
-
gaze: 'AWAY_RIGHT', emotion: 'neutral', engagement: false,
|
|
252
|
-
idle: { sway: 1.15, blinkGap: [1.8, 4.2] },
|
|
253
|
-
// Sideways and up, never steep-down: lateral is where real intimacy/
|
|
254
|
-
// distraction aversions live, and a steep down target seals this rig's
|
|
255
|
-
// eyes (see WORKING).
|
|
256
|
-
wander: { targets: ['AWAY_RIGHT', 'AWAY_THINKING', 'SCREEN_LEFT', 'SCREEN_TOP'],
|
|
257
|
-
every: [2.8, 6.8] },
|
|
258
|
-
},
|
|
259
|
-
// The buying-time move: hunting for a control on screen. Distinct from
|
|
260
|
-
// REVIEWING_SCREEN by *hunt* quality — reading dwells (1.8–5 s) become
|
|
261
|
-
// search saccades (0.8–2 s) with revisits (targets repeat in the wander
|
|
262
|
-
// set), plus the idle layer's flick: the tiny "no, not this one" yaw
|
|
263
|
-
// wiggle nobody makes while merely reading. Server semantics: a filler
|
|
264
|
-
// while an async activity completes; the server exits it when done.
|
|
265
|
-
SEARCHING_SCREEN: {
|
|
266
|
-
gaze: 'SCREEN_CENTER', emotion: 'neutral', engagement: false,
|
|
267
|
-
idle: { sway: 0.65, blinkGap: [5.0, 6.8], breathRate: 1.05,
|
|
268
|
-
flick: { amp: 0.30, every: [3.5, 7.0] } },
|
|
269
|
-
// The only screen state that never named its own scan, so it ran the
|
|
270
|
-
// default [0.5, 1.6] at full amplitude: a second full-size jump landing on
|
|
271
|
-
// top of a wander hop, which is most of what reads as snapping. A hunt's
|
|
272
|
-
// refixations are faster and much smaller than a reader's — quick little
|
|
273
|
-
// checks around the thing being looked at — so this is the other end of
|
|
274
|
-
// the dial from WORKING's [0.9, 2.0, 0.12] rather than a copy of it.
|
|
275
|
-
scan: [0.4, 1.1, 0.55],
|
|
276
|
-
wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_TOP', 'SCREEN_WORK',
|
|
277
|
-
'SCREEN_RIGHT', 'SCREEN_CENTER', 'SCREEN_LEFT'],
|
|
278
|
-
every: [0.8, 2.0] },
|
|
279
|
-
// Line-face scaled (see CANT_HEAR). Note the corners: peep's REST mouth
|
|
280
|
-
// is drawn smiling, so "not smiling" is a clearly negative net corner,
|
|
281
|
-
// not zero.
|
|
282
|
-
pose: { squintL: 0.40, squintR: 0.40, mouthPress: 0.65,
|
|
283
|
-
mouthCornerL: -0.25, mouthCornerR: -0.25, browRaiseL: -0.26, browRaiseR: -0.20 },
|
|
284
|
-
},
|
|
285
|
-
// --- floor management ----------------------------------------------------
|
|
286
|
-
// Turn-taking is the part of a voice call that goes wrong most often: the
|
|
287
|
-
// user either talks over the agent or sits in silence waiting for a signal
|
|
288
|
-
// that never comes. These are states rather than clips because the floor is a
|
|
289
|
-
// condition and not an event — WANTS_IN in particular has to hold for as long
|
|
290
|
-
// as it takes the other person to notice it.
|
|
291
|
-
//
|
|
292
|
-
// Each of these lifts the shoulders and parts the lips, because that is what an
|
|
293
|
-
// inbreath looks like from outside, and an inbreath is the cue humans actually
|
|
294
|
-
// use to predict that someone is about to speak. The head comes *up* rather
|
|
295
|
-
// than down: a lowered head is deferential and reads as yielding.
|
|
296
|
-
TAKING_FLOOR: {
|
|
297
|
-
gaze: 'USER', emotion: 'neutral', idle: { sway: 0.6 }, engagement: false,
|
|
298
|
-
pose: {
|
|
299
|
-
browRaiseL: 0.26, browRaiseR: 0.22, lidL: -0.10, lidR: -0.10,
|
|
300
|
-
headPitch: -0.10, torsoLean: 0.22, shoulderL: 0.30, shoulderR: 0.30,
|
|
301
|
-
mouthOpen: 0.10, mouthPress: -0.10,
|
|
302
|
-
},
|
|
303
|
-
},
|
|
304
|
-
// The one signal the rig had no way to give at all. An agent needs to be
|
|
305
|
-
// able to say "I'd like to come in" without talking over the user, and
|
|
306
|
-
// every part of this pose is doing that job: held still (idle is low on
|
|
307
|
-
// purpose — stillness is what makes it read as intent rather than as fidget),
|
|
308
|
-
// leaning in, lips apart and staying apart.
|
|
309
|
-
WANTS_IN: {
|
|
310
|
-
gaze: 'USER', emotion: 'neutral', idle: { sway: 0.45 }, engagement: false,
|
|
311
|
-
pose: {
|
|
312
|
-
browRaiseL: 0.42, browRaiseR: 0.38, lidL: -0.14, lidR: -0.14,
|
|
313
|
-
headPitch: -0.14, torsoLean: 0.42, shoulderL: 0.45, shoulderR: 0.45,
|
|
314
|
-
mouthOpen: 0.16, mouthWidth: 0.30, mouthPress: -0.14,
|
|
315
|
-
},
|
|
316
|
-
},
|
|
317
|
-
// Interrupted mid-word. The mouth shutting is the whole message, and it has to
|
|
318
|
-
// happen faster than anything else on the face — see YIELD_FLOOR, which is
|
|
319
|
-
// what actually delivers the snap.
|
|
320
|
-
YIELDED: {
|
|
321
|
-
gaze: 'USER', emotion: 'neutral', idle: { sway: 0.9 }, engagement: false,
|
|
322
|
-
pose: {
|
|
323
|
-
browRaiseL: 0.10, browRaiseR: 0.08,
|
|
324
|
-
torsoLean: -0.18, shoulderL: -0.12, shoulderR: -0.12,
|
|
325
|
-
},
|
|
326
|
-
},
|
|
327
|
-
|
|
328
|
-
// --- connection ----------------------------------------------------------
|
|
329
|
-
// Neither of these may close the eyes. A lid at 0.3 reads sleepy and one at
|
|
330
|
-
// 0.95 reads asleep (research-perception.md §6), and on a photographic face
|
|
331
|
-
// a disconnected call rendered as someone falling asleep on camera. The SVG
|
|
332
|
-
// faces also desaturate (`filter`); a rig that does not is carried by pose
|
|
333
|
-
// and gaze alone, so those have to say it.
|
|
334
|
-
//
|
|
335
|
-
// Our side of the call is struggling: still with the user, a little worried
|
|
336
|
-
// about it — inner brows up, mouth pressed — and quieter than listening.
|
|
337
|
-
DEGRADED: { gaze: 'USER', emotion: 'neutral', engagement: false,
|
|
338
|
-
aversion: 'LISTEN',
|
|
339
|
-
idle: { sway: 0.4, blinkGap: [3.4, 5.0] },
|
|
340
|
-
pose: { browInnerL: 0.30, browInnerR: 0.26, browRaiseL: -0.04, browRaiseR: -0.04,
|
|
341
|
-
mouthPress: 0.20, mouthCornerL: -0.10, mouthCornerR: -0.10 },
|
|
342
|
-
filter: 'grayscale(.55) brightness(.82)' },
|
|
343
|
-
// The call is gone. What a person does when their call drops: eyes on their
|
|
344
|
-
// own screen, waiting, and now and then a look back up to see whether it
|
|
345
|
-
// has come back. Still, with long holds — not in conversation — but awake.
|
|
346
|
-
OFFLINE: { gaze: 'OWN_SCREEN', emotion: 'neutral', engagement: false,
|
|
347
|
-
scan: [1.2, 2.6],
|
|
348
|
-
idle: { sway: 0.3, blinkGap: [4.5, 7.0],
|
|
349
|
-
hold: { every: [4.0, 8.0], dur: [1.0, 2.0] } },
|
|
350
|
-
glance: { to: 'USER', every: [7, 12], hold: [0.8, 1.3] },
|
|
351
|
-
pose: { lidL: -0.08, lidR: -0.08, mouthPress: 0.25 },
|
|
352
|
-
filter: 'grayscale(1) brightness(.6)' },
|
|
353
|
-
};
|
|
354
|
-
|
|
355
|
-
export const STATE_NAMES = Object.keys(STATES);
|
|
356
|
-
|
|
357
|
-
/**
|
|
358
|
-
* The shoulder line's share of a held tilt, per unit of `headRoll`.
|
|
359
|
-
*
|
|
360
|
-
* Roll is the one head axis with nothing under it: a turn recruits the trunk
|
|
361
|
-
* and a nod bends the neck, but a tilt on a 2.5-D head is a rotation about a
|
|
362
|
-
* point near the chin and every other pixel stays exactly where it was. That is
|
|
363
|
-
* the read reported as "a hinge". Anatomically a tilt is lower-cervical
|
|
364
|
-
* (docs/research-head-rotation.md §2.1) — spread down the neck, ending at the
|
|
365
|
-
* girdle — so the shoulder line tips a little with it, and that tip is what says
|
|
366
|
-
* the neck bent rather than the head swinging off a pin.
|
|
367
|
-
*
|
|
368
|
-
* Sized to stay an accompaniment: at the roll clamp the line tips about a
|
|
369
|
-
* quarter of the trunk's own share of a turn, which lifts a shoulder ~0.11 of
|
|
370
|
-
* its shrug — inside the band a weight shift already occupies. A third, the
|
|
371
|
-
* figure Live2D gives its body angles, put 6 px of shoulder on an 8 degree tilt
|
|
372
|
-
* and read as a shrug arriving with the head.
|
|
373
|
-
*
|
|
374
|
-
* Exported for the rig instruments.
|
|
375
|
-
*/
|
|
376
|
-
export const SHOULDER_TILT = 0.08;
|
|
377
|
-
|
|
378
|
-
export function createAvatar(opts = {}) {
|
|
379
|
-
const mount = typeof opts.mount === 'string' ? document.querySelector(opts.mount) : opts.mount;
|
|
380
|
-
if (!mount) throw new Error('createAvatar: mount element required');
|
|
381
|
-
|
|
382
|
-
// `opts.face` is a Face record — `{ create, meta }`. Passed in rather than
|
|
383
|
-
// named: a name needs a table, and a table imports every drawing to answer
|
|
384
|
-
// one lookup (`src/faces.js` has that table, for tooling that wants them).
|
|
385
|
-
const entry = opts.rig ? null : opts.face;
|
|
386
|
-
if (!opts.rig && !entry) {
|
|
387
|
-
throw new Error('createAvatar: a `face` (see src/faces.js) or a `rig` is required');
|
|
388
|
-
}
|
|
389
|
-
const face = entry ? entry.create(mount, opts.theme) : null;
|
|
390
|
-
// A renderer-neutral rig needs no SVG descriptor: `meta` is what SVG hosts
|
|
391
|
-
// and tools frame the drawing with, never a requirement of the rig contract.
|
|
392
|
-
// A face carries its own — a `{ create }` with no META used to be tolerated
|
|
393
|
-
// here and the viewBox re-read off the produced svg, which meant a face could
|
|
394
|
-
// ship half a descriptor and nothing would say so.
|
|
395
|
-
const meta = face ? entry.meta : null;
|
|
396
|
-
const gaze = new GazeLayer();
|
|
397
|
-
const idle = new IdleLayer();
|
|
398
|
-
const speech = new VisemeTrack();
|
|
399
|
-
const prosody = new SpeechProsody({ brows: opts.brows });
|
|
400
|
-
|
|
401
|
-
let gazeOverrideByClip = null;
|
|
402
|
-
const clip = new ClipPlayer({
|
|
403
|
-
onGaze: (g) => { gazeOverrideByClip = g; applyGaze(); },
|
|
404
|
-
onBlink: () => idle.blink(),
|
|
405
|
-
});
|
|
406
|
-
const engagement = new ListeningEngine();
|
|
407
|
-
const performTrack = new PerformTrack();
|
|
408
|
-
// The current SVG hand is a renderer adapter for the first-class `frame.hand`
|
|
409
|
-
// control. `hand: false` only disables its SVG rendering; gesture actions
|
|
410
|
-
// still emit the semantic hand frame for a supplied custom rig.
|
|
411
|
-
const hand = face && opts.hand !== false ? createHand(face.svg, face.theme, meta, { dir: opts.handSide }) : null;
|
|
412
|
-
const rig = opts.rig ? opts.rig(mount, opts.rigOptions) : createSvgRig(face, hand);
|
|
413
|
-
|
|
414
|
-
gaze.onLargeShift = (forced) => idle.blink(true, forced);
|
|
415
|
-
|
|
416
|
-
const listeners = {
|
|
417
|
-
state: [], speakEnd: [], clipEnd: [], performEnd: [], gestureEnd: [],
|
|
418
|
-
};
|
|
419
|
-
const emit = (ev, ...a) => listeners[ev] && listeners[ev].forEach((f) => f(...a));
|
|
420
|
-
clip.onEnd = (c) => { if (c) emit('clipEnd', c.id); };
|
|
421
|
-
speech.onEnd = () => { prosody.closeTurn(); emit('speakEnd'); };
|
|
422
|
-
performTrack.onEnd = () => { emit('performEnd'); };
|
|
423
|
-
|
|
424
|
-
// --- live state -----------------------------------------------------------
|
|
425
|
-
let stateName = 'IDLE';
|
|
426
|
-
let emotion = 'neutral';
|
|
427
|
-
let emotionAmt = 1;
|
|
428
|
-
let gazeName = 'USER';
|
|
429
|
-
let gazeCustom = null;
|
|
430
|
-
let overrides = null; // demo/debug direct param injection
|
|
431
|
-
// Articulation gain. VISEME_SHAPES is tuned for a face at conversational
|
|
432
|
-
// size; sharing the screen with live video the same shapes read as
|
|
433
|
-
// under-articulated. Scales every viseme away from rest, so shape identities
|
|
434
|
-
// and their relative sizes survive and only the excursion changes. Above
|
|
435
|
-
// ~1.5 the open vowels saturate against the clamp — the intended ceiling.
|
|
436
|
-
let mouthGain = opts.mouthGain ?? 1;
|
|
437
|
-
let handSide = opts.handSide === -1 ? 'left' : 'right';
|
|
438
|
-
/**
|
|
439
|
-
* This avatar's own addressable motions, on top of the core intents.
|
|
440
|
-
*
|
|
441
|
-
* An avatar is a drawing with a body, and some bodies can do things the wire
|
|
442
|
-
* has no portable word for. The nod types the listening research separates —
|
|
443
|
-
* a continuer, an assessment, a realisation — are one
|
|
444
|
-
* `ACKNOWLEDGE` to a server, because that is all a server can ask of every
|
|
445
|
-
* face; the *shapes* are sized in a rig's own units and belong to the rig
|
|
446
|
-
* (`packages/avatar/client/three/sequences.ts` is the first table of them).
|
|
447
|
-
*
|
|
448
|
-
* Deliberately not a registry and not a loader: it is a plain object the
|
|
449
|
-
* avatar module passes in, and it can only add. `ACTIONS` wins on a name
|
|
450
|
-
* collision, so no asset can quietly change what a core action means.
|
|
451
|
-
*/
|
|
452
|
-
const sequences = opts.sequences || {};
|
|
453
|
-
/**
|
|
454
|
-
* This avatar's own *rendering* of an id this renderer already has — same id,
|
|
455
|
-
* same intent, a shape sized for its body.
|
|
456
|
-
*
|
|
457
|
-
* These are an intent vocabulary, and `ACK_NOD` is one implementation of
|
|
458
|
-
* "the avatar nodded", not a promise about anatomy (interjections.js). The
|
|
459
|
-
* shared clips are authored in pose units that mean pixels on a line face;
|
|
460
|
-
* on a rig whose unit is a degree the same keys can land outside what the
|
|
461
|
-
* gesture *means* — the shared nod renders 15° on tara, where a continuer is
|
|
462
|
-
* 3-5° — and scaling it would not fix it, because a continuer is one stroke
|
|
463
|
-
* where an agreement is two. So a renderer may re-author the shape.
|
|
464
|
-
*
|
|
465
|
-
* What it may not do is change the vocabulary: every id here must already be
|
|
466
|
-
* one this renderer publishes, so no avatar can add a word through this door
|
|
467
|
-
* or make one mean something else. `sequences` is the door for adding.
|
|
468
|
-
*/
|
|
469
|
-
const actionShapes = opts.actions || {};
|
|
470
|
-
for (const id of Object.keys(actionShapes)) {
|
|
471
|
-
if (!ACTIONS[id]) throw new Error(`actions: ${id} is not one of this renderer's own`);
|
|
472
|
-
}
|
|
473
|
-
/**
|
|
474
|
-
* This avatar's own rendering of a state — the same door as `actions`, for
|
|
475
|
-
* the held face instead of the gesture. A rig replaces a state's fields
|
|
476
|
-
* whole (`pose`, say), and only for a state the table already has: it can
|
|
477
|
-
* re-render the vocabulary, never extend it.
|
|
478
|
-
*/
|
|
479
|
-
const states = { ...STATES };
|
|
480
|
-
for (const [id, own] of Object.entries(opts.states || {})) {
|
|
481
|
-
if (!STATES[id]) throw new Error(`states: ${id} is not a state`);
|
|
482
|
-
states[id] = { ...STATES[id], ...own };
|
|
483
|
-
}
|
|
484
|
-
let handAction = null;
|
|
485
|
-
const handQueue = [];
|
|
486
|
-
// Gesture gain, same idea for the clip layer: small gestures under-render
|
|
487
|
-
// through the head's τ — see internal-mixer.md § Smoothing.
|
|
488
|
-
let gestureGain = opts.gestureGain ?? 1;
|
|
489
|
-
// Body-liveness gain. Constraint 8 (this widget shares the screen with a
|
|
490
|
-
// live video call) argues for the smallest idle motion that still reads, and
|
|
491
|
-
// the amplitudes in idle.js are set where they read. A host that is actually
|
|
492
|
-
// re-encoding the avatar — compositing it into an outgoing stream rather
|
|
493
|
-
// than rendering it locally as SVG, where the motion costs nothing — turns
|
|
494
|
-
// this down instead of the default being a body that does not move.
|
|
495
|
-
idle.gain = opts.motionGain ?? 1;
|
|
496
|
-
// Speech-rhythm gains. A pose unit is a different angle on every rig — peep's
|
|
497
|
-
// head travels 17 px per unit of pitch, a mesh head turns a few degrees — so
|
|
498
|
-
// the rig's own module sets these once rather than the library guessing.
|
|
499
|
-
// Defaults to the uncalibrated scale; `prosody.js` has what that stands for.
|
|
500
|
-
const prosodyHeadGain = opts.prosodyHeadGain ?? UNCALIBRATED_HEAD_GAIN;
|
|
501
|
-
const prosodyFaceGain = opts.prosodyFaceGain ?? 1;
|
|
502
|
-
// Fixational-saccade gain, the same idea for the eyes: how far a scan
|
|
503
|
-
// step, a drift or a line of reading travels. Sized for a line face's
|
|
504
|
-
// pupils at 1; a photographic eye needs several times that to move at all.
|
|
505
|
-
// Aversions get their own gain: a look-away must read as one from across
|
|
506
|
-
// the call, where a fixation step must not, so one number cannot size both.
|
|
507
|
-
const saccadeGain = opts.saccadeGain ?? 1;
|
|
508
|
-
// How far this face may hold its head off centre, per axis (internal-mixer.md
|
|
509
|
-
// § The held-head budget): the rig supplies it; an axis left out is unbudgeted.
|
|
510
|
-
const headHold = opts.headHold || {};
|
|
511
|
-
// Per-axis gain on the head's *continuous* drive — speech phrasing and idle,
|
|
512
|
-
// both sized in pose units for a line drawing. Measuring a 2.5-D face found
|
|
513
|
-
// the drive spending a third of the pitch a speaking human uses and nearly
|
|
514
|
-
// twice the yaw. In front of 6b deliberately: a gain that skipped the budget
|
|
515
|
-
// would be measuring the face's failure rather than the layer's range.
|
|
516
|
-
const headGain = typeof opts.headGain === 'number'
|
|
517
|
-
? { headYaw: opts.headGain, headPitch: opts.headGain, headRoll: opts.headGain }
|
|
518
|
-
: { headYaw: 1, headPitch: 1, headRoll: 1, ...(opts.headGain || {}) };
|
|
519
|
-
gaze.scanGain = saccadeGain;
|
|
520
|
-
gaze.avertGain = opts.aversionGain ?? 1;
|
|
521
|
-
// A rig that says what its pose units are in degrees gets the eye-head
|
|
522
|
-
// system sized for it (gaze.js): its own look targets, how an aversion
|
|
523
|
-
// splits between eyes and head, lids that follow the eye both ways, and the
|
|
524
|
-
// reflex in step 8b.
|
|
525
|
-
const ocu = opts.oculomotor || {};
|
|
526
|
-
if (ocu.targets) gaze.targets = { ...GAZE_TARGETS, ...ocu.targets };
|
|
527
|
-
if (ocu.avert) gaze.avertSplit = ocu.avert;
|
|
528
|
-
if (ocu.angles) gaze.angles = ocu.angles;
|
|
529
|
-
if (ocu.lidFollow) gaze.lidFollow = ocu.lidFollow;
|
|
530
|
-
if (ocu.head) { gaze.headAccel = ocu.head.accel; gaze.headSpeed = ocu.head.speed; }
|
|
531
|
-
// Pupil units of counter-rotation per head unit: the reflex's gain times
|
|
532
|
-
// the ratio of what one unit of each is in degrees. The gain may differ by
|
|
533
|
-
// axis ({x, y}): a face whose pitch reads weaker than its yaw wants less
|
|
534
|
-
// of the eyes' answer to it.
|
|
535
|
-
const vorGain = typeof ocu.vor === 'object' ? ocu.vor : { x: ocu.vor, y: ocu.vor };
|
|
536
|
-
const vor = ocu.vor && ocu.angles
|
|
537
|
-
? { x: vorGain.x * ocu.angles.head.x / ocu.angles.eye.x,
|
|
538
|
-
y: vorGain.y * ocu.angles.head.y / ocu.angles.eye.y }
|
|
539
|
-
: null;
|
|
540
|
-
// How far the reflex may carry the eye in its socket, in pupil units:
|
|
541
|
-
// Guitton & Volle's effective oculomotor range; the rig supplies the reach.
|
|
542
|
-
const reach = ocu.range || { x: 1, up: 1, down: 1 };
|
|
543
|
-
const reflexX = (px) => clamp(px + vor.x * (aim.x - cur.headYaw), -reach.x, reach.x);
|
|
544
|
-
const reflexY = (py) => clamp(py + vor.y * (aim.y - cur.headPitch), -reach.up, reach.down);
|
|
545
|
-
// The head the eyes' target was authored against, smoothed at the eye's
|
|
546
|
-
// own tau so a shift's compensation moves with the saccade, not ahead of it.
|
|
547
|
-
const aim = { x: 0, y: 0 };
|
|
548
|
-
let wanderAt = 0;
|
|
549
|
-
let driftAt = 0;
|
|
550
|
-
let trunkYaw = 0;
|
|
551
|
-
let glanceAt = 0;
|
|
552
|
-
let glanceUntil = 0;
|
|
553
|
-
let lastBack = null;
|
|
554
|
-
// THINKING's second look inside a look away: when it moves, where to, and
|
|
555
|
-
// how long the brows stay up for it.
|
|
556
|
-
let dartAt = 0, dartBrowUntil = 0;
|
|
557
|
-
const dart = { x: 0, y: 0 };
|
|
558
|
-
// The state whose gaze is showing, and when the current state takes it
|
|
559
|
-
// over if that is still pending (GAP_SETTLE). Usually the same state.
|
|
560
|
-
let gazeState = 'IDLE';
|
|
561
|
-
let settleAt = 0;
|
|
562
|
-
// Reading scan: position in the line and the block, see readStep.
|
|
563
|
-
let readCol = 0, readCols = 0, readRow = 0, readRows = 0;
|
|
564
|
-
let attendUntil = 0;
|
|
565
|
-
let speakClock = null;
|
|
566
|
-
let speakStart = 0;
|
|
567
|
-
|
|
568
|
-
const cur = Object.assign({}, REST);
|
|
569
|
-
const target = Object.assign({}, REST);
|
|
570
|
-
// The head axes again, carrying only what is held (step 6b).
|
|
571
|
-
const hold = { headYaw: 0, headPitch: 0, headRoll: 0 };
|
|
572
|
-
// What the rig is handed: `cur` with the reflex applied to the eyes and the
|
|
573
|
-
// lips stopped at shut.
|
|
574
|
-
const shown = Object.assign({}, REST);
|
|
575
|
-
|
|
576
|
-
function applyGaze(blink) {
|
|
577
|
-
const g = gazeOverrideByClip || gazeName;
|
|
578
|
-
gaze.set(g, gazeOverrideByClip ? null : gazeCustom, blink);
|
|
579
|
-
}
|
|
580
|
-
|
|
581
|
-
// --- the frame ------------------------------------------------------------
|
|
582
|
-
let raf = 0;
|
|
583
|
-
let last = 0;
|
|
584
|
-
let elapsed = 0;
|
|
585
|
-
// `manual` withholds the rAF loop so a tool can drive frames itself.
|
|
586
|
-
const manual = !!opts.manual;
|
|
587
|
-
|
|
588
|
-
function frame(now) {
|
|
589
|
-
raf = requestAnimationFrame(frame);
|
|
590
|
-
if (!last) last = now;
|
|
591
|
-
// Cap dt so a backgrounded tab doesn't fast-forward the whole rig on return.
|
|
592
|
-
const dt = Math.min(0.05, (now - last) / 1000);
|
|
593
|
-
last = now;
|
|
594
|
-
elapsed += dt;
|
|
595
|
-
step(dt, dt * 1000);
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
function step(dt, dtMs) {
|
|
599
|
-
// 0. the performance timeline. Sampled before the pose is built so a verb
|
|
600
|
-
// firing this frame shapes this frame.
|
|
601
|
-
performTrack.update();
|
|
602
|
-
|
|
603
|
-
if (settleAt && elapsed >= settleAt) { settleAt = 0; enterGaze(stateName, {}, true); }
|
|
604
|
-
const st = states[stateName] || states.IDLE;
|
|
605
|
-
// What the eyes are doing belongs to the state that has the gaze.
|
|
606
|
-
const gst = states[gazeState] || st;
|
|
607
|
-
|
|
608
|
-
// 1. base pose: rest + emotion + state-specific overlay
|
|
609
|
-
for (const c of CHANNELS) target[c] = REST[c];
|
|
610
|
-
const ep = emotionPose(emotion, emotionAmt);
|
|
611
|
-
for (const k in ep) target[k] = REST[k] + ep[k];
|
|
612
|
-
if (st.pose) for (const k in st.pose) target[k] = (target[k] || 0) + st.pose[k];
|
|
613
|
-
// The head the state and emotion hold is an attitude, and the eyes are
|
|
614
|
-
// authored inside it: the reflex (8b) keeps them on their target against
|
|
615
|
-
// everything that moves the head except this.
|
|
616
|
-
const poseYaw = target.headYaw, posePitch = target.headPitch;
|
|
617
|
-
// What the head *holds*, accumulated alongside the pose as each layer that
|
|
618
|
-
// holds one adds to it: the attitude above, the gaze, a phrase's pose and
|
|
619
|
-
// the idle posture. Strokes, beats and clip deltas stay out — see 6b.
|
|
620
|
-
for (const c of HEAD_AXES) hold[c] = target[c];
|
|
621
|
-
|
|
622
|
-
// 2. gaze (absolute: pupils + partial head follow, plus the lid that rides
|
|
623
|
-
// with the eye — looking down without it bares sclera and reads as alarm)
|
|
624
|
-
const g = gaze.update(elapsed, dt);
|
|
625
|
-
for (const k in g) {
|
|
626
|
-
if (k === 'trunkYaw' || k === 'aimYaw' || k === 'aimPitch') continue;
|
|
627
|
-
target[k] = (k.startsWith('head') ? target[k] : 0) + g[k];
|
|
628
|
-
}
|
|
629
|
-
for (const c of HEAD_AXES) hold[c] += g[c];
|
|
630
|
-
// The lid follows the eye as drawn, reflex and all — last frame's reading
|
|
631
|
-
// of it, since the head it corrects for is not smoothed yet.
|
|
632
|
-
const lid = gaze.lidBias(vor ? reflexY(g.pupilY) : g.pupilY);
|
|
633
|
-
target.lidL += lid;
|
|
634
|
-
target.lidR += lid;
|
|
635
|
-
if (vor) {
|
|
636
|
-
aim.x = approach(aim.x, poseYaw + g.aimYaw, TAU.pupilX, dt);
|
|
637
|
-
aim.y = approach(aim.y, posePitch + g.aimPitch, TAU.pupilY, dt);
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
// 2b. the trunk follows the head. Sampled HERE, after gaze and before the
|
|
641
|
-
// clip layer, on purpose: a sustained turn toward the screen recruits
|
|
642
|
-
// the trunk, and a nod or a head shake does not — a body that swings
|
|
643
|
-
// with every gesture reads as a mannequin on a turntable. The lag is
|
|
644
|
-
// not authored anywhere; torsoTurn chases the same target at a slower
|
|
645
|
-
// TAU than the head (params.js), so the trunk leaves late and settles
|
|
646
|
-
// late for free, at the share TRUNK_FOLLOW names. It follows where the
|
|
647
|
-
// head is going and not the looks riding on it, and it holds through a
|
|
648
|
-
// glance: checking on the user is a look, not a turn toward them.
|
|
649
|
-
if (!glanceUntil) trunkYaw = target.headYaw - g.headYaw + g.trunkYaw;
|
|
650
|
-
target.torsoTurn += trunkYaw * TRUNK_FOLLOW;
|
|
651
|
-
|
|
652
|
-
// 3. state-driven autonomous behaviour
|
|
653
|
-
if (gst.wander && elapsed > wanderAt) {
|
|
654
|
-
const w = gst.wander;
|
|
655
|
-
// Dwell first, and sometimes for two periods. A dwell of exactly one
|
|
656
|
-
// period every time is the metronome the eye reads as mechanical. The
|
|
657
|
-
// old code broke that up by accident: a draw landing on the target
|
|
658
|
-
// already held left the eyes still for another period. The freeze was a
|
|
659
|
-
// bug — the schedule counted a hop that moved nothing — but the dwell
|
|
660
|
-
// variety it produced was not, so it is kept deliberately, at the rate
|
|
661
|
-
// it used to happen: 1/n, the odds a uniform draw repeats the target.
|
|
662
|
-
const linger = Math.random() < 1 / w.targets.length ? 2 : 1;
|
|
663
|
-
wanderAt = elapsed + linger * (w.every[0] + Math.random() * (w.every[1] - w.every[0]));
|
|
664
|
-
// Then look somewhere the eyes are not. A wander set names a target more
|
|
665
|
-
// than once deliberately — SEARCHING_SCREEN's repeats are its revisits,
|
|
666
|
-
// the hunt coming back to the middle — but a revisit is arriving
|
|
667
|
-
// somewhere again, not never having left it. Same bounded redraw as
|
|
668
|
-
// pickBack, so a one-target set still terminates.
|
|
669
|
-
let g = pick(w.targets);
|
|
670
|
-
for (let i = 0; i < 4 && g === gazeName; i++) g = pick(w.targets);
|
|
671
|
-
setGaze(g);
|
|
672
|
-
}
|
|
673
|
-
// Periodic glance (WORKING's look-up-at-you beat, THINKING's check-in
|
|
674
|
-
// between two looks away). The return leg goes to the state's own gaze, or
|
|
675
|
-
// to a fresh pick from `back` — a thinker does not look away at the same
|
|
676
|
-
// spot every time. Either leg's gaze-evoked blink is the shift's odds
|
|
677
|
-
// unless the state says (`blinkTo`, `blinkBack`), because which leg blinks
|
|
678
|
-
// is part of what the glance means.
|
|
679
|
-
if (gst.glance) {
|
|
680
|
-
const gl = gst.glance;
|
|
681
|
-
if (glanceUntil && elapsed > glanceUntil) {
|
|
682
|
-
glanceUntil = 0;
|
|
683
|
-
const away = rand(gl.every);
|
|
684
|
-
glanceAt = elapsed + away;
|
|
685
|
-
setGaze(gl.back ? pickBack(gl.back, gl.stick) : gst.gaze, null, gl.blinkBack);
|
|
686
|
-
// At most one second look per look away, somewhere in its middle: a
|
|
687
|
-
// move at the start is the same look landing, and one at the end runs
|
|
688
|
-
// into the glance back.
|
|
689
|
-
dartAt = gl.dart && Math.random() < gl.dart.p ? elapsed + away * (0.35 + Math.random() * 0.3) : 0;
|
|
690
|
-
} else if (!glanceUntil && elapsed > glanceAt) {
|
|
691
|
-
glanceUntil = elapsed + rand(gl.hold);
|
|
692
|
-
setGaze(gl.to, null, gl.blinkTo);
|
|
693
|
-
dartAt = 0;
|
|
694
|
-
dart.x = dart.y = 0;
|
|
695
|
-
}
|
|
696
|
-
if (dartAt && elapsed > dartAt) {
|
|
697
|
-
// On from where the eyes are, not back toward the user or across to
|
|
698
|
-
// the other side: the thought moves, it does not turn round.
|
|
699
|
-
dartAt = 0;
|
|
700
|
-
const t = gaze.target;
|
|
701
|
-
const a = Math.atan2(t.py, t.px) + (Math.random() < 0.5 ? -1 : 1) * DART_TURN;
|
|
702
|
-
const m = gl.dart.mag * saccadeGain;
|
|
703
|
-
dart.x = Math.cos(a) * m;
|
|
704
|
-
dart.y = Math.sin(a) * m;
|
|
705
|
-
dartBrowUntil = elapsed + DART_BROW;
|
|
706
|
-
}
|
|
707
|
-
const brow = (glanceUntil ? gl.brow || 0 : 0)
|
|
708
|
-
+ (gl.dart && elapsed < dartBrowUntil ? gl.dart.brow : 0);
|
|
709
|
-
target.browRaiseL += brow;
|
|
710
|
-
target.browRaiseR += brow;
|
|
711
|
-
}
|
|
712
|
-
gaze.scanEvery = gst.scan || null;
|
|
713
|
-
// A state that reads scans a line; one with a dart holds its second look.
|
|
714
|
-
// Neither during a glance: a check-in on the user is steady or it is not
|
|
715
|
-
// one.
|
|
716
|
-
if (gst.read && !glanceUntil) {
|
|
717
|
-
if (elapsed > driftAt) readStep(gst.read);
|
|
718
|
-
} else {
|
|
719
|
-
gaze.drift.x = glanceUntil ? 0 : dart.x;
|
|
720
|
-
gaze.drift.y = glanceUntil ? 0 : dart.y;
|
|
721
|
-
}
|
|
722
|
-
// Aversion is a property of the state, but it is held off around a turn
|
|
723
|
-
// boundary: the floor is handed over under mutual gaze, and an avatar that
|
|
724
|
-
// looks away exactly as the user finishes has declined it. `attend` is the
|
|
725
|
-
// mixer's one-frame veto — anything that means "the user is checking
|
|
726
|
-
// whether I am with them" sets it (see api.attend).
|
|
727
|
-
gaze.setAversion(gst.aversion ? AVERSION[gst.aversion] : null);
|
|
728
|
-
gaze.hold = attendUntil > elapsed || clip.playing;
|
|
729
|
-
|
|
730
|
-
engagement.enabled = !!st.engagement && !clip.playing;
|
|
731
|
-
engagement.update(dt);
|
|
732
|
-
// Engagement posture: forward lean while the user holds the floor, spent
|
|
733
|
-
// only in the states that are *about* the user holding the floor. The
|
|
734
|
-
// research (docs/research-biomechanics.md §6.3) puts sustained attentive
|
|
735
|
-
// lean at +0.15–0.25; engage glides, and torsoLean's 0.24s tau smooths
|
|
736
|
-
// the state gate, so the lean arrives and leaves like weight shifting.
|
|
737
|
-
if (st.engagement) {
|
|
738
|
-
target.torsoLean += 0.16 * engagement.engage;
|
|
739
|
-
// The shoulders come with it. A lean is a whole upper body arriving, and
|
|
740
|
-
// in a head-and-shoulders crop the shoulder line is the part of it that
|
|
741
|
-
// is actually on screen — §6.1 calls a shoulder rise the most legible
|
|
742
|
-
// thing this framing can draw. Without this the listening shoulders were
|
|
743
|
-
// the posture shift's ±0.07 and nothing else, so the one channel that
|
|
744
|
-
// could show attention sat still through every turn the user took.
|
|
745
|
-
// It rides `engage`, so it is contingent on the user's voice rather than
|
|
746
|
-
// on the state: quick in, slow out, and gone a few seconds into silence.
|
|
747
|
-
target.shoulderL += ENGAGE_SHOULDER * engagement.engage;
|
|
748
|
-
target.shoulderR += ENGAGE_SHOULDER * engagement.engage;
|
|
749
|
-
}
|
|
750
|
-
// Straining leans harder while there is actually a faint voice to strain
|
|
751
|
-
// after. engage already tracks "the user is (barely) talking", so this
|
|
752
|
-
// costs nothing; with no user signal the static pose carries the state.
|
|
753
|
-
else if (stateName === 'CANT_HEAR') target.torsoLean += 0.10 * engagement.engage;
|
|
754
|
-
|
|
755
|
-
// 4. mouth. The server's viseme track wins; a clip's mouth track fills the
|
|
756
|
-
// gaps. There is deliberately no third leg: with no cues the mouth stays
|
|
757
|
-
// shut, and a still mouth under speech is the *visible* symptom of a
|
|
758
|
-
// backend that could not align.
|
|
759
|
-
const clipOut = clip.update(dtMs);
|
|
760
|
-
let mouth = speech.sample();
|
|
761
|
-
let mouthOwner = mouth ? 'speech' : null;
|
|
762
|
-
if (!mouth && clipOut.ownsMouth && clipOut.mouth) { mouth = clipOut.mouth; mouthOwner = 'clip'; }
|
|
763
|
-
if (mouth) {
|
|
764
|
-
const shape = mouth.letter !== SILENT
|
|
765
|
-
? shapeFor(mouth.letter, mouth.intensity)
|
|
766
|
-
: shapeFor(SILENT, 1);
|
|
767
|
-
// Rounding coming up is held against the shape's own (VisemeTrack.sample).
|
|
768
|
-
if (mouth.round > shape.mouthRound) shape.mouthRound = mouth.round;
|
|
769
|
-
// Gain pivots on the rest shape, not on zero: scaling absolute values would
|
|
770
|
-
// drag the closed mouth open, which is the one thing lipsync must never do.
|
|
771
|
-
for (const k in shape) {
|
|
772
|
-
target[k] = mouthGain === 1
|
|
773
|
-
? shape[k]
|
|
774
|
-
: REST_SHAPE[k] + (shape[k] - REST_SHAPE[k]) * mouthGain;
|
|
775
|
-
}
|
|
776
|
-
// Clip-owned mouths are exempt — a spoken OKAY *is* the warmth episode —
|
|
777
|
-
// and only the base is scaled, so a gesture clip can still smile over a
|
|
778
|
-
// sentence by authoring corner keys (they add, unscaled, in step 5).
|
|
779
|
-
if (mouthOwner !== 'clip') {
|
|
780
|
-
target.mouthCornerL *= SPEAK_SMILE_RETAIN;
|
|
781
|
-
target.mouthCornerR *= SPEAK_SMILE_RETAIN;
|
|
782
|
-
}
|
|
783
|
-
}
|
|
784
|
-
|
|
785
|
-
// 4b. speech prosody: pause blinks, inbreaths, a head held per phrase and
|
|
786
|
-
// moved between them, beats and turn-edge warmth, read off the cue
|
|
787
|
-
// track (prosody.js has the research, head.js the hold-and-move). Off while a clip is gesturing, since the clip is
|
|
788
|
-
// already the head's and the brows' story. The warmth is withheld
|
|
789
|
-
// under an emotion whose corners are down: a smile starting a turn of
|
|
790
|
-
// concern is the wrong face, whatever the rhythm says.
|
|
791
|
-
const pro = prosody.update(speech, mouthOwner === 'speech' && !clip.playing, dt);
|
|
792
|
-
// The warmth rides over a clip, because an acknowledgement's smile is
|
|
793
|
-
// *with* its nod: behind the clip gate it arrived as the nod finished, a
|
|
794
|
-
// smile at nothing. It is withheld under a clip whose own corners go down
|
|
795
|
-
// (a shake, a sorry) for the same reason as under a concerned emotion.
|
|
796
|
-
const clipFrowns = !!clipOut.delta
|
|
797
|
-
&& Math.min(clipOut.delta.mouthCornerL || 0, clipOut.delta.mouthCornerR || 0) < 0;
|
|
798
|
-
const warm = clipFrowns || ((EMOTIONS[emotion] || EMOTIONS.neutral).mouthCornerL || 0) < 0
|
|
799
|
-
? 0 : prosodyFaceGain;
|
|
800
|
-
target.mouthCornerL += pro.mouthCornerL * warm;
|
|
801
|
-
target.mouthCornerR += pro.mouthCornerR * warm;
|
|
802
|
-
target.squintL += pro.squintL * warm;
|
|
803
|
-
target.squintR += pro.squintR * warm;
|
|
804
|
-
if (!clip.playing) {
|
|
805
|
-
target.headPitch += pro.headPitch * prosodyHeadGain * headGain.headPitch;
|
|
806
|
-
target.headYaw += pro.headYaw * prosodyHeadGain * headGain.headYaw;
|
|
807
|
-
target.headRoll += pro.headRoll * prosodyHeadGain * headGain.headRoll;
|
|
808
|
-
for (const c of HEAD_AXES) hold[c] += pro.hold[c] * prosodyHeadGain * headGain[c];
|
|
809
|
-
// The trunk follows a speech pose the way it follows a gaze turn (2b),
|
|
810
|
-
// and for a second reason on a 2.5-D head: a turn the shoulders take
|
|
811
|
-
// part of is a turn the neck does not have to stretch for, and the
|
|
812
|
-
// stretch is the one thing a video reviewer saw on every pose change.
|
|
813
|
-
// Scaled with the yaw it follows: a trunk that kept its old share of a
|
|
814
|
-
// wider turn would be the neck stretching again, which is the defect
|
|
815
|
-
// this line exists to fix.
|
|
816
|
-
target.torsoTurn += pro.trunkYaw * prosodyHeadGain * headGain.headYaw * TRUNK_FOLLOW;
|
|
817
|
-
target.browRaiseL += pro.browRaiseL * prosodyFaceGain;
|
|
818
|
-
target.browRaiseR += pro.browRaiseR * prosodyFaceGain;
|
|
819
|
-
target.browInnerL += pro.browInnerL * prosodyFaceGain;
|
|
820
|
-
target.browInnerR += pro.browInnerR * prosodyFaceGain;
|
|
821
|
-
target.browAngleL += pro.browAngleL * prosodyFaceGain;
|
|
822
|
-
target.browAngleR += pro.browAngleR * prosodyFaceGain;
|
|
823
|
-
target.lidL += pro.lidL * prosodyFaceGain;
|
|
824
|
-
target.lidR += pro.lidR * prosodyFaceGain;
|
|
825
|
-
target.breath += pro.breath * idle.gain;
|
|
826
|
-
// The trunk's share of speech rhythm. Scaled by `idle.gain` and not by
|
|
827
|
-
// `prosodyHeadGain`, because this is body liveness and that is the knob a
|
|
828
|
-
// host turns down when it is re-encoding the avatar into an outgoing
|
|
829
|
-
// stream — the same factor `idle.js`'s shoulders and lean already take.
|
|
830
|
-
target.shoulderL += pro.shoulderL * idle.gain;
|
|
831
|
-
target.shoulderR += pro.shoulderR * idle.gain;
|
|
832
|
-
target.torsoLean += pro.torsoLean * idle.gain;
|
|
833
|
-
}
|
|
834
|
-
if (pro.blink) idle.phraseBlink();
|
|
835
|
-
|
|
836
|
-
// 5. gesture deltas (additive, so a nod survives whatever else is happening)
|
|
837
|
-
if (clipOut.delta) {
|
|
838
|
-
for (const k in clipOut.delta) {
|
|
839
|
-
// Never let a clip's mouth keyframes fight the live viseme stream.
|
|
840
|
-
if (mouthOwner === 'speech' && MOUTH_LOCK.has(k)) continue;
|
|
841
|
-
target[k] = (target[k] || 0) + clipOut.delta[k] * gestureGain;
|
|
842
|
-
}
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
// 6. idle: sway, breath, blink
|
|
846
|
-
// The torso's share of the liveness follows whether sound is actually being
|
|
847
|
-
// produced, not what state the avatar is nominally in — a SPEAKING state with the
|
|
848
|
-
// track finished should already be settling.
|
|
849
|
-
idle.talk = approach(idle.talk, mouthOwner ? 1 : 0, 0.25, dt);
|
|
850
|
-
idle.setProfile(st.idle);
|
|
851
|
-
const il = idle.update(dt);
|
|
852
|
-
for (const k in il.add) target[k] = (target[k] || 0) + il.add[k] * (headGain[k] ?? 1);
|
|
853
|
-
for (const c of HEAD_AXES) hold[c] += (il.add[c] || 0) * headGain[c];
|
|
854
|
-
|
|
855
|
-
// 6b. the held-head budget (internal-mixer.md § The held-head budget).
|
|
856
|
-
// Only the excess over `soften` comes off, and only off the hold: a
|
|
857
|
-
// nod, a beat and a clip keep every degree they were authored with,
|
|
858
|
-
// which is why this is subtracted here rather than applied to the pose.
|
|
859
|
-
for (const c of HEAD_AXES) {
|
|
860
|
-
if (headHold[c] === undefined) continue;
|
|
861
|
-
target[c] -= hold[c] - soften(hold[c], headHold[c]);
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
// 6c. the body answers a held tilt (SHOULDER_TILT). The trunk takes the
|
|
865
|
-
// same share of it that it takes of a turn in 2b — Live2D gives its
|
|
866
|
-
// body the same fraction of AngleZ as of AngleX (research-head-rotation
|
|
867
|
-
// .md §3) — and the shoulder line tips with the head. Both channels
|
|
868
|
-
// smooth slower than the head (TAU), so the body leaves late and
|
|
869
|
-
// settles late, and that follow-through is most of what separates a
|
|
870
|
-
// neck bending from a hinge.
|
|
871
|
-
// Only the *held* roll, and after the budget: a stroke or a clip's roll
|
|
872
|
-
// is a gesture riding on the pose, and a body that answers those is 2b's
|
|
873
|
-
// mannequin on a turntable. The idle layer's own posture is already
|
|
874
|
-
// coupled the other way round, from the weight shift to the head that
|
|
875
|
-
// counter-tips on it (idle.js `nextPosture`); this is that arrangement
|
|
876
|
-
// read from the head's end, and where both are in play they agree in
|
|
877
|
-
// sign — weight onto a side, head over that side, that shoulder up.
|
|
878
|
-
const heldRoll = headHold.headRoll === undefined
|
|
879
|
-
? hold.headRoll
|
|
880
|
-
: soften(hold.headRoll, headHold.headRoll);
|
|
881
|
-
target.torsoTurn += heldRoll * TRUNK_FOLLOW;
|
|
882
|
-
target.shoulderR += heldRoll * SHOULDER_TILT;
|
|
883
|
-
target.shoulderL -= heldRoll * SHOULDER_TILT;
|
|
884
|
-
|
|
885
|
-
// 7. clamp, then blink wins outright over whatever the lids were doing
|
|
886
|
-
for (const c of CHANNELS) {
|
|
887
|
-
const r = RANGE[c];
|
|
888
|
-
target[c] = clamp(target[c], r[0], r[1]);
|
|
889
|
-
}
|
|
890
|
-
// 7a. a bilabial aims past shut (`LIP_CONTACT_AIM`), after the clamp
|
|
891
|
-
// because it is the one target no pose is allowed to hold: only the
|
|
892
|
-
// chase sees it, and step 8b stops what is drawn at shut.
|
|
893
|
-
if (mouth && mouth.letter === 'A') target.mouthOpen = -LIP_CONTACT_AIM;
|
|
894
|
-
if (il.blink > 0) {
|
|
895
|
-
target.lidL = Math.max(target.lidL, il.blink);
|
|
896
|
-
target.lidR = Math.max(target.lidR, il.blink);
|
|
897
|
-
}
|
|
898
|
-
|
|
899
|
-
if (overrides) for (const k in overrides) target[k] = overrides[k];
|
|
900
|
-
|
|
901
|
-
// 8. smooth toward the target — this is where co-articulation happens
|
|
902
|
-
for (const c of CHANNELS) cur[c] = approach(cur[c], target[c], TAU[c], dt);
|
|
903
|
-
|
|
904
|
-
// 8b. What is drawn: the lips stop at shut however far past it they aimed.
|
|
905
|
-
Object.assign(shown, cur);
|
|
906
|
-
if (shown.mouthOpen < 0) shown.mouthOpen = 0;
|
|
907
|
-
|
|
908
|
-
// 8c. The vestibulo-ocular reflex. Here and not in the gaze layer because
|
|
909
|
-
// only here is the head that is actually drawn known: prosody, clips
|
|
910
|
-
// and idle all land after gaze.
|
|
911
|
-
if (vor) {
|
|
912
|
-
if (!overrides || overrides.pupilX === undefined) {
|
|
913
|
-
shown.pupilX = clamp(reflexX(cur.pupilX), RANGE.pupilX[0], RANGE.pupilX[1]);
|
|
914
|
-
}
|
|
915
|
-
if (!overrides || overrides.pupilY === undefined) {
|
|
916
|
-
shown.pupilY = clamp(reflexY(cur.pupilY), RANGE.pupilY[0], RANGE.pupilY[1]);
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
// 9. First-class hand control. The semantic frame is generated here, above
|
|
921
|
-
// every renderer, so SVG, WebGL, and video rigs receive exactly the same
|
|
922
|
-
// gesture/progress information. A handless rig simply ignores `frame.hand`.
|
|
923
|
-
const handFrame = updateHandAction(elapsed * 1000);
|
|
924
|
-
rig.apply(avatarFrame(shown, handFrame || undefined));
|
|
925
|
-
}
|
|
926
|
-
|
|
927
|
-
const REST_SHAPE = shapeFor(SILENT, 1);
|
|
928
|
-
|
|
929
|
-
// How much of a sustained head turn the trunk takes up. Well under 1: people
|
|
930
|
-
// under-rotate the head and then under-rotate the trunk again behind it, and
|
|
931
|
-
// at a head-and-shoulders crop the trunk's share is the part you register
|
|
932
|
-
// without noticing. A rig may pass its own `trunkFollow`: on a mesh head whose
|
|
933
|
-
// neck twists, the trunk's sway is most of what moves the neck's outline, so
|
|
934
|
-
// the share that reads as a body on a line face reads there as the neck
|
|
935
|
-
// sliding.
|
|
936
|
-
const TRUNK_FOLLOW = opts.trunkFollow ?? 0.45;
|
|
937
|
-
|
|
938
|
-
// The channels speech owns outright — exactly the params.js mouth group
|
|
939
|
-
// (mouth corners stay free: a clip may smile over a sentence).
|
|
940
|
-
const MOUTH_LOCK = new Set(GROUPS.mouth);
|
|
941
|
-
|
|
942
|
-
/** The shoulders' share of the attentive posture: two thirds of the lean the
|
|
943
|
-
* engagement layer spends — the shoulders come up with it, they do not lead it. */
|
|
944
|
-
const ENGAGE_SHOULDER = 0.10;
|
|
945
|
-
|
|
946
|
-
// What survives of the resting/emotion smile while speech owns the mouth.
|
|
947
|
-
// A smile held static through a sentence is discounted as insincere, and
|
|
948
|
-
// corners riding every open viseme read as laughing through the words: warmth
|
|
949
|
-
// must be episodic (research-perception.md §3). Full warmth returns the moment
|
|
950
|
-
// the track ends, which is that onset/offset.
|
|
951
|
-
const SPEAK_SMILE_RETAIN = 0.35;
|
|
952
|
-
|
|
953
|
-
// Between the user's turn and the reply the server's claim can change
|
|
954
|
-
// several times a second — THINKING, a tool's WORKING, THINKING again, a
|
|
955
|
-
// grace timer's CANT_HEAR — and every change used to retarget the eyes and
|
|
956
|
-
// restart the state's looks: one fast tool call was away, down at the
|
|
957
|
-
// screen, back to the user and away again inside a second, a blink at each.
|
|
958
|
-
// Among these states the eyes change over only once the new one has held
|
|
959
|
-
// this long, and a claim that returns before then moves nothing. The pose
|
|
960
|
-
// still changes at once — it is the eyes that make a flicker visible.
|
|
961
|
-
// SPEAKING and LISTENING are never held back: those are the floor.
|
|
962
|
-
const GAP_STATES = new Set(['THINKING', 'WORKING', 'CANT_HEAR']);
|
|
963
|
-
const GAP_SETTLE = 0.5;
|
|
964
|
-
|
|
965
|
-
// A dart leaves at up to 40° off the line of the look it is inside, and
|
|
966
|
-
// lifts the brows for half a second — a thought arriving, not a stare.
|
|
967
|
-
const DART_TURN = 0.7;
|
|
968
|
-
const DART_BROW = 0.5;
|
|
969
|
-
|
|
970
|
-
/** A look away for the return leg of a glance. `stick` of the time it keeps
|
|
971
|
-
* the side the last one took, spot and all — the side a person thinks
|
|
972
|
-
* toward is theirs, and it outlasts one thinking pause. Otherwise any look
|
|
973
|
-
* but the one just taken: a thinker who goes back to the same patch of
|
|
974
|
-
* wall every time *without* meaning to is a loop. */
|
|
975
|
-
function pickBack(xs, stick = 0) {
|
|
976
|
-
const side = (n) => Math.sign(gaze.targets[n].px);
|
|
977
|
-
let g;
|
|
978
|
-
if (lastBack && Math.random() < stick) {
|
|
979
|
-
const same = xs.filter((n) => side(n) === side(lastBack));
|
|
980
|
-
g = pick(same.length ? same : xs);
|
|
981
|
-
} else {
|
|
982
|
-
g = pick(xs);
|
|
983
|
-
for (let i = 0; i < 4 && g === lastBack; i++) g = pick(xs);
|
|
984
|
-
}
|
|
985
|
-
lastBack = g;
|
|
986
|
-
return g;
|
|
987
|
-
}
|
|
988
|
-
|
|
989
|
-
/**
|
|
990
|
-
* One fixation of a reading scan: a step along the line, or at its end a
|
|
991
|
-
* sweep back to the start of the next, or at the end of the block a pause
|
|
992
|
-
* on the result. That is what busy-at-a-screen looks like from the other
|
|
993
|
-
* side of a webcam — an irregular left-to-right march with returns, which
|
|
994
|
-
* a random drift about a point is not. Steps are held, like a reader's
|
|
995
|
-
* fixations, and the head sits out all of it (DRIFT_HEAD in gaze.js).
|
|
996
|
-
*/
|
|
997
|
-
function readStep(r) {
|
|
998
|
-
const n = ([a, b]) => a + ((Math.random() * (b - a + 1)) | 0);
|
|
999
|
-
if (readCol < readCols) {
|
|
1000
|
-
readCol++;
|
|
1001
|
-
driftAt = elapsed + rand(r.fix);
|
|
1002
|
-
} else {
|
|
1003
|
-
readCol = 0;
|
|
1004
|
-
readCols = n(r.steps);
|
|
1005
|
-
if (++readRow >= readRows) {
|
|
1006
|
-
readRow = 0;
|
|
1007
|
-
readRows = n(r.lines);
|
|
1008
|
-
driftAt = elapsed + rand(r.pause);
|
|
1009
|
-
} else {
|
|
1010
|
-
driftAt = elapsed + rand(r.fix) * 1.4;
|
|
1011
|
-
}
|
|
1012
|
-
}
|
|
1013
|
-
gaze.drift.x = (readCol / readCols - 0.5) * r.width * saccadeGain;
|
|
1014
|
-
gaze.drift.y = (readRow - (readRows - 1) / 2) * r.line * saccadeGain;
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
// --- API ------------------------------------------------------------------
|
|
1018
|
-
|
|
1019
|
-
function setState(name, o = {}) {
|
|
1020
|
-
if (!states[name]) throw new Error(`unknown state: ${name}`);
|
|
1021
|
-
const changed = name !== stateName;
|
|
1022
|
-
stateName = name;
|
|
1023
|
-
const st = states[name];
|
|
1024
|
-
if (o.emotion !== undefined) emotion = o.emotion;
|
|
1025
|
-
else if (changed) emotion = st.emotion;
|
|
1026
|
-
if (o.intensity !== undefined) emotionAmt = o.intensity;
|
|
1027
|
-
idle.setProfile(st.idle);
|
|
1028
|
-
// SVG's desaturation filter is a legacy renderer detail. A generic rig
|
|
1029
|
-
// receives the same state pose and may express degradation its own way.
|
|
1030
|
-
if (face) {
|
|
1031
|
-
face.svg.style.filter = st.filter || '';
|
|
1032
|
-
face.svg.style.transition = 'filter .5s ease';
|
|
1033
|
-
}
|
|
1034
|
-
// The eyes wait out a flicker between gap states (GAP_SETTLE), and a
|
|
1035
|
-
// repeat of the state still settling leaves it settling.
|
|
1036
|
-
const deferrable = !o.gaze && !o.keepGaze && GAP_STATES.has(gazeState) && GAP_STATES.has(name);
|
|
1037
|
-
if (deferrable && changed) settleAt = name === gazeState ? 0 : elapsed + GAP_SETTLE;
|
|
1038
|
-
else if (!(deferrable && settleAt)) { settleAt = 0; enterGaze(name, o, changed); }
|
|
1039
|
-
// The floor has come back to the user: the face receives it (prosody.js
|
|
1040
|
-
// `listen`). It is a state change rather than a VAD event on purpose —
|
|
1041
|
-
// what is being welcomed is the turn, and the server is the one that knows
|
|
1042
|
-
// a turn has changed hands.
|
|
1043
|
-
if (changed && name === 'LISTENING') prosody.listen();
|
|
1044
|
-
if (changed) emit('state', name);
|
|
1045
|
-
return api;
|
|
1046
|
-
}
|
|
1047
|
-
|
|
1048
|
-
// Whether the eyes were last aimed by somebody who meant it — the public
|
|
1049
|
-
// `setGaze`, or a performance's `gaze` verb — rather than by the state's own
|
|
1050
|
-
// schedule. It is the one thing that outranks a state's `gaze`, and it is
|
|
1051
|
-
// why `speak()` can take the eyes back without overriding a caller.
|
|
1052
|
-
let gazeExplicit = false;
|
|
1053
|
-
|
|
1054
|
-
/** The state `name` takes the gaze: its target and its schedules. */
|
|
1055
|
-
function enterGaze(name, o, blink) {
|
|
1056
|
-
gazeState = name;
|
|
1057
|
-
const st = states[name];
|
|
1058
|
-
const gl = st.glance;
|
|
1059
|
-
// A state with an `opening` enters as though its check-in on the user is
|
|
1060
|
-
// already under way, and leaves it when that runs out.
|
|
1061
|
-
if (!o.keepGaze) { setGaze(o.gaze || (gl && gl.opening ? gl.to : st.gaze)); gazeExplicit = false; }
|
|
1062
|
-
// Arm every scheduler fresh, so entering a state never fires a timestamp
|
|
1063
|
-
// left over from the last one — the wander in particular, which used to
|
|
1064
|
-
// pick a new target on the first frame and override the state's own gaze.
|
|
1065
|
-
glanceUntil = gl && gl.opening && !o.gaze && !o.keepGaze ? elapsed + rand(gl.opening) : 0;
|
|
1066
|
-
glanceAt = elapsed + (gl ? rand(gl.every) : 0);
|
|
1067
|
-
wanderAt = elapsed + (st.wander ? rand(st.wander.every) : 0);
|
|
1068
|
-
driftAt = elapsed + (st.read ? rand(st.read.fix) : 0);
|
|
1069
|
-
readCol = readCols = readRow = readRows = 0;
|
|
1070
|
-
dartAt = dartBrowUntil = 0;
|
|
1071
|
-
dart.x = dart.y = 0;
|
|
1072
|
-
if (blink) idle.blink(true);
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
|
-
function setEmotion(name, intensity = 1) { emotion = name; emotionAmt = intensity; return api; }
|
|
1076
|
-
|
|
1077
|
-
/** @param {string} name @param {{x:number,y:number}} [custom] normalized -1..1
|
|
1078
|
-
* @param {boolean} [blink] the mixer's own say over the evoked blink; see GazeLayer.set */
|
|
1079
|
-
function setGaze(name, custom, blink) {
|
|
1080
|
-
gazeName = GAZE_TARGETS[name] ? name : 'USER';
|
|
1081
|
-
gazeCustom = custom || null;
|
|
1082
|
-
applyGaze(blink);
|
|
1083
|
-
return api;
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
/**
|
|
1087
|
-
* @param {object} o
|
|
1088
|
-
* @param {Array<{t:number,v:string,i?:number}>} o.cues
|
|
1089
|
-
* @param {HTMLMediaElement} [o.audio] preferred clock source
|
|
1090
|
-
* @param {() => number} [o.clock] custom ms clock, if you drive audio yourself
|
|
1091
|
-
*/
|
|
1092
|
-
function speak(o = {}) {
|
|
1093
|
-
// Speech owns the mouth in the mixer, but a server action still gets to
|
|
1094
|
-
// complete its physical landing on the other channels. Do not cancel it
|
|
1095
|
-
// here: a hand or head cannot disappear simply because playout began.
|
|
1096
|
-
// A new clock is a new turn. The same clock handed back is the accurate
|
|
1097
|
-
// leg rewriting this one (AvatarClient re-speaks a splice), and that must
|
|
1098
|
-
// not roll the turn-start look a second time.
|
|
1099
|
-
const newTurn = !o.clock || o.clock !== speakClock;
|
|
1100
|
-
speakStart = performance.now();
|
|
1101
|
-
speakClock = o.clock
|
|
1102
|
-
? o.clock
|
|
1103
|
-
: o.audio
|
|
1104
|
-
? () => o.audio.currentTime * 1000
|
|
1105
|
-
: () => performance.now() - speakStart;
|
|
1106
|
-
speech.start(o.cues || [], speakClock);
|
|
1107
|
-
prosody.reset(newTurn);
|
|
1108
|
-
// The eyes come back to the user when the audio starts: a reply arriving
|
|
1109
|
-
// while THINKING was looking away would otherwise spend its whole turn
|
|
1110
|
-
// aimed off the user, because nothing in SPEAKING retargets. `keepGaze` is
|
|
1111
|
-
// for the one caller that means it — a performance that aimed the eyes with
|
|
1112
|
-
// its own `gaze` verb keeps them, an instruction and not a leftover.
|
|
1113
|
-
if (stateName !== 'SPEAKING') setState('SPEAKING', { keepGaze: gazeExplicit });
|
|
1114
|
-
if (o.audio && o.audio.paused) o.audio.play().catch(() => {});
|
|
1115
|
-
return api;
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
/**
|
|
1119
|
-
* "The user may be checking whether I am with them — hold their eyes."
|
|
1120
|
-
*
|
|
1121
|
-
* This is the widget's half of the **gaze window**. In face-to-face talk a
|
|
1122
|
-
* speaker periodically looks at the listener, mutual gaze is established, the
|
|
1123
|
-
* listener responds inside that window, and the speaker looks away again
|
|
1124
|
-
* (Bavelas, Coates & Johnson 2002).
|
|
1125
|
-
*
|
|
1126
|
-
* We cannot see the user, so we cannot observe the window opening. What a
|
|
1127
|
-
* caller *can* do is name the moments that co-occur with it — a mid-turn
|
|
1128
|
-
* pause, a tag question ("...right?", "you know?"), a completed clause with
|
|
1129
|
-
* the turn analyzer's completion probability high, the user answering a
|
|
1130
|
-
* question the bot asked. For `ms` the face stops averting and holds the
|
|
1131
|
-
* user, which is the prerequisite for any response to be *seen*.
|
|
1132
|
-
*
|
|
1133
|
-
* @param {number} [ms=1200] how long to hold. Binetti (N=498) puts preferred
|
|
1134
|
-
* mutual gaze at 3295 ± 706 ms, so this is a fraction of the ceiling.
|
|
1135
|
-
*/
|
|
1136
|
-
function attend(ms = 1200) {
|
|
1137
|
-
attendUntil = Math.max(attendUntil, elapsed + ms / 1000);
|
|
1138
|
-
return api;
|
|
1139
|
-
}
|
|
1140
|
-
|
|
1141
|
-
function pushCues(cues) { speech.push(cues); return api; }
|
|
1142
|
-
|
|
1143
|
-
function stopSpeaking() { speech.stop(); return api; }
|
|
1144
|
-
|
|
1145
|
-
function interject(id) {
|
|
1146
|
-
const c = INTERNAL_CLIPS[id];
|
|
1147
|
-
if (!c) throw new Error(`unknown interjection: ${id}`);
|
|
1148
|
-
clip.play(c, c.audioEl);
|
|
1149
|
-
return api;
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
/**
|
|
1153
|
-
* A hand gesture: the hand at the frame edge, plus the face half that makes it
|
|
1154
|
-
* belong to somebody.
|
|
1155
|
-
*
|
|
1156
|
-
* The face half is not a convenience — a hand rising to the jaw over a head
|
|
1157
|
-
* and shoulders sitting perfectly still is a cut-out, not a gesture. Each
|
|
1158
|
-
* entry in HAND_GESTURES names the matching semantic face action; firing it here is the
|
|
1159
|
-
* library composing two authored things, not the client inventing motion.
|
|
1160
|
-
*
|
|
1161
|
-
* On an avatar mounted with `hand: false` this internal helper degrades to
|
|
1162
|
-
* the face action alone.
|
|
1163
|
-
*/
|
|
1164
|
-
function gesture(id) {
|
|
1165
|
-
const def = HAND_GESTURES[id];
|
|
1166
|
-
if (!def) throw new Error(`unknown hand gesture: ${id}`);
|
|
1167
|
-
if (hand) hand.play(id, elapsed * 1000);
|
|
1168
|
-
if (def.face) interject(def.face);
|
|
1169
|
-
return api;
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
/** One self-completing server action. State continues to resolve underneath;
|
|
1173
|
-
* face and hand layers queue their next movement so an in-flight physical
|
|
1174
|
-
* gesture always gets to land. */
|
|
1175
|
-
function action(wireId) {
|
|
1176
|
-
// `ACKNOWLEDGE` is the whole backchannel family in one word, and which of
|
|
1177
|
-
// them a face makes is a rendering decision the server is not holding the
|
|
1178
|
-
// drawing for. The floor decides: a nod while the user still has it is a
|
|
1179
|
-
// continuer, "go on"; once they have stopped it is a receipt, and a nod
|
|
1180
|
-
// there reads as agreement with whatever they just said. Both are only ever
|
|
1181
|
-
// this explicit action — nothing here starts one on its own.
|
|
1182
|
-
const id = wireId === 'ACKNOWLEDGE'
|
|
1183
|
-
? (engagement.speaking ? 'ACK_NOD' : 'ACK_RECEIVE')
|
|
1184
|
-
: wireId;
|
|
1185
|
-
const handDef = HAND_GESTURES[id];
|
|
1186
|
-
if (handDef) {
|
|
1187
|
-
startHandAction(id, handDef);
|
|
1188
|
-
if (handDef.face) {
|
|
1189
|
-
const faceClip = actionShapes[handDef.face] || ACTIONS[handDef.face];
|
|
1190
|
-
if (faceClip) clip.play(faceClip, faceClip.audioEl, { queue: true });
|
|
1191
|
-
}
|
|
1192
|
-
return api;
|
|
1193
|
-
}
|
|
1194
|
-
// Own renderings first, always. An avatar may *add* to what a server can
|
|
1195
|
-
// ask for and may never redefine a core intent: `ACKNOWLEDGE` has to mean
|
|
1196
|
-
// the same thing on every face or it is not a protocol. It may draw that
|
|
1197
|
-
// meaning in its own shape (`actionShapes`), which is a rendering, not a
|
|
1198
|
-
// redefinition — the id and the intent are fixed before this line.
|
|
1199
|
-
//
|
|
1200
|
-
// Unknown is a no-op, not a throw. The wire's action vocabulary is open, so
|
|
1201
|
-
// a server asking this face for a motion it does not have is the expected
|
|
1202
|
-
// case and not somebody's broken build — the same forward-compat rule an
|
|
1203
|
-
// unknown `cmd` gets. This is also where a sequence used to arrive by its
|
|
1204
|
-
// own method; one open vocabulary means one door.
|
|
1205
|
-
const faceClip = actionShapes[id] || ACTIONS[id] || sequences[id];
|
|
1206
|
-
if (!faceClip) return api;
|
|
1207
|
-
clip.play(faceClip, faceClip.audioEl, { queue: true });
|
|
1208
|
-
// The acknowledgements smile. Only these, and only ever because the
|
|
1209
|
-
// server sent one: a smile the renderer timed for itself would be an
|
|
1210
|
-
// acknowledgement nobody sent.
|
|
1211
|
-
if (id === 'ACK_NOD' || id === 'ACK_RECEIVE') prosody.acknowledge();
|
|
1212
|
-
// Now that warmth rides over clips, the reply's opening smile would
|
|
1213
|
-
// otherwise carry on through the interrupted face.
|
|
1214
|
-
if (id === 'RESPONSE_INTERRUPTED') prosody.cool();
|
|
1215
|
-
return api;
|
|
1216
|
-
}
|
|
1217
|
-
|
|
1218
|
-
function startHandAction(id, def) {
|
|
1219
|
-
const gesture = HAND_ACTION_TO_FRAME_GESTURE[id];
|
|
1220
|
-
if (!gesture) return;
|
|
1221
|
-
if (handAction) {
|
|
1222
|
-
if (handAction.id !== id && !handQueue.some((item) => item.id === id)) {
|
|
1223
|
-
handQueue.push({ id, def, gesture });
|
|
1224
|
-
}
|
|
1225
|
-
return;
|
|
1226
|
-
}
|
|
1227
|
-
handAction = { id, def, gesture, start: elapsed * 1000 };
|
|
1228
|
-
}
|
|
1229
|
-
|
|
1230
|
-
function updateHandAction(nowMs) {
|
|
1231
|
-
if (!handAction) return null;
|
|
1232
|
-
const progress = (nowMs - handAction.start) / handAction.def.dur;
|
|
1233
|
-
if (progress >= 1) {
|
|
1234
|
-
const done = handAction;
|
|
1235
|
-
const next = handQueue.shift();
|
|
1236
|
-
handAction = next ? { ...next, start: nowMs } : null;
|
|
1237
|
-
emit('gestureEnd', done.id);
|
|
1238
|
-
return handAction ? { gesture: handAction.gesture, progress: 0, side: handSide } : null;
|
|
1239
|
-
}
|
|
1240
|
-
return { gesture: handAction.gesture, progress: Math.max(0, progress), side: handSide };
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
function setUserSpeaking(b) { engagement.setUserSpeaking(b); return api; }
|
|
1244
|
-
|
|
1245
|
-
// What one action does when its moment comes. Enum validity is checked here,
|
|
1246
|
-
// where the enums live: a bad value warns and is skipped, because one stale
|
|
1247
|
-
// verb must never take down the performance around it.
|
|
1248
|
-
function dispatchAction(a) {
|
|
1249
|
-
try {
|
|
1250
|
-
// `state` defaults to keepGaze — a timeline that wants the gaze moved
|
|
1251
|
-
// says so with a `gaze` verb at the moment it means, which is how every
|
|
1252
|
-
// composed turn in the demo already behaves.
|
|
1253
|
-
if (a.do === 'state') setState(a.name, { keepGaze: a.keepGaze !== false });
|
|
1254
|
-
else if (a.do === 'emotion') setEmotion(a.name, a.i ?? 1);
|
|
1255
|
-
else if (a.do === 'gaze') { setGaze(a.name); gazeExplicit = true; }
|
|
1256
|
-
else if (a.do === 'action') action(a.id);
|
|
1257
|
-
} catch (e) {
|
|
1258
|
-
console.warn(`perform: ${a.do} at ${a.t}ms skipped — ${e.message}`);
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
|
-
|
|
1262
|
-
let performGen = 0;
|
|
1263
|
-
|
|
1264
|
-
/**
|
|
1265
|
-
* @param {Array<{t: number, do: string}>} actions
|
|
1266
|
-
* @param {{audio?: HTMLMediaElement, clock?: () => number,
|
|
1267
|
-
* onAction?: (a: object) => void}} [o]
|
|
1268
|
-
* @returns {{stop: () => void}} stop() cancels the *future* of this
|
|
1269
|
-
* performance only: an in-flight interjection finishes, a live cue track
|
|
1270
|
-
* is untouched, and 'performEnd' does not fire. A handle whose
|
|
1271
|
-
* performance was already replaced by a newer perform() is a no-op.
|
|
1272
|
-
*/
|
|
1273
|
-
function perform(actions, o = {}) {
|
|
1274
|
-
const start = performance.now();
|
|
1275
|
-
const clock = o.clock
|
|
1276
|
-
? o.clock
|
|
1277
|
-
: o.audio
|
|
1278
|
-
? () => o.audio.currentTime * 1000
|
|
1279
|
-
: () => performance.now() - start;
|
|
1280
|
-
performTrack.onAction = (a) => { dispatchAction(a); if (o.onAction) o.onAction(a); };
|
|
1281
|
-
const gen = ++performGen;
|
|
1282
|
-
performTrack.start(actions, clock);
|
|
1283
|
-
return { stop: () => { if (gen === performGen) performTrack.stop(); } };
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
|
-
const api = {
|
|
1287
|
-
setState, setEmotion, setGaze: (name, custom) => { gazeExplicit = true; return setGaze(name, custom); }, speak, pushCues, stopSpeaking, attend,
|
|
1288
|
-
action, perform,
|
|
1289
|
-
/** Which hand the character gestures with: +1 the viewer's right (its own
|
|
1290
|
-
* left), -1 the other. Both are anatomically real — the thumb splays away
|
|
1291
|
-
* from the body either way — so this is a character choice, not a fix. */
|
|
1292
|
-
setHandSide: (d) => { handSide = d === -1 ? 'left' : 'right'; return api; },
|
|
1293
|
-
setUserSpeaking,
|
|
1294
|
-
/** Articulation gain: 1 is the VISEME_SHAPES table as authored. */
|
|
1295
|
-
setMouthGain: (g) => { mouthGain = g; return api; },
|
|
1296
|
-
get mouthGain() { return mouthGain; },
|
|
1297
|
-
/** Gesture gain: scales every clip delta. 1 is the timelines as authored. */
|
|
1298
|
-
setGestureGain: (g) => { gestureGain = g; return api; },
|
|
1299
|
-
get gestureGain() { return gestureGain; },
|
|
1300
|
-
/** Idle body-motion gain: 1 is the liveness layer as authored, 0 freezes it. */
|
|
1301
|
-
setMotionGain: (g) => { idle.gain = g; return api; },
|
|
1302
|
-
get motionGain() { return idle.gain; },
|
|
1303
|
-
blink: () => { idle.blink(); return api; },
|
|
1304
|
-
/** Advance one frame by hand. Only meaningful under `{manual: true}`;
|
|
1305
|
-
* fixed-dt stepping is what makes a motion render reproducible. */
|
|
1306
|
-
step: (dt) => { elapsed += dt; step(dt, dt * 1000); return api; },
|
|
1307
|
-
/** Direct parameter injection — for tuning UIs, not production. */
|
|
1308
|
-
setOverrides: (o) => { overrides = o; return api; },
|
|
1309
|
-
on: (ev, fn) => { (listeners[ev] || (listeners[ev] = [])).push(fn); return api; },
|
|
1310
|
-
get state() { return stateName; },
|
|
1311
|
-
get emotion() { return emotion; },
|
|
1312
|
-
get gaze() { return gazeName; },
|
|
1313
|
-
get speaking() { return speech.playing; },
|
|
1314
|
-
get performing() { return performTrack.playing; },
|
|
1315
|
-
get clip() { return clip.id; },
|
|
1316
|
-
/** Semantic hand gesture in flight, independent of renderer capability. */
|
|
1317
|
-
get gesturing() { return handAction ? handAction.id : null; },
|
|
1318
|
-
get params() { return shown; },
|
|
1319
|
-
get userSpeaking() { return engagement.speaking; },
|
|
1320
|
-
// Legacy SVG inspection fields. New AvatarRig implementations should not
|
|
1321
|
-
// rely on or provide them; the renderer-neutral contract is apply/destroy.
|
|
1322
|
-
svg: face?.svg || null,
|
|
1323
|
-
meta: meta || null,
|
|
1324
|
-
/** The mounted rig's palette, merged with any `opts.theme` overrides. A
|
|
1325
|
-
* host that has to paint anything *around* the widget — a tile margin, a
|
|
1326
|
-
* page behind a transparent mount — needs the same colours the drawing
|
|
1327
|
-
* used, and guessing them per avatar is how the two drift apart. */
|
|
1328
|
-
theme: face?.theme,
|
|
1329
|
-
destroy() {
|
|
1330
|
-
cancelAnimationFrame(raf);
|
|
1331
|
-
rig.destroy();
|
|
1332
|
-
},
|
|
1333
|
-
};
|
|
1334
|
-
|
|
1335
|
-
setState('IDLE');
|
|
1336
|
-
if (!manual) raf = requestAnimationFrame(frame);
|
|
1337
|
-
return api;
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
// The parameter space itself. A renderer-neutral rig has to answer "what is
|
|
1341
|
-
// this channel's neutral value" before it can map the channel onto whatever it
|
|
1342
|
-
// controls — a morph target's influence is `(pose - rest) / (1 - rest)`, and a
|
|
1343
|
-
// rig that hard-codes those rests has quietly forked `params.js`.
|
|
1344
|
-
export { REST, CHANNELS, RANGE, TAU, clamp, approach, makeParams } from './params.js';
|
|
1345
|
-
// The smoothing law itself, for a tool that steps the rig by hand. A
|
|
1346
|
-
// filmstrip that reimplemented `approach` would be measuring its own copy of
|
|
1347
|
-
// the thing under test — and the smoothing between keyframes is what the face
|
|
1348
|
-
// actually does (docs/internal-mixer.md § Smoothing).
|
|
1349
|
-
// A pose with no mixer above it: `makeParams(overrides)` fills the rests,
|
|
1350
|
-
// `avatarFrame` wraps it, `createSvgRig(face).apply` draws it. That is the whole
|
|
1351
|
-
// path an instrument needs to hold a face at one named extreme — no clock, no
|
|
1352
|
-
// client, no animation — and the reason it is exported is that a pose sheet
|
|
1353
|
-
// that cannot reach it forks the channel rests instead.
|
|
1354
|
-
export { avatarFrame, createSvgRig } from './rig.js';
|
|
1355
|
-
export { ACTION_IDS, ACTIONS, attachAudio } from './interjections.js';
|
|
1356
|
-
// The full authoring catalogue and the player that steps it. Not a server
|
|
1357
|
-
// vocabulary — `ACTIONS` is that, and most of these clips are reachable only
|
|
1358
|
-
// from inside the mixer. They are exported for the filmstrip instrument,
|
|
1359
|
-
// which lays one clip out as frames and therefore has to drive a real
|
|
1360
|
-
// `ClipPlayer` rather than re-sample its keys.
|
|
1361
|
-
export { INTERNAL_CLIPS } from './interjections.js';
|
|
1362
|
-
export { ClipPlayer } from './clips.js';
|
|
1363
|
-
export { GAZE_NAMES, GAZE_TARGETS } from './gaze.js';
|
|
1364
|
-
export { normalizeActions } from './perform.js';
|
|
1365
|
-
export { checkHandFraming } from './hand.js';
|
|
1366
|
-
export { EMOTION_NAMES, emotionPose } from './emotions.js';
|
|
1367
|
-
// The mouth clock travels with the rest of it. Someone has to turn a cue array
|
|
1368
|
-
// plus a clock into "which letter is on screen right now", every renderer needs
|
|
1369
|
-
// exactly that, and none of them should write it twice — so it is a plain class
|
|
1370
|
-
// to construct, not a contract to implement.
|
|
1371
|
-
export {
|
|
1372
|
-
VISEME_LETTERS, VISEME_SHAPES, VisemeTrack, shapeFor, JAW_OF_OPEN, SILENT,
|
|
1373
|
-
normalizeCues, textToCues,
|
|
1374
|
-
ARPABET_TO_VISEME, AZURE_VISEME_TO_LETTER, LEAD_MS,
|
|
1375
|
-
} from './visemes.js';
|
|
1376
|
-
// No THEME re-export: each face module owns its palette, and `api.theme` is
|
|
1377
|
-
// the mounted avatar's. A single barrel THEME was one rig's palette wearing a
|
|
1378
|
-
// public name — misleading the moment that rig stopped being the default.
|