@voqalize/avatar 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +692 -0
- package/client/dist/Avatar.d.ts +24 -0
- package/client/dist/Avatar.d.ts.map +1 -0
- package/client/dist/Avatar.js +7 -0
- package/client/dist/Avatar.js.map +1 -0
- package/client/dist/AvatarClient.d.ts +173 -0
- package/client/dist/AvatarClient.d.ts.map +1 -0
- package/client/dist/AvatarClient.js +274 -0
- package/client/dist/AvatarClient.js.map +1 -0
- package/client/dist/pipecat.d.ts +21 -0
- package/client/dist/pipecat.d.ts.map +1 -0
- package/client/dist/pipecat.js +21 -0
- package/client/dist/pipecat.js.map +1 -0
- package/client/dist/react.d.ts +16 -0
- package/client/dist/react.d.ts.map +1 -0
- package/client/dist/react.js +17 -0
- package/client/dist/react.js.map +1 -0
- package/client/dist/types.d.ts +101 -0
- package/client/dist/types.d.ts.map +1 -0
- package/client/dist/types.js +31 -0
- package/client/dist/types.js.map +1 -0
- package/client/dist/useAvatar.d.ts +53 -0
- package/client/dist/useAvatar.d.ts.map +1 -0
- package/client/dist/useAvatar.js +68 -0
- package/client/dist/useAvatar.js.map +1 -0
- package/client/src/Avatar.tsx +38 -0
- package/client/src/AvatarClient.ts +343 -0
- package/client/src/pipecat.ts +38 -0
- package/client/src/react.ts +34 -0
- package/client/src/types.ts +127 -0
- package/client/src/useAvatar.ts +113 -0
- package/docs/contract-avatar.md +337 -0
- package/docs/contract-protocol.md +401 -0
- package/package.json +89 -0
- package/src/audio-fallback.js +100 -0
- package/src/avatar.d.ts +241 -0
- package/src/avatar.js +722 -0
- package/src/clips.js +144 -0
- package/src/emotions.js +55 -0
- package/src/face-core.js +154 -0
- package/src/face-myna.js +725 -0
- package/src/face-peep.js +767 -0
- package/src/face-wren.js +470 -0
- package/src/gaze.js +155 -0
- package/src/idle.js +535 -0
- package/src/interjections.js +578 -0
- package/src/line-art.js +111 -0
- package/src/params.js +176 -0
- package/src/perform.js +105 -0
- package/src/visemes.js +230 -0
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Interjection library.
|
|
3
|
+
*
|
|
4
|
+
* These are the highest-value animations in the whole widget: they're the
|
|
5
|
+
* real-time feedback channel, and they fire on a single server token with zero
|
|
6
|
+
* generation latency because the phrase set is fixed and everything is
|
|
7
|
+
* pre-baked.
|
|
8
|
+
*
|
|
9
|
+
* Each entry carries three things:
|
|
10
|
+
* · a gesture timeline (additive deltas, normalized time 0..1)
|
|
11
|
+
* · a baked viseme track with real millisecond timings, tuned so the clip
|
|
12
|
+
* plays convincingly even with no audio at all (muted / degraded mode)
|
|
13
|
+
* · an `audio` slot for you to drop in a TTS clip in the agent's own voice
|
|
14
|
+
*
|
|
15
|
+
* If you supply audio, the mouth track re-schedules against the audio clock
|
|
16
|
+
* automatically; the timings below are calibrated to a natural, slightly brisk
|
|
17
|
+
* delivery, so a matching clip should line up with only small nudges.
|
|
18
|
+
*
|
|
19
|
+
* Timeline channel names are rig parameters. Values are deltas added on top of
|
|
20
|
+
* whatever the base pose, gaze and viseme layers are already doing.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** @typedef {{id:string,label:string,text:string,duration:number,keys:object,mouthCues?:Array,gaze?:string,blinkAt?:number[],audio?:string}} Clip */
|
|
24
|
+
|
|
25
|
+
const nod = (peak, at = 0.24, rebound = 0.56) => [
|
|
26
|
+
[0, 0], [at, peak], [rebound, -peak * 0.24], [1, 0],
|
|
27
|
+
];
|
|
28
|
+
|
|
29
|
+
export const INTERJECTIONS = {
|
|
30
|
+
// ---------------------------------------------------------------------
|
|
31
|
+
// Pure backchannel — no words, fired automatically while LISTENING.
|
|
32
|
+
// ---------------------------------------------------------------------
|
|
33
|
+
// The nod peaks below are pre-compensated for the head's 160ms time constant.
|
|
34
|
+
// Authored at their perceptual value (~0.3) they rendered at ~0.6x and read as
|
|
35
|
+
// no gesture at all — the smoothing that gives the head its mass also eats any
|
|
36
|
+
// gesture faster than it. Judge these on screen, never as numbers: 0.55 here is
|
|
37
|
+
// the same *rendered* nod that ~0.35 would be if the head had no inertia.
|
|
38
|
+
//
|
|
39
|
+
// The three nods are the mocap taxonomy (docs/research-biomechanics.md §3.3):
|
|
40
|
+
// short continuer / long assessment / long-with-upswing realization, in the
|
|
41
|
+
// corpus proportions 49/40/12. Their internal structure follows §3.4's three
|
|
42
|
+
// laws — a long nod STARTS bigger (the head knows how long the nod will be
|
|
43
|
+
// before it begins), each cycle is smaller than the last, and the final cycle
|
|
44
|
+
// drops further than the trend — and everything stays under the 1.5 Hz line
|
|
45
|
+
// where sustained attention turns into impatience.
|
|
46
|
+
NOD_SMALL: {
|
|
47
|
+
id: 'NOD_SMALL', label: 'nod (continuer)', text: '', duration: 800,
|
|
48
|
+
keys: {
|
|
49
|
+
// One cycle: single-cycle nods are 42% of the corpus, and the continuer
|
|
50
|
+
// is the quietest citizen of the family — "go on", not "I agree".
|
|
51
|
+
headPitch: nod(0.55, 0.22, 0.52),
|
|
52
|
+
browRaiseL: [[0, 0], [0.3, 0.14], [1, 0]],
|
|
53
|
+
browRaiseR: [[0, 0], [0.3, 0.14], [1, 0]],
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
NOD_SLOW: {
|
|
57
|
+
id: 'NOD_SLOW', label: 'nod (assessment, two-beat)', text: '', duration: 1420,
|
|
58
|
+
keys: {
|
|
59
|
+
// Two cycles at ~1.4 Hz. The first beat is the deepest thing any nod
|
|
60
|
+
// does (anticipatory rising) and the second sits well below it —
|
|
61
|
+
// declination plus final lowering, which is what makes the gesture
|
|
62
|
+
// resolve instead of merely stopping. The lift between beats crosses
|
|
63
|
+
// slightly above rest so the cycles read as separate strokes.
|
|
64
|
+
headPitch: [[0, 0], [0.16, 0.85], [0.38, -0.10], [0.60, 0.55], [0.85, -0.06], [1, 0]],
|
|
65
|
+
mouthCornerL: [[0, 0], [0.45, 0.12], [1, 0]],
|
|
66
|
+
mouthCornerR: [[0, 0], [0.45, 0.12], [1, 0]],
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
// The "ah — I see" nod. What separates it from agreement is the upswing:
|
|
70
|
+
// the head rises FIRST (the realization arriving), then commits to the
|
|
71
|
+
// deep beat. Brows and lids lift with the upswing and decay through the
|
|
72
|
+
// rest — the face catches on a half-beat before the head does.
|
|
73
|
+
NOD_UP: {
|
|
74
|
+
id: 'NOD_UP', label: 'nod (realization, upswing)', text: '', duration: 1750,
|
|
75
|
+
keys: {
|
|
76
|
+
headPitch: [[0, 0], [0.13, -0.34], [0.34, 0.70], [0.52, -0.08], [0.68, 0.44], [1, 0]],
|
|
77
|
+
browRaiseL: [[0, 0], [0.10, 0.42], [0.45, 0.18], [1, 0]],
|
|
78
|
+
browRaiseR: [[0, 0], [0.11, 0.38], [0.45, 0.16], [1, 0]],
|
|
79
|
+
lidL: [[0, 0], [0.12, -0.10], [0.5, 0], [1, 0]],
|
|
80
|
+
lidR: [[0, 0], [0.12, -0.10], [0.5, 0], [1, 0]],
|
|
81
|
+
// The settle carries a trace of satisfaction — getting it feels good.
|
|
82
|
+
mouthCornerL: [[0, 0], [0.5, 0.10], [0.85, 0.16], [1, 0]],
|
|
83
|
+
mouthCornerR: [[0, 0], [0.5, 0.10], [0.85, 0.16], [1, 0]],
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
BROW_ACK: {
|
|
87
|
+
id: 'BROW_ACK', label: 'brow acknowledge', text: '', duration: 720,
|
|
88
|
+
keys: {
|
|
89
|
+
browRaiseL: [[0, 0], [0.24, 0.30], [0.7, 0.06], [1, 0]],
|
|
90
|
+
browRaiseR: [[0, 0], [0.24, 0.26], [0.7, 0.06], [1, 0]],
|
|
91
|
+
headPitch: [[0, 0], [0.3, 0.08], [1, 0]],
|
|
92
|
+
mouthCornerL: [[0, 0], [0.35, 0.14], [1, 0]],
|
|
93
|
+
mouthCornerR: [[0, 0], [0.35, 0.14], [1, 0]],
|
|
94
|
+
},
|
|
95
|
+
blinkAt: [0.22],
|
|
96
|
+
},
|
|
97
|
+
// The disagree family follows the same three laws as the nods (§3.4): the
|
|
98
|
+
// first swing is the biggest, every cycle decays, and the whole gesture
|
|
99
|
+
// stays at or under ~1.5 Hz. Both are SERVER-SENT ONLY — an agent must
|
|
100
|
+
// never disagree autonomously, so neither is in the listening engine's
|
|
101
|
+
// picker.
|
|
102
|
+
HEAD_SHAKE: {
|
|
103
|
+
id: 'HEAD_SHAKE', label: 'no (firm)', text: '', duration: 1350,
|
|
104
|
+
keys: {
|
|
105
|
+
// Two decaying cycles at ~1.5 Hz, peaks pre-compensated like the nods
|
|
106
|
+
// (the old ±0.20 rendered as ambient drift, not as "no"). The mouth
|
|
107
|
+
// firms and the brows drop a touch: a "no" with a resting smile under
|
|
108
|
+
// it reads as teasing.
|
|
109
|
+
headYaw: [[0, 0], [0.16, -0.55], [0.42, 0.45], [0.68, -0.26], [0.88, 0.10], [1, 0]],
|
|
110
|
+
mouthPress: [[0, 0], [0.20, 0.35], [0.80, 0.30], [1, 0]],
|
|
111
|
+
mouthCornerL: [[0, 0], [0.25, -0.18], [1, 0]],
|
|
112
|
+
mouthCornerR: [[0, 0], [0.25, -0.18], [1, 0]],
|
|
113
|
+
browRaiseL: [[0, 0], [0.20, -0.20], [0.85, -0.10], [1, 0]],
|
|
114
|
+
browRaiseR: [[0, 0], [0.20, -0.20], [0.85, -0.10], [1, 0]],
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
// "Hmm, not quite" — the polite disagreement backchannel. Slower (~1 Hz),
|
|
118
|
+
// smaller, a cycle and a half dying away, with the sympathetic head tilt
|
|
119
|
+
// and knit brows of someone sorry to be disagreeing. The regretful set is
|
|
120
|
+
// what separates not-quite from no.
|
|
121
|
+
HEAD_SHAKE_SOFT: {
|
|
122
|
+
id: 'HEAD_SHAKE_SOFT', label: 'not quite (soft)', text: '', duration: 1700,
|
|
123
|
+
keys: {
|
|
124
|
+
// At tile size the amplitude difference from HEAD_SHAKE is thin; what
|
|
125
|
+
// actually separates not-quite from no here is the sympathetic head
|
|
126
|
+
// TILT (line-face scaled — under 0.3 the roll never survives the 5.5°
|
|
127
|
+
// multiplier's floor), the slower swing, and the sorry face.
|
|
128
|
+
headYaw: [[0, 0], [0.22, -0.26], [0.55, 0.18], [0.82, -0.09], [1, 0]],
|
|
129
|
+
headRoll: [[0, 0], [0.30, 0.50], [0.85, 0.35], [1, 0.06]],
|
|
130
|
+
browInnerL: [[0, 0], [0.30, 0.50], [1, 0.08]],
|
|
131
|
+
browInnerR: [[0, 0], [0.30, 0.42], [1, 0.06]],
|
|
132
|
+
mouthCornerL: [[0, 0], [0.40, -0.24], [1, -0.06]],
|
|
133
|
+
mouthCornerR: [[0, 0], [0.40, -0.24], [1, -0.06]],
|
|
134
|
+
mouthPress: [[0, 0], [0.35, 0.30], [1, 0]],
|
|
135
|
+
},
|
|
136
|
+
},
|
|
137
|
+
/**
|
|
138
|
+
* The deliberate ~600ms blink, paired with a barely-there nod — the studied
|
|
139
|
+
* stimulus (docs/research-biomechanics.md §3.6): listeners who got long
|
|
140
|
+
* blinks with nods gave measurably shorter answers, with no awareness of
|
|
141
|
+
* why. It reads as "that's noted — move on". SERVER-SENT ONLY: nothing
|
|
142
|
+
* autonomous may ever pick it, because it genuinely shortens what the user
|
|
143
|
+
* says next. Distinct from THINKING's 0.34s slow blink by duration alone.
|
|
144
|
+
* The lids close in ~90ms, stay down half a second, and release slower
|
|
145
|
+
* than they fell; the mouth is untouched.
|
|
146
|
+
*/
|
|
147
|
+
BLINK_LONG: {
|
|
148
|
+
id: 'BLINK_LONG', label: 'long blink (noted)', text: '', duration: 850,
|
|
149
|
+
keys: {
|
|
150
|
+
lidL: [[0, 0], [0.10, 1.0], [0.68, 1.0], [0.92, 0], [1, 0]],
|
|
151
|
+
lidR: [[0, 0], [0.10, 1.0], [0.68, 1.0], [0.92, 0], [1, 0]],
|
|
152
|
+
headPitch: [[0, 0], [0.30, 0.30], [0.75, -0.05], [1, 0]],
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
// ---------------------------------------------------------------------
|
|
157
|
+
// Floor management.
|
|
158
|
+
//
|
|
159
|
+
// These carry no words and no meaning about the conversation — they are pure
|
|
160
|
+
// turn-taking. Their whole job is to be *predictive*: a claim has to land
|
|
161
|
+
// before the audio it predicts, or it is describing the past.
|
|
162
|
+
// ---------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The inbreath before speaking. Fire this ~350ms ahead of the first audio
|
|
166
|
+
* sample — the server knows it is about to speak long before the samples
|
|
167
|
+
* exist, so the lead time is free.
|
|
168
|
+
*
|
|
169
|
+
* This is the cue that prevents collisions. Everything in it is one gesture
|
|
170
|
+
* seen from outside: the shoulders rise, the chest fills, the head comes up,
|
|
171
|
+
* the lips part. It ends held rather than resolved, because the speech it
|
|
172
|
+
* precedes is what resolves it — the clip's own ramp-out does the blending.
|
|
173
|
+
*/
|
|
174
|
+
CLAIM_FLOOR: {
|
|
175
|
+
id: 'CLAIM_FLOOR', label: 'about to speak (inhale)', text: '', duration: 480,
|
|
176
|
+
keys: {
|
|
177
|
+
breath: [[0, 0], [0.45, 0.55], [1, 0.22]],
|
|
178
|
+
shoulderL: [[0, 0], [0.45, 0.36], [1, 0.14]],
|
|
179
|
+
shoulderR: [[0, 0], [0.45, 0.34], [1, 0.13]],
|
|
180
|
+
torsoLean: [[0, 0], [0.5, 0.22], [1, 0.10]],
|
|
181
|
+
browRaiseL: [[0, 0], [0.35, 0.26], [1, 0.10]],
|
|
182
|
+
browRaiseR: [[0, 0], [0.35, 0.22], [1, 0.08]],
|
|
183
|
+
headPitch: [[0, 0], [0.4, -0.18], [1, -0.06]],
|
|
184
|
+
lidL: [[0, 0], [0.4, -0.10], [1, -0.04]],
|
|
185
|
+
lidR: [[0, 0], [0.4, -0.10], [1, -0.04]],
|
|
186
|
+
mouthOpen: [[0, 0], [0.55, 0.20], [1, 0.12]],
|
|
187
|
+
},
|
|
188
|
+
},
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Interrupted mid-word. The mouth shutting *is* the message, and it has to be
|
|
192
|
+
* the fastest thing on the face — a yield that takes as long as a nod reads as
|
|
193
|
+
* the agent finishing its sentence anyway, which is the opposite of the meaning.
|
|
194
|
+
* The peak is at 12% of 420ms, so the lips are closed inside 50ms.
|
|
195
|
+
*/
|
|
196
|
+
YIELD_FLOOR: {
|
|
197
|
+
id: 'YIELD_FLOOR', label: 'yield (interrupted)', text: '', duration: 420,
|
|
198
|
+
keys: {
|
|
199
|
+
mouthOpen: [[0, 0], [0.12, -0.34], [0.5, -0.14], [1, 0]],
|
|
200
|
+
mouthPress: [[0, 0], [0.14, 0.30], [0.6, 0.12], [1, 0]],
|
|
201
|
+
// Settling back is the second half of it: the body gives the floor up as
|
|
202
|
+
// well as the voice. Without this the face just stops, which reads as a
|
|
203
|
+
// dropped connection rather than as deference.
|
|
204
|
+
torsoLean: [[0, 0], [0.3, -0.24], [1, -0.08]],
|
|
205
|
+
shoulderL: [[0, 0], [0.25, -0.20], [1, -0.06]],
|
|
206
|
+
shoulderR: [[0, 0], [0.25, -0.20], [1, -0.06]],
|
|
207
|
+
browRaiseL: [[0, 0], [0.2, 0.18], [1, 0.04]],
|
|
208
|
+
browRaiseR: [[0, 0], [0.2, 0.15], [1, 0.04]],
|
|
209
|
+
headPitch: [[0, 0], [0.28, 0.10], [1, 0.02]],
|
|
210
|
+
},
|
|
211
|
+
blinkAt: [0.1],
|
|
212
|
+
},
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* "May I come in." This used to be a raised palm, and it is now the same
|
|
216
|
+
* signal without one: the body claims a little space and the face asks. The
|
|
217
|
+
* hold is what carries it either way — the signal is the *hold*, and anything
|
|
218
|
+
* that goes up and straight back down reads as a twitch.
|
|
219
|
+
*
|
|
220
|
+
* Distinct from CLAIM_FLOOR by intent rather than by parts. CLAIM_FLOOR is a
|
|
221
|
+
* speaker taking a breath and is followed by audio; this is a listener asking
|
|
222
|
+
* and is followed by waiting. What separates them on screen is the head: this
|
|
223
|
+
* one comes *up* and stays up, holding eye contact, which is a question. The
|
|
224
|
+
* inbreath drops the chin, which is a preparation.
|
|
225
|
+
*/
|
|
226
|
+
RAISE_HAND: {
|
|
227
|
+
id: 'RAISE_HAND', label: 'may I come in', text: '', duration: 1600,
|
|
228
|
+
keys: {
|
|
229
|
+
browRaiseL: [[0, 0], [0.2, 0.44], [0.8, 0.38], [1, 0.14]],
|
|
230
|
+
browRaiseR: [[0, 0], [0.2, 0.40], [0.8, 0.34], [1, 0.12]],
|
|
231
|
+
browInnerL: [[0, 0], [0.24, 0.20], [0.8, 0.16], [1, 0]],
|
|
232
|
+
browInnerR: [[0, 0], [0.24, 0.20], [0.8, 0.16], [1, 0]],
|
|
233
|
+
shoulderL: [[0, 0], [0.25, 0.36], [0.8, 0.30], [1, 0.10]],
|
|
234
|
+
shoulderR: [[0, 0], [0.25, 0.32], [0.8, 0.26], [1, 0.08]],
|
|
235
|
+
torsoLean: [[0, 0], [0.3, 0.30], [0.8, 0.24], [1, 0.08]],
|
|
236
|
+
// Chin up and held, not a nod. Held is the whole gesture.
|
|
237
|
+
headPitch: [[0, 0], [0.3, -0.26], [0.85, -0.22], [1, -0.06]],
|
|
238
|
+
lidL: [[0, 0], [0.3, -0.12], [0.85, -0.10], [1, 0]],
|
|
239
|
+
lidR: [[0, 0], [0.3, -0.12], [0.85, -0.10], [1, 0]],
|
|
240
|
+
mouthOpen: [[0, 0], [0.3, 0.18], [0.85, 0.14], [1, 0]],
|
|
241
|
+
},
|
|
242
|
+
},
|
|
243
|
+
|
|
244
|
+
// ---------------------------------------------------------------------
|
|
245
|
+
// Gestures that were arm gestures.
|
|
246
|
+
//
|
|
247
|
+
// The rig had a forearm and a hand; they were removed (see the note in
|
|
248
|
+
// params.js). These four kept their IDs, because the ID set is a wire
|
|
249
|
+
// contract the server targets and dropping entries from it is a breaking
|
|
250
|
+
// change for a reason the server has no way to know about. What each one
|
|
251
|
+
// means is unchanged; only the body part saying it moved.
|
|
252
|
+
//
|
|
253
|
+
// Re-authoring a gesture off the face is not the same as deleting a track and
|
|
254
|
+
// keeping the rest. A wave whose arm is gone is not a quieter wave, it is a
|
|
255
|
+
// smile with a stray head-roll on it. Each of these had to be rebuilt around
|
|
256
|
+
// whatever channel could carry the meaning on its own.
|
|
257
|
+
// ---------------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Greeting. What a face does when an arm is not available is the eyebrow
|
|
261
|
+
* flash: a fast, high brow raise held under a fifth of a second, which is a
|
|
262
|
+
* near-universal human greeting display and is *the* recognition signal at
|
|
263
|
+
* conversational distance. It has to be fast — the difference between a
|
|
264
|
+
* greeting and surprise is almost entirely duration — so the peak lands at
|
|
265
|
+
* 11% of 1300ms and is off again by 40%.
|
|
266
|
+
*
|
|
267
|
+
* The smile outlasts the brows by design, and the head tilt outlasts both.
|
|
268
|
+
* That ordering is the gesture: brows recognise, mouth greets, head settles
|
|
269
|
+
* into listening. All three rising and falling together reads as a single
|
|
270
|
+
* twitch instead of a greeting with a shape to it.
|
|
271
|
+
*
|
|
272
|
+
* Shorter than the wave it replaces (1300 vs 2200) because there is no
|
|
273
|
+
* oscillation left to fill the time; a held smile past about a second stops
|
|
274
|
+
* being a greeting and becomes an expression.
|
|
275
|
+
*/
|
|
276
|
+
WAVE: {
|
|
277
|
+
id: 'WAVE', label: 'greet (brow flash)', text: '', duration: 1300,
|
|
278
|
+
keys: {
|
|
279
|
+
browRaiseL: [[0, 0], [0.11, 0.85], [0.4, 0.22], [0.75, 0.12], [1, 0]],
|
|
280
|
+
browRaiseR: [[0, 0], [0.11, 0.80], [0.4, 0.20], [0.75, 0.10], [1, 0]],
|
|
281
|
+
lidL: [[0, 0], [0.13, -0.20], [0.45, -0.06], [1, 0]],
|
|
282
|
+
lidR: [[0, 0], [0.13, -0.20], [0.45, -0.06], [1, 0]],
|
|
283
|
+
mouthCornerL: [[0, 0], [0.26, 0.62], [0.72, 0.50], [1, 0]],
|
|
284
|
+
mouthCornerR: [[0, 0], [0.26, 0.62], [0.72, 0.50], [1, 0]],
|
|
285
|
+
// The squint is what makes it a real smile rather than a social one, and
|
|
286
|
+
// it lags the corners on purpose: an orbicularis contraction that arrives
|
|
287
|
+
// with the mouth reads as posed, one that arrives just after reads as felt.
|
|
288
|
+
squintL: [[0, 0], [0.34, 0.38], [0.78, 0.32], [1, 0]],
|
|
289
|
+
squintR: [[0, 0], [0.34, 0.38], [0.78, 0.32], [1, 0]],
|
|
290
|
+
// Chin up on the recognition, then a small settle. A greeting head goes
|
|
291
|
+
// up first; only an apology starts by going down.
|
|
292
|
+
headPitch: [[0, 0], [0.16, -0.24], [0.5, 0.06], [1, 0]],
|
|
293
|
+
headRoll: [[0, 0], [0.42, -0.20], [0.85, -0.14], [1, 0]],
|
|
294
|
+
},
|
|
295
|
+
blinkAt: [0.14],
|
|
296
|
+
},
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Approval. The thumb is gone, so the nod carries it — and a nod that means
|
|
300
|
+
* "good" is not the nod that means "yes". It is slower, deeper, and the head
|
|
301
|
+
* comes back up to level rather than overshooting, because the rebound is
|
|
302
|
+
* what makes a nod read as assent to a question. Here there is no question.
|
|
303
|
+
*
|
|
304
|
+
* Deep enough to be unmistakable: peak 0.62 against NOD_SMALL's 0.52, over
|
|
305
|
+
* nearly three times the duration.
|
|
306
|
+
*/
|
|
307
|
+
THUMBS_UP: {
|
|
308
|
+
id: 'THUMBS_UP', label: 'approve', text: '', duration: 1500,
|
|
309
|
+
keys: {
|
|
310
|
+
headPitch: [[0, 0], [0.3, 0.62], [0.58, 0.20], [0.78, 0.34], [1, 0]],
|
|
311
|
+
mouthCornerL: [[0, 0], [0.36, 0.58], [0.8, 0.48], [1, 0]],
|
|
312
|
+
mouthCornerR: [[0, 0], [0.36, 0.58], [0.8, 0.48], [1, 0]],
|
|
313
|
+
squintL: [[0, 0], [0.45, 0.40], [0.8, 0.34], [1, 0]],
|
|
314
|
+
squintR: [[0, 0], [0.45, 0.40], [0.8, 0.34], [1, 0]],
|
|
315
|
+
browRaiseL: [[0, 0], [0.24, 0.30], [0.7, 0.10], [1, 0]],
|
|
316
|
+
browRaiseR: [[0, 0], [0.24, 0.28], [0.7, 0.10], [1, 0]],
|
|
317
|
+
// Eyes close a little further than the squint alone would take them. A
|
|
318
|
+
// wholehearted approval is a slightly shut-eyed expression; wide eyes
|
|
319
|
+
// with a big smile is delight, which is a different and stranger thing
|
|
320
|
+
// for an attentive agent to be doing.
|
|
321
|
+
lidL: [[0, 0], [0.4, 0.14], [0.8, 0.10], [1, 0]],
|
|
322
|
+
lidR: [[0, 0], [0.4, 0.14], [0.8, 0.10], [1, 0]],
|
|
323
|
+
},
|
|
324
|
+
},
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* The shoulders now carry this alone, so they are pushed to the top of their
|
|
328
|
+
* range and held. The old comment here noted that shoulders are legible only
|
|
329
|
+
* in profile and that face-on the hands carried it; that is true of a small
|
|
330
|
+
* shrug and false of a large one. The fix is amplitude plus the face — brows
|
|
331
|
+
* up hard, mouth corners pulled *down* rather than up. Raised shoulders with
|
|
332
|
+
* a neutral mouth is a flinch; it is the inverted mouth that makes the same
|
|
333
|
+
* shoulders mean "I don't know".
|
|
334
|
+
*/
|
|
335
|
+
SHRUG: {
|
|
336
|
+
id: 'SHRUG', label: 'shrug', text: '', duration: 1250,
|
|
337
|
+
keys: {
|
|
338
|
+
shoulderL: [[0, 0], [0.26, 1], [0.66, 0.95], [1, 0]],
|
|
339
|
+
shoulderR: [[0, 0], [0.26, 1], [0.66, 0.95], [1, 0]],
|
|
340
|
+
browRaiseL: [[0, 0], [0.24, 0.62], [0.66, 0.56], [1, 0]],
|
|
341
|
+
browRaiseR: [[0, 0], [0.24, 0.60], [0.66, 0.54], [1, 0]],
|
|
342
|
+
browInnerL: [[0, 0], [0.28, 0.24], [0.7, 0.20], [1, 0]],
|
|
343
|
+
browInnerR: [[0, 0], [0.28, 0.24], [0.7, 0.20], [1, 0]],
|
|
344
|
+
headRoll: [[0, 0], [0.3, 0.18], [0.7, 0.14], [1, 0]],
|
|
345
|
+
// The chin tucks with the shoulders — the head sinks a little between
|
|
346
|
+
// them, which is most of what a shrug looks like from the front.
|
|
347
|
+
headPitch: [[0, 0], [0.28, 0.18], [0.7, 0.14], [1, 0]],
|
|
348
|
+
mouthCornerL: [[0, 0], [0.4, -0.30], [0.7, -0.26], [1, 0]],
|
|
349
|
+
mouthCornerR: [[0, 0], [0.4, -0.26], [0.7, -0.22], [1, 0]],
|
|
350
|
+
mouthPress: [[0, 0], [0.4, 0.30], [0.7, 0.26], [1, 0]],
|
|
351
|
+
},
|
|
352
|
+
},
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* "Keep going" — the emphatic one, as against the routine GO_ON backchannel.
|
|
356
|
+
* The beckon is gone; what is left is the part of "go on" that was never in
|
|
357
|
+
* the arm anyway. Eyes widen, brows go up and *stay* up, and the head tilts
|
|
358
|
+
* and holds. The hold is the message: a face that returns to neutral has
|
|
359
|
+
* stopped waiting, and this clip's whole job is to say the floor is still
|
|
360
|
+
* yours.
|
|
361
|
+
*
|
|
362
|
+
* Two head beats rather than one continuous lift, which is what a listener
|
|
363
|
+
* urging someone on actually does — the second beat is the "and?".
|
|
364
|
+
*/
|
|
365
|
+
GO_ON_ARM: {
|
|
366
|
+
id: 'GO_ON_ARM', label: 'go on (emphatic)', text: '', duration: 1400,
|
|
367
|
+
keys: {
|
|
368
|
+
browRaiseL: [[0, 0], [0.18, 0.58], [0.75, 0.44], [1, 0]],
|
|
369
|
+
browRaiseR: [[0, 0], [0.18, 0.55], [0.75, 0.42], [1, 0]],
|
|
370
|
+
lidL: [[0, 0], [0.22, -0.26], [0.8, -0.20], [1, 0]],
|
|
371
|
+
lidR: [[0, 0], [0.22, -0.26], [0.8, -0.20], [1, 0]],
|
|
372
|
+
headPitch: [[0, 0], [0.2, 0.26], [0.42, -0.04], [0.6, 0.20], [1, 0]],
|
|
373
|
+
headRoll: [[0, 0], [0.35, 0.16], [0.8, 0.12], [1, 0]],
|
|
374
|
+
torsoLean: [[0, 0], [0.3, 0.22], [0.8, 0.18], [1, 0]],
|
|
375
|
+
mouthCornerL: [[0, 0], [0.5, 0.26], [1, 0]],
|
|
376
|
+
mouthCornerR: [[0, 0], [0.5, 0.26], [1, 0]],
|
|
377
|
+
mouthOpen: [[0, 0], [0.35, 0.12], [0.8, 0.10], [1, 0]],
|
|
378
|
+
},
|
|
379
|
+
},
|
|
380
|
+
|
|
381
|
+
// ---------------------------------------------------------------------
|
|
382
|
+
// Spoken interjections.
|
|
383
|
+
// ---------------------------------------------------------------------
|
|
384
|
+
MM_HMM: {
|
|
385
|
+
id: 'MM_HMM', label: 'mm-hmm', text: 'mm-hmm', duration: 820,
|
|
386
|
+
// Lips stay shut the whole way; the meaning is entirely in the nod.
|
|
387
|
+
mouthCues: [{ t: 0, v: 'A' }, { t: 620, v: 'X' }],
|
|
388
|
+
keys: {
|
|
389
|
+
headPitch: [[0, 0], [0.15, 0.34], [0.33, 0.06], [0.50, 0.28], [0.72, 0.02], [1, 0]],
|
|
390
|
+
mouthCornerL: [[0, 0], [0.4, 0.16], [1, 0]],
|
|
391
|
+
mouthCornerR: [[0, 0], [0.4, 0.16], [1, 0]],
|
|
392
|
+
},
|
|
393
|
+
},
|
|
394
|
+
OKAY: {
|
|
395
|
+
id: 'OKAY', label: 'okay', text: 'okay', duration: 860,
|
|
396
|
+
// oʊ · k · eɪ
|
|
397
|
+
mouthCues: [
|
|
398
|
+
{ t: 0, v: 'F' }, { t: 140, v: 'B' }, { t: 235, v: 'C' },
|
|
399
|
+
{ t: 350, v: 'B' }, { t: 480, v: 'X' },
|
|
400
|
+
],
|
|
401
|
+
keys: {
|
|
402
|
+
headPitch: nod(0.42, 0.22, 0.52),
|
|
403
|
+
browRaiseL: [[0, 0], [0.18, 0.16], [0.6, 0], [1, 0]],
|
|
404
|
+
browRaiseR: [[0, 0], [0.18, 0.16], [0.6, 0], [1, 0]],
|
|
405
|
+
mouthCornerL: [[0, 0], [0.55, 0.20], [1, 0]],
|
|
406
|
+
mouthCornerR: [[0, 0], [0.55, 0.20], [1, 0]],
|
|
407
|
+
},
|
|
408
|
+
},
|
|
409
|
+
YES: {
|
|
410
|
+
id: 'YES', label: 'yes', text: 'yes', duration: 740,
|
|
411
|
+
// j · ɛ · s
|
|
412
|
+
mouthCues: [{ t: 0, v: 'B' }, { t: 70, v: 'C' }, { t: 195, v: 'B' }, { t: 330, v: 'X' }],
|
|
413
|
+
keys: {
|
|
414
|
+
headPitch: [[0, 0], [0.15, 0.55], [0.40, -0.14], [0.68, 0.08], [1, 0]],
|
|
415
|
+
browRaiseL: [[0, 0], [0.13, 0.26], [0.5, 0.04], [1, 0]],
|
|
416
|
+
browRaiseR: [[0, 0], [0.13, 0.26], [0.5, 0.04], [1, 0]],
|
|
417
|
+
mouthCornerL: [[0, 0], [0.5, 0.30], [1, 0]],
|
|
418
|
+
mouthCornerR: [[0, 0], [0.5, 0.30], [1, 0]],
|
|
419
|
+
},
|
|
420
|
+
},
|
|
421
|
+
SURE: {
|
|
422
|
+
id: 'SURE', label: 'sure', text: 'sure', duration: 860,
|
|
423
|
+
// ʃ · ʊ · r
|
|
424
|
+
mouthCues: [{ t: 0, v: 'B' }, { t: 85, v: 'F' }, { t: 200, v: 'E' }, { t: 350, v: 'X' }],
|
|
425
|
+
keys: {
|
|
426
|
+
headPitch: nod(0.34, 0.26, 0.6),
|
|
427
|
+
mouthCornerL: [[0, 0], [0.45, 0.42], [1, 0]],
|
|
428
|
+
mouthCornerR: [[0, 0], [0.45, 0.42], [1, 0]],
|
|
429
|
+
squintL: [[0, 0], [0.45, 0.20], [1, 0]],
|
|
430
|
+
squintR: [[0, 0], [0.45, 0.20], [1, 0]],
|
|
431
|
+
browRaiseL: [[0, 0], [0.2, 0.14], [1, 0]],
|
|
432
|
+
browRaiseR: [[0, 0], [0.2, 0.14], [1, 0]],
|
|
433
|
+
},
|
|
434
|
+
},
|
|
435
|
+
I_SEE: {
|
|
436
|
+
id: 'I_SEE', label: 'I see', text: 'I see', duration: 1050,
|
|
437
|
+
// aɪ · s · iː
|
|
438
|
+
mouthCues: [{ t: 0, v: 'D' }, { t: 120, v: 'C' }, { t: 215, v: 'B' }, { t: 450, v: 'X' }],
|
|
439
|
+
keys: {
|
|
440
|
+
headPitch: [[0, 0], [0.28, 0.44], [0.66, 0.02], [1, 0]],
|
|
441
|
+
browRaiseL: [[0, 0], [0.18, 0.32], [0.55, 0.05], [1, 0]],
|
|
442
|
+
browRaiseR: [[0, 0], [0.18, 0.30], [0.55, 0.05], [1, 0]],
|
|
443
|
+
},
|
|
444
|
+
blinkAt: [0.34],
|
|
445
|
+
},
|
|
446
|
+
RIGHT: {
|
|
447
|
+
id: 'RIGHT', label: 'right', text: 'right', duration: 740,
|
|
448
|
+
// r · aɪ · t
|
|
449
|
+
mouthCues: [{ t: 0, v: 'E' }, { t: 75, v: 'D' }, { t: 200, v: 'B' }, { t: 330, v: 'X' }],
|
|
450
|
+
keys: {
|
|
451
|
+
headPitch: nod(0.42, 0.20, 0.5),
|
|
452
|
+
browRaiseL: [[0, 0], [0.16, 0.20], [1, 0]],
|
|
453
|
+
browRaiseR: [[0, 0], [0.16, 0.20], [1, 0]],
|
|
454
|
+
},
|
|
455
|
+
},
|
|
456
|
+
GO_ON: {
|
|
457
|
+
id: 'GO_ON', label: 'go on', text: 'go on', duration: 820,
|
|
458
|
+
// g · oʊ · ɒ · n
|
|
459
|
+
mouthCues: [{ t: 0, v: 'B' }, { t: 80, v: 'E' }, { t: 290, v: 'A' }, { t: 420, v: 'X' }],
|
|
460
|
+
keys: {
|
|
461
|
+
headPitch: nod(0.22, 0.25, 0.6),
|
|
462
|
+
browRaiseL: [[0, 0], [0.2, 0.36], [0.65, 0.14], [1, 0]],
|
|
463
|
+
browRaiseR: [[0, 0], [0.2, 0.36], [0.65, 0.14], [1, 0]],
|
|
464
|
+
// Eyes widen — the "keep talking, I'm with you" signal.
|
|
465
|
+
lidL: [[0, 0], [0.25, -0.14], [0.8, -0.08], [1, 0]],
|
|
466
|
+
lidR: [[0, 0], [0.25, -0.14], [0.8, -0.08], [1, 0]],
|
|
467
|
+
mouthCornerL: [[0, 0], [0.5, 0.18], [1, 0]],
|
|
468
|
+
mouthCornerR: [[0, 0], [0.5, 0.18], [1, 0]],
|
|
469
|
+
// A hint of the lean, deliberately much smaller than GO_ON_ARM. This clip
|
|
470
|
+
// fires as routine backchannel, and a full-sized gesture every time the
|
|
471
|
+
// user pauses is precisely the fidget that makes an avatar read as a
|
|
472
|
+
// gimmick rather than as a listener.
|
|
473
|
+
torsoLean: [[0, 0], [0.25, 0.10], [0.7, 0.08], [1, 0]],
|
|
474
|
+
},
|
|
475
|
+
},
|
|
476
|
+
ONE_MOMENT: {
|
|
477
|
+
id: 'ONE_MOMENT', label: 'one moment', text: 'one moment', duration: 1350,
|
|
478
|
+
// w · ʌ · n · m · oʊ · m · ə · n · t
|
|
479
|
+
mouthCues: [
|
|
480
|
+
{ t: 0, v: 'F' }, { t: 110, v: 'C' }, { t: 210, v: 'A' }, { t: 320, v: 'F' },
|
|
481
|
+
{ t: 420, v: 'C' }, { t: 510, v: 'A' }, { t: 590, v: 'B' }, { t: 730, v: 'X' },
|
|
482
|
+
],
|
|
483
|
+
// Breaking eye contact is what actually communicates "hold on".
|
|
484
|
+
gaze: 'AWAY_RIGHT',
|
|
485
|
+
keys: {
|
|
486
|
+
headRoll: [[0, 0], [0.3, 0.12], [0.8, 0.09], [1, 0]],
|
|
487
|
+
headYaw: [[0, 0], [0.35, 0.13], [1, 0]],
|
|
488
|
+
browRaiseL: [[0, 0], [0.2, 0.34], [0.7, 0.18], [1, 0]],
|
|
489
|
+
browRaiseR: [[0, 0], [0.2, 0.30], [0.7, 0.16], [1, 0]],
|
|
490
|
+
// This used to be a raised index finger, and that was the one gesture in
|
|
491
|
+
// the set a mime did better than the words: legible muted, at thumbnail
|
|
492
|
+
// size, across a language barrier. Nothing on the face replaces it, so no
|
|
493
|
+
// attempt is made to — what carries the clip now is the gaze break above,
|
|
494
|
+
// which is what actually communicates "hold on" and always was. The face
|
|
495
|
+
// says "wait" by looking away; it never said it with the finger.
|
|
496
|
+
},
|
|
497
|
+
blinkAt: [0.12],
|
|
498
|
+
},
|
|
499
|
+
SORRY: {
|
|
500
|
+
id: 'SORRY', label: 'sorry', text: 'sorry', duration: 1050,
|
|
501
|
+
// s · ɒ · r · i
|
|
502
|
+
mouthCues: [{ t: 0, v: 'B' }, { t: 80, v: 'E' }, { t: 260, v: 'B' }, { t: 420, v: 'X' }],
|
|
503
|
+
keys: {
|
|
504
|
+
// AU1: the inner-brow lift. This single channel is the whole apology.
|
|
505
|
+
browInnerL: [[0, 0], [0.25, 0.62], [0.75, 0.40], [1, 0]],
|
|
506
|
+
browInnerR: [[0, 0], [0.25, 0.62], [0.75, 0.40], [1, 0]],
|
|
507
|
+
browRaiseL: [[0, 0], [0.3, -0.06], [1, 0]],
|
|
508
|
+
browRaiseR: [[0, 0], [0.3, -0.06], [1, 0]],
|
|
509
|
+
headPitch: [[0, 0], [0.32, 0.28], [0.8, 0.14], [1, 0]],
|
|
510
|
+
headRoll: [[0, 0], [0.35, 0.13], [1, 0]],
|
|
511
|
+
mouthCornerL: [[0, 0], [0.4, -0.18], [1, 0]],
|
|
512
|
+
mouthCornerR: [[0, 0], [0.4, -0.18], [1, 0]],
|
|
513
|
+
},
|
|
514
|
+
},
|
|
515
|
+
HMM: {
|
|
516
|
+
id: 'HMM', label: 'hmm', text: 'hmm', duration: 1250,
|
|
517
|
+
mouthCues: [{ t: 0, v: 'A' }, { t: 900, v: 'X' }],
|
|
518
|
+
gaze: 'AWAY_THINKING',
|
|
519
|
+
keys: {
|
|
520
|
+
browRaiseL: [[0, 0], [0.2, -0.24], [0.75, -0.18], [1, 0]],
|
|
521
|
+
browRaiseR: [[0, 0], [0.2, -0.18], [0.75, -0.14], [1, 0]],
|
|
522
|
+
browInnerL: [[0, 0], [0.25, 0.30], [0.8, 0.20], [1, 0]],
|
|
523
|
+
browInnerR: [[0, 0], [0.25, 0.30], [0.8, 0.20], [1, 0]],
|
|
524
|
+
headRoll: [[0, 0], [0.4, 0.10], [1, 0]],
|
|
525
|
+
mouthCornerL: [[0, 0], [0.4, -0.10], [1, 0]],
|
|
526
|
+
mouthCornerR: [[0, 0], [0.4, 0.04], [1, 0]],
|
|
527
|
+
},
|
|
528
|
+
},
|
|
529
|
+
GOT_IT: {
|
|
530
|
+
id: 'GOT_IT', label: 'got it', text: 'got it', duration: 820,
|
|
531
|
+
// g · ɒ · t · ɪ · t
|
|
532
|
+
mouthCues: [{ t: 0, v: 'B' }, { t: 80, v: 'D' }, { t: 210, v: 'B' }, { t: 400, v: 'X' }],
|
|
533
|
+
keys: {
|
|
534
|
+
headPitch: [[0, 0], [0.16, 0.48], [0.44, -0.10], [1, 0]],
|
|
535
|
+
browRaiseL: [[0, 0], [0.15, 0.22], [1, 0]],
|
|
536
|
+
browRaiseR: [[0, 0], [0.15, 0.22], [1, 0]],
|
|
537
|
+
mouthCornerL: [[0, 0], [0.5, 0.24], [1, 0]],
|
|
538
|
+
mouthCornerR: [[0, 0], [0.5, 0.24], [1, 0]],
|
|
539
|
+
},
|
|
540
|
+
},
|
|
541
|
+
TAKE_YOUR_TIME: {
|
|
542
|
+
id: 'TAKE_YOUR_TIME', label: 'take your time', text: 'take your time', duration: 1500,
|
|
543
|
+
mouthCues: [
|
|
544
|
+
{ t: 0, v: 'B' }, { t: 90, v: 'C' }, { t: 200, v: 'B' }, { t: 310, v: 'F' },
|
|
545
|
+
{ t: 430, v: 'E' }, { t: 560, v: 'A' }, { t: 650, v: 'D' }, { t: 800, v: 'A' },
|
|
546
|
+
{ t: 900, v: 'X' },
|
|
547
|
+
],
|
|
548
|
+
keys: {
|
|
549
|
+
headPitch: [[0, 0], [0.2, 0.22], [0.45, 0.02], [0.62, 0.16], [1, 0]],
|
|
550
|
+
mouthCornerL: [[0, 0], [0.5, 0.40], [1, 0]],
|
|
551
|
+
mouthCornerR: [[0, 0], [0.5, 0.40], [1, 0]],
|
|
552
|
+
squintL: [[0, 0], [0.5, 0.24], [1, 0]],
|
|
553
|
+
squintR: [[0, 0], [0.5, 0.24], [1, 0]],
|
|
554
|
+
browRaiseL: [[0, 0], [0.25, 0.16], [1, 0]],
|
|
555
|
+
browRaiseR: [[0, 0], [0.25, 0.16], [1, 0]],
|
|
556
|
+
},
|
|
557
|
+
},
|
|
558
|
+
};
|
|
559
|
+
|
|
560
|
+
export const INTERJECTION_IDS = Object.keys(INTERJECTIONS);
|
|
561
|
+
|
|
562
|
+
/** Clips that carry words, vs. wordless backchannel. */
|
|
563
|
+
export const SPOKEN_IDS = INTERJECTION_IDS.filter((k) => INTERJECTIONS[k].text);
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Attach audio to a clip. Call once at boot with your own TTS renders:
|
|
567
|
+
* attachAudio('OKAY', '/audio/agent-okay.mp3')
|
|
568
|
+
* The baked viseme track is then scheduled against that file's clock instead of
|
|
569
|
+
* the local timer, so any timing drift resolves in the audio's favour.
|
|
570
|
+
*/
|
|
571
|
+
export function attachAudio(id, url) {
|
|
572
|
+
const clip = INTERJECTIONS[id];
|
|
573
|
+
if (!clip) throw new Error(`unknown interjection: ${id}`);
|
|
574
|
+
const el = new Audio(url);
|
|
575
|
+
el.preload = 'auto';
|
|
576
|
+
clip.audioEl = el;
|
|
577
|
+
return el;
|
|
578
|
+
}
|