@voqalize/avatar 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -154
- package/dist/index.d.ts +37 -26
- package/dist/index.js +48 -25
- package/dist/pin.d.ts +2 -0
- package/dist/pin.js +3 -0
- package/dist/react.d.ts +21 -6
- package/dist/react.js +24 -6
- package/package.json +8 -67
- package/LICENSE-APACHE-2.0 +0 -201
- package/LICENSE-CC-BY-4.0 +0 -396
- package/assets/README.md +0 -60
- package/assets/tanvi.glb +0 -0
- package/assets/tanya.glb +0 -0
- package/assets/tara.glb +0 -0
- package/assets/tess.glb +0 -0
- package/assets/tushar.glb +0 -0
- package/client/Avatar.tsx +0 -57
- package/client/AvatarClient.ts +0 -818
- package/client/createAvatar.ts +0 -177
- package/client/index.ts +0 -44
- package/client/internal.ts +0 -132
- package/client/playout.ts +0 -95
- package/client/react.ts +0 -13
- package/client/supports.ts +0 -27
- package/client/three/assets.ts +0 -26
- package/client/three/budgets.ts +0 -27
- package/client/three/character-rig.ts +0 -1713
- package/client/three/createCharacter.ts +0 -109
- package/client/three/holds.ts +0 -69
- package/client/three/internal.ts +0 -55
- package/client/three/motion-limits.json +0 -124
- package/client/three/sequences.ts +0 -281
- package/client/three/tanvi-asset.ts +0 -6
- package/client/three/tanvi.ts +0 -30
- package/client/three/tanya-asset.ts +0 -6
- package/client/three/tanya.ts +0 -29
- package/client/three/tara-asset.ts +0 -25
- package/client/three/tara.ts +0 -36
- package/client/three/tess-asset.ts +0 -6
- package/client/three/tess.ts +0 -30
- package/client/three/tushar-asset.ts +0 -6
- package/client/three/tushar.ts +0 -28
- package/client/types.ts +0 -235
- package/client/useAvatar.ts +0 -74
- package/dist/Avatar.d.ts +0 -44
- package/dist/Avatar.d.ts.map +0 -1
- package/dist/Avatar.js +0 -7
- package/dist/Avatar.js.map +0 -1
- package/dist/AvatarClient.d.ts +0 -274
- package/dist/AvatarClient.d.ts.map +0 -1
- package/dist/AvatarClient.js +0 -712
- package/dist/AvatarClient.js.map +0 -1
- package/dist/createAvatar.d.ts +0 -135
- package/dist/createAvatar.d.ts.map +0 -1
- package/dist/createAvatar.js +0 -75
- package/dist/createAvatar.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/internal.d.ts +0 -34
- package/dist/internal.d.ts.map +0 -1
- package/dist/internal.js +0 -56
- package/dist/internal.js.map +0 -1
- package/dist/playout.d.ts +0 -42
- package/dist/playout.d.ts.map +0 -1
- package/dist/playout.js +0 -85
- package/dist/playout.js.map +0 -1
- package/dist/react.d.ts.map +0 -1
- package/dist/react.js.map +0 -1
- package/dist/supports.d.ts +0 -19
- package/dist/supports.d.ts.map +0 -1
- package/dist/supports.js +0 -25
- package/dist/supports.js.map +0 -1
- package/dist/three/assets.d.ts +0 -10
- package/dist/three/assets.d.ts.map +0 -1
- package/dist/three/assets.js +0 -23
- package/dist/three/assets.js.map +0 -1
- package/dist/three/budgets.d.ts +0 -17
- package/dist/three/budgets.d.ts.map +0 -1
- package/dist/three/budgets.js +0 -21
- package/dist/three/budgets.js.map +0 -1
- package/dist/three/character-rig.d.ts +0 -366
- package/dist/three/character-rig.d.ts.map +0 -1
- package/dist/three/character-rig.js +0 -1611
- package/dist/three/character-rig.js.map +0 -1
- package/dist/three/createCharacter.d.ts +0 -60
- package/dist/three/createCharacter.d.ts.map +0 -1
- package/dist/three/createCharacter.js +0 -84
- package/dist/three/createCharacter.js.map +0 -1
- package/dist/three/holds.d.ts +0 -38
- package/dist/three/holds.d.ts.map +0 -1
- package/dist/three/holds.js +0 -61
- package/dist/three/holds.js.map +0 -1
- package/dist/three/internal.d.ts +0 -52
- package/dist/three/internal.d.ts.map +0 -1
- package/dist/three/internal.js +0 -49
- package/dist/three/internal.js.map +0 -1
- package/dist/three/motion-limits.json +0 -124
- package/dist/three/sequences.d.ts +0 -71
- package/dist/three/sequences.d.ts.map +0 -1
- package/dist/three/sequences.js +0 -262
- package/dist/three/sequences.js.map +0 -1
- package/dist/three/tanvi-asset.d.ts +0 -7
- package/dist/three/tanvi-asset.d.ts.map +0 -1
- package/dist/three/tanvi-asset.js +0 -7
- package/dist/three/tanvi-asset.js.map +0 -1
- package/dist/three/tanvi.d.ts +0 -24
- package/dist/three/tanvi.d.ts.map +0 -1
- package/dist/three/tanvi.js +0 -24
- package/dist/three/tanvi.js.map +0 -1
- package/dist/three/tanya-asset.d.ts +0 -7
- package/dist/three/tanya-asset.d.ts.map +0 -1
- package/dist/three/tanya-asset.js +0 -7
- package/dist/three/tanya-asset.js.map +0 -1
- package/dist/three/tanya.d.ts +0 -23
- package/dist/three/tanya.d.ts.map +0 -1
- package/dist/three/tanya.js +0 -23
- package/dist/three/tanya.js.map +0 -1
- package/dist/three/tara-asset.d.ts +0 -26
- package/dist/three/tara-asset.d.ts.map +0 -1
- package/dist/three/tara-asset.js +0 -26
- package/dist/three/tara-asset.js.map +0 -1
- package/dist/three/tara.d.ts +0 -30
- package/dist/three/tara.d.ts.map +0 -1
- package/dist/three/tara.js +0 -30
- package/dist/three/tara.js.map +0 -1
- package/dist/three/tess-asset.d.ts +0 -7
- package/dist/three/tess-asset.d.ts.map +0 -1
- package/dist/three/tess-asset.js +0 -7
- package/dist/three/tess-asset.js.map +0 -1
- package/dist/three/tess.d.ts +0 -24
- package/dist/three/tess.d.ts.map +0 -1
- package/dist/three/tess.js +0 -24
- package/dist/three/tess.js.map +0 -1
- package/dist/three/tushar-asset.d.ts +0 -7
- package/dist/three/tushar-asset.d.ts.map +0 -1
- package/dist/three/tushar-asset.js +0 -7
- package/dist/three/tushar-asset.js.map +0 -1
- package/dist/three/tushar.d.ts +0 -22
- package/dist/three/tushar.d.ts.map +0 -1
- package/dist/three/tushar.js +0 -22
- package/dist/three/tushar.js.map +0 -1
- package/dist/types.d.ts +0 -149
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -140
- package/dist/types.js.map +0 -1
- package/dist/useAvatar.d.ts +0 -39
- package/dist/useAvatar.d.ts.map +0 -1
- package/dist/useAvatar.js +0 -35
- package/dist/useAvatar.js.map +0 -1
- package/src/avatar.d.ts +0 -436
- package/src/avatar.js +0 -1370
- package/src/behavior.d.ts +0 -42
- package/src/behavior.js +0 -114
- package/src/camera.js +0 -29
- package/src/clips.js +0 -155
- package/src/conformance.js +0 -119
- package/src/emotions.js +0 -58
- package/src/face-core.js +0 -180
- package/src/face-myna.d.ts +0 -7
- package/src/face-myna.js +0 -786
- package/src/face-peep-control-plane.js +0 -167
- package/src/face-peep.d.ts +0 -7
- package/src/face-peep.js +0 -925
- package/src/face-wren.d.ts +0 -7
- package/src/face-wren.js +0 -468
- package/src/faces.d.ts +0 -12
- package/src/faces.js +0 -53
- package/src/gaze.js +0 -504
- package/src/hand.js +0 -709
- package/src/head.js +0 -134
- package/src/idle.js +0 -668
- package/src/interjections.js +0 -747
- package/src/line-art.js +0 -137
- package/src/params.js +0 -180
- package/src/perform.js +0 -107
- package/src/prosody.js +0 -741
- package/src/rig.d.ts +0 -31
- package/src/rig.js +0 -32
- package/src/speech-timing.js +0 -32
- package/src/visemes.js +0 -410
package/src/prosody.js
DELETED
|
@@ -1,741 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Speech prosody: what a speaking face does *around* the mouth, read off the
|
|
3
|
-
* same viseme track that drives the mouth, so it can never disagree with it.
|
|
4
|
-
*
|
|
5
|
-
* A talking head is mostly moved by its voice. Head motion alone accounts for
|
|
6
|
-
* over 63 % of the variance in a speaker's F0 (Munhall et al. 2004), its
|
|
7
|
-
* first canonical correlation with prosody runs about 0.7 (Busso et al.
|
|
8
|
-
* 2007), and a face animated with the speaker's own natural head motion is
|
|
9
|
-
* understood better than the same face held still — or moved at double
|
|
10
|
-
* amplitude, which scored no better than none. That last result is the
|
|
11
|
-
* sizing rule for everything here: natural, not more. Research, with the
|
|
12
|
-
* numbers: docs/research-biomechanics.md §3.8.
|
|
13
|
-
*
|
|
14
|
-
* Each behaviour is here because a study found people use it:
|
|
15
|
-
*
|
|
16
|
-
* - A blink at each pause. Speakers' blinks cluster at breakpoints of speech,
|
|
17
|
-
* and listeners entrain to exactly those blinks (Nakano & Kitazawa 2010;
|
|
18
|
-
* research-biomechanics.md §3.7, §5.4). A blink placed at a clause boundary
|
|
19
|
-
* is worth several on a timer, so the idle timer is re-armed each time.
|
|
20
|
-
* - A pose held through each phrase and changed between them. Most of a
|
|
21
|
-
* speaker's head motion is not the beats but a pose held and changed at
|
|
22
|
-
* phrase scale (Hadar et al. 1983's slow band). The change comes *before*
|
|
23
|
-
* the voice — the head moves first at over 70 % of phrase onsets (Graf et
|
|
24
|
-
* al. 2002) — so it is booked inside the pause, with the inbreath: speech
|
|
25
|
-
* breathing is a quick inbreath at the pause and a long outbreath over the
|
|
26
|
-
* phrase (§6.1).
|
|
27
|
-
* - Beats on the prominent vowels, about one a second where the speech has
|
|
28
|
-
* them. Brows up and a head stroke, the brows leading the accented syllable
|
|
29
|
-
* by ~60 ms (Flecha-García 2010). Graf's speaker nodded on most pitch
|
|
30
|
-
* accents and swung to a new position on a fifth of them, mostly in pitch,
|
|
31
|
-
* sometimes diagonally with yaw, and repeated her own motions — so the side
|
|
32
|
-
* a diagonal leans to is kept for the turn. The gate is the point: brow
|
|
33
|
-
* raises placed at random give no benefit (Al Moubayed & Beskow 2009), so a
|
|
34
|
-
* vowel that does not stand out gets no beat at all.
|
|
35
|
-
* - A settle on the last vowel of a phrase: the head comes down a little as
|
|
36
|
-
* a phrase closes (final lowering, §3.4) and *stays* down through the
|
|
37
|
-
* pause, until the next phrase lifts it.
|
|
38
|
-
* - Warmth at the edges of a turn. A smile held through a sentence is
|
|
39
|
-
* discounted as insincere (research-perception.md §3), so this is an
|
|
40
|
-
* episode: it rises as a turn starts and again as it ends, and is gone in
|
|
41
|
-
* between. The mixer withholds it under an emotion whose corners are down.
|
|
42
|
-
*
|
|
43
|
-
* **The head moves by holding, and that is the 2026-09-12 redesign.** Every
|
|
44
|
-
* head behaviour above used to be a continuous envelope — a drift re-aimed each
|
|
45
|
-
* phrase, strokes with counter-lobes, a lead-in dip — summed and low-passed,
|
|
46
|
-
* which drew a head in motion from the first word to the last and read as
|
|
47
|
-
* floating. It is now discrete: the held pose moves on a minimum-jerk path and
|
|
48
|
-
* then is exactly still, and a beat is a pulse on top of it (head.js has the
|
|
49
|
-
* argument). The amplitudes came down with it, to what a webcam shows of a
|
|
50
|
-
* person talking — a few degrees — because they had been sized to the lab's
|
|
51
|
-
* whole-sentence ranges, and on tara, a photograph on shallow geometry, a
|
|
52
|
-
* turn that size is the moment she reads as a cutout.
|
|
53
|
-
*
|
|
54
|
-
* Prominence is estimated from what the cue stream carries. English stress is
|
|
55
|
-
* mostly duration and loudness, and a cue has both: its length, and `i`, which
|
|
56
|
-
* is TTS energy. That is a proxy for the pitch accent the studies measured, so
|
|
57
|
-
* everything here is sized to be missed rather than to be wrong. The one
|
|
58
|
-
* channel family it never writes is the mouth's articulation; lipsync is the
|
|
59
|
-
* headline, and the corners it lifts are the ones visemes never touch.
|
|
60
|
-
*
|
|
61
|
-
* These are speech-rhythm movements, not acknowledgements. A beat is part of
|
|
62
|
-
* how the avatar talks, like the torso moving more while it talks; a nod that
|
|
63
|
-
* *means* yes is still only ever a server action.
|
|
64
|
-
*/
|
|
65
|
-
|
|
66
|
-
import { SILENT } from './visemes.js';
|
|
67
|
-
import { HeadPose } from './head.js';
|
|
68
|
-
|
|
69
|
-
// A silence this long is a pause, not the gap between two words. Word gaps in
|
|
70
|
-
// fluent TTS run ~50-200 ms; clause pauses start around a quarter second.
|
|
71
|
-
const PAUSE_MS = 250;
|
|
72
|
-
// Speakers blink ~20-26 times a minute. Clause pauses in fast speech can come
|
|
73
|
-
// closer than that, and a blink at every one reads as nervous.
|
|
74
|
-
const BLINK_MIN_GAP_MS = 1400;
|
|
75
|
-
// Only fire on a pause we arrive at, not one the track was joined midway into.
|
|
76
|
-
const BLINK_FRESH_MS = 150;
|
|
77
|
-
|
|
78
|
-
// The held pose, in pose units at gain 1. What a unit is worth in degrees is
|
|
79
|
-
// the rig's own scaling and is deliberately not repeated here: the figure that
|
|
80
|
-
// used to be, was wrong for the whole of the day the yaw envelope moved.
|
|
81
|
-
// Each phrase picks one, usually on the other side from the last,
|
|
82
|
-
// so consecutive phrases are a visible change of position rather than a
|
|
83
|
-
// wobble about the same one. Yaw is the narrowest in degrees because it is the
|
|
84
|
-
// axis a projected photograph survives least; pitch leans slightly chin-up,
|
|
85
|
-
// which is where a person talking to someone holds their head.
|
|
86
|
-
// **Every axis went up on 2026-09-21, and it is paying for something
|
|
87
|
-
// specific.** The speaking face used to get much of its movement from a gaze
|
|
88
|
-
// aversion — a look away that took the neck with it — and that is gone
|
|
89
|
-
// (gaze.js): the eyes now hold the user for the whole turn. Taking it out and
|
|
90
|
-
// changing nothing else left the speaking head measurably *stiller* than
|
|
91
|
-
// before, which is the defect the aversion had been introduced to fix. So the
|
|
92
|
-
// movement comes back as the pose the head holds per phrase, which is what a
|
|
93
|
-
// speaker's head actually does and is communicative rather than evasive.
|
|
94
|
-
//
|
|
95
|
-
// Sized against `presence.test.ts`, which renders a turn through the real
|
|
96
|
-
// mixer and prints the degrees. On tara the widest yaw here is ~4.9° and the
|
|
97
|
-
// widest chin-up ~4.4°, inside the 6° and 5° she was measured at for a *held*
|
|
98
|
-
// pose; what the same table showed before the raise was a speaking head
|
|
99
|
-
// reaching 4.2° of yaw and 2.1° of pitch, the second of which is under half of
|
|
100
|
-
// what the aversion had been supplying. `headHold` softens the sum in any
|
|
101
|
-
// case, so a character measured tighter than tara gets her own angle here for
|
|
102
|
-
// free.
|
|
103
|
-
const POSE = {
|
|
104
|
-
headYaw: [0.15, 0.44],
|
|
105
|
-
// **Pitch widens downward only, and the chin-up end is pinned where it was.**
|
|
106
|
-
// Opening it to -0.26 with the rest measured as a speaking face reading
|
|
107
|
-
// *sleepy*: the lids follow the globe, and on a rig with a reflex a chin-up
|
|
108
|
-
// head means eyes rolled down to hold the user, which narrows the aperture
|
|
109
|
-
// for real. The audit's open-lid median went 0.149 -> 0.163 against a 0.15
|
|
110
|
-
// ceiling (research-perception.md § 6) on that change alone. So the chin-up
|
|
111
|
-
// end stays at the -0.20 it was measured safe at, and the extra travel comes
|
|
112
|
-
// off the other end, where the reflex rolls the eyes *up* and the lid opens.
|
|
113
|
-
// A phrase pose returns, so this is not the chin-drift that reads downcast.
|
|
114
|
-
headPitch: [-0.20, 0.22],
|
|
115
|
-
headRoll: [0.12, 0.34],
|
|
116
|
-
// The mouth corners take a small share of each pose: a speaking mouth whose
|
|
117
|
-
// corners never move reads as dubbed. They ease rather than move, since a
|
|
118
|
-
// corner that jumps reads as a twitch.
|
|
119
|
-
corner: [0, 0.09],
|
|
120
|
-
cornerTauS: 0.9,
|
|
121
|
-
// **The brows take a level per phrase too, and it is signed.** Measured over
|
|
122
|
-
// the audit's 56 s turn, `browRaiseL` sat at exactly 0.000 for 84% of frames
|
|
123
|
-
// and never once went negative: between beats the upper face was pinned at
|
|
124
|
-
// the floor of its range. Two video reviewers named that stillness — "the
|
|
125
|
-
// eyebrows, forehead, and the muscles around the eyes do not move" — as the
|
|
126
|
-
// single biggest reason these faces read as uncanny, ahead of the mouth. The
|
|
127
|
-
// corners above already had a held level and measured 0.1%, so the brows
|
|
128
|
-
// were the channel with no *pose*, not the channel with no events.
|
|
129
|
-
//
|
|
130
|
-
// Signed is the half that carries it. `browRaise` is the one channel the
|
|
131
|
-
// Blender rigs split by sign into two photographed maps — a raise above zero
|
|
132
|
-
// and a knit below it — so a phrase sitting negative is the only route a
|
|
133
|
-
// real furrow has into a sentence. Pinned at zero, half the asset was
|
|
134
|
-
// unreachable by construction. The floor is `thoughtful`'s own -0.14, which
|
|
135
|
-
// character-rig's knit weight already cites as a third of the crease; the ceiling
|
|
136
|
-
// stays well under a beat's 0.34 so a held level is never read as an accent.
|
|
137
|
-
brow: [-0.14, 0.18],
|
|
138
|
-
browTauS: 0.8,
|
|
139
|
-
// One brow leads, the way BEAT.browAmp's 0.34/0.30 already has it: a matched
|
|
140
|
-
// pair is the drawing, not the face.
|
|
141
|
-
browAsym: 0.82,
|
|
142
|
-
switchP: 0.7,
|
|
143
|
-
// A phrase with no pause long enough to book a move keeps its pose, unless
|
|
144
|
-
// the pose is this old: then the phrase takes a new one as it starts.
|
|
145
|
-
holdMs: 3200,
|
|
146
|
-
};
|
|
147
|
-
|
|
148
|
-
// The move into a new phrase's pose. It starts inside the pause and lands
|
|
149
|
-
// around the first syllable, and only a pause this long books one — a shorter
|
|
150
|
-
// one is a breath between clauses, not a new thought.
|
|
151
|
-
const ONSET = { minPauseMs: 400, leadMs: 280, durMs: 380 };
|
|
152
|
-
// The inbreath: fast in across the pause's end, then let out over the phrase.
|
|
153
|
-
// The idle layer's quiet breathing steps back while speech runs (idle.js), so
|
|
154
|
-
// this is the breath a talking body shows rather than one on top of it.
|
|
155
|
-
const INHALE = [[-420, 0], [-140, 1], [700, 0.55], [2200, 0]];
|
|
156
|
-
const INHALE_AMP = 0.7;
|
|
157
|
-
|
|
158
|
-
const BEAT = {
|
|
159
|
-
// The open vowels carry the energy, and the jaw drop that a stressed vowel
|
|
160
|
-
// gets; the closures and the narrow shapes are never the peak.
|
|
161
|
-
vowels: new Set(['C', 'D', 'E']),
|
|
162
|
-
// Measured over a 42 s track of real cue data, a stricter gate made one beat
|
|
163
|
-
// per 4.2 s of voiced speech, where accents come about once a second; the
|
|
164
|
-
// median phrase on that track is 1.26 s, so a long floor disqualified a
|
|
165
|
-
// third of the phrases outright.
|
|
166
|
-
minPhraseMs: 600,
|
|
167
|
-
minVowels: 2,
|
|
168
|
-
minDurMs: 100,
|
|
169
|
-
// How far above the phrase's mean a vowel must stand to count as prominent,
|
|
170
|
-
// and it must also be the strongest within `localMs` either side of it. A
|
|
171
|
-
// phrase of equal vowels still gets nothing.
|
|
172
|
-
salience: 1.05,
|
|
173
|
-
localMs: 380,
|
|
174
|
-
// Beats at least this far apart; closer and they read as a metronome, and
|
|
175
|
-
// their stroke frequency climbs past the 1.5 Hz line (CLAUDE.md).
|
|
176
|
-
gapMs: 620,
|
|
177
|
-
// Each vowel is weighed once, as it enters this window ahead of the clock:
|
|
178
|
-
// late enough that the cues around it have arrived, early enough for the
|
|
179
|
-
// head stroke's run-up.
|
|
180
|
-
commitMs: 340,
|
|
181
|
-
windowMs: 180,
|
|
182
|
-
// Relative to the vowel onset. The brows hold ~150 ms and come down over
|
|
183
|
-
// ~300 ms, the slower release of a real frontalis, and the upper lids lift
|
|
184
|
-
// a little with them — a brow flash that leaves the lids is a frown's half.
|
|
185
|
-
brow: [[-140, 0], [-40, 1], [110, 1], [410, 0]],
|
|
186
|
-
browAmp: [0.34, 0.30],
|
|
187
|
-
lidAmp: -0.05,
|
|
188
|
-
// **How big a beat is, the syllable decides.** Over a 20.7 s turn of real
|
|
189
|
-
// cue data every beat was authored at exactly 0.340 — 55 of them across five
|
|
190
|
-
// seeds, sd 0.000 — while the loudness of the vowels that won them ran 0.51
|
|
191
|
-
// to 1.00. A vowel can win on length alone, since `score` is duration times
|
|
192
|
-
// loudness, so a quiet long syllable drew the same flash as a shouted one. A
|
|
193
|
-
// face whose every accent is the same size is the "repetitive, like a
|
|
194
|
-
// puppet" read the mouth already had before `shapeFor` scaled it by this
|
|
195
|
-
// same `i`; nothing scaled these.
|
|
196
|
-
//
|
|
197
|
-
// It varies *upward*, because the complaint underneath this one is a still
|
|
198
|
-
// upper face: no beat may shrink. The quiet end lands at 0.349, above the
|
|
199
|
-
// 0.340 every beat used to take, and the range runs to 0.408. That is also
|
|
200
|
-
// what keeps the line faces safe — `browPath` lifts a brow 15 units of an
|
|
201
|
-
// 800-unit frame, so a beat is only ~0.8 px at the 130 px acceptance size,
|
|
202
|
-
// and the mouth's own `0.45 + 0.55i` would have halved it into nothing. The
|
|
203
|
-
// ceiling is the asset's: tara's raise map reads `browRaise / 0.6` and
|
|
204
|
-
// clamps, so under `POSE.brow`'s 0.18 held level a gain past ~1.24 buys
|
|
205
|
-
// light that is already saturated.
|
|
206
|
-
browGain: [0.85, 0.35],
|
|
207
|
-
// **A brow has more than one shape.** Measured over a speaking turn,
|
|
208
|
-
// `browAngleL` and `browInnerL` once moved exactly 0.000: every beat drew the
|
|
209
|
-
// same symmetric raise. The plain raise stays the common one; the inner lift
|
|
210
|
-
// is AU1, the appeal, and deliberately the rarest, because a face that keeps
|
|
211
|
-
// lifting its inner brows reads as worried; the outer-end tilt exists so two
|
|
212
|
-
// beats in a row are not the same drawing. Shapes of one event, on one
|
|
213
|
-
// envelope.
|
|
214
|
-
browForms: [
|
|
215
|
-
{ p: 0.60, inner: 0.00, angle: 0.00 },
|
|
216
|
-
{ p: 0.25, inner: 0.26, angle: 0.00 },
|
|
217
|
-
{ p: 0.15, inner: 0.00, angle: 0.30 },
|
|
218
|
-
],
|
|
219
|
-
// What the head does with a beat. A swing is Graf's one-way movement — the
|
|
220
|
-
// beat taken as a move to a new pose — and is only offered once the pose has
|
|
221
|
-
// been held a while, so a phrase does not change position twice in a breath.
|
|
222
|
-
// Of the rest most are a nod and some leave the head alone: a beat on every
|
|
223
|
-
// accent is a metronome, and the brows still carry it.
|
|
224
|
-
//
|
|
225
|
-
// **Raised on 2026-09-21, and this is where the speaking head's motion went.**
|
|
226
|
-
// Deleting the speaking aversion (gaze.js) took its head share with it, and
|
|
227
|
-
// that share had been most of what moved the head *inside* a sentence: the
|
|
228
|
-
// per-sentence deviation fell from 0.79 deg of yaw to 0.55 measured on the
|
|
229
|
-
// audit's real cue track, because a held phrase pose contributes nothing to a
|
|
230
|
-
// deviation taken about that phrase's own mean. The swing is the honest way
|
|
231
|
-
// to put it back — it is a move to a new pose rather than a look away from
|
|
232
|
-
// the user, which is the whole distinction the removal was about, and Graf
|
|
233
|
-
// has the speaker doing it. Offered more often and after a shorter hold; the
|
|
234
|
-
// hold is still long enough that a phrase does not reposition twice in a
|
|
235
|
-
// breath.
|
|
236
|
-
swingP: 0.42,
|
|
237
|
-
swingAfterMs: 1400,
|
|
238
|
-
swingMs: 320,
|
|
239
|
-
nodP: 0.7,
|
|
240
|
-
// The nod: down and back, over by 0.4 s, and nothing after it. It peaks just
|
|
241
|
-
// ahead of the vowel so that, through the head's 160 ms smoothing, it lands on
|
|
242
|
-
// it. About 2.5° asked of tara, of which the smoothing renders most.
|
|
243
|
-
nod: { pitch: 0.15, attack: 130, release: 240, lead: 170 },
|
|
244
|
-
// A diagonal nod's yaw, on the turn's side. Raised with the swing above, and
|
|
245
|
-
// for the same reason — a diagonal beat moves the head across the sentence
|
|
246
|
-
// where a pure pitch nod only moves it down and back. The amplitude is
|
|
247
|
-
// untouched: the nod's read defect was its geometry, never its size.
|
|
248
|
-
yawP: 0.45,
|
|
249
|
-
yaw: 0.12,
|
|
250
|
-
};
|
|
251
|
-
|
|
252
|
-
// **The body's share of a beat.** A speaker's trunk is not decorative on an
|
|
253
|
-
// accent: emphasis travels down, the shoulders lift and set slightly on a
|
|
254
|
-
// stressed syllable, and a phrase boundary is where posture resets (Hadar et
|
|
255
|
-
// al. 1983 tied the pattern of movement against stillness to juncture). It is
|
|
256
|
-
// not a gesture — there is no arm — but the head's stroke arriving late and
|
|
257
|
-
// small at the shoulders, which `params.TAU` supplies for free by chasing the
|
|
258
|
-
// shoulders at 0.19 s and the lean at 0.24 s. Each stroke returns through
|
|
259
|
-
// neutral: a trunk that only ever lifts is a standing shrug, which is what a
|
|
260
|
-
// filmstrip once showed as the figure sitting larger and lower in frame.
|
|
261
|
-
// Raised on 2026-09-21 with the rest of the body: the trunk has no measured
|
|
262
|
-
// hold budget the way the head does — nothing about a shoulder tells a
|
|
263
|
-
// projected photograph it has been turned too far — so it is the channel where
|
|
264
|
-
// "a bit more movement" is free, and it is the one the head-on-a-stick read
|
|
265
|
-
// comes from.
|
|
266
|
-
const BEAT_BODY = {
|
|
267
|
-
keys: [[-380, 0], [-60, 1], [140, 0.25], [320, -0.30], [600, 0]],
|
|
268
|
-
shoulder: 0.36, lean: 0.15, spread: 0.35,
|
|
269
|
-
};
|
|
270
|
-
// The lift a phrase carries, over its own span: up on the first stressed
|
|
271
|
-
// syllable and settling below where it started as the breath goes out.
|
|
272
|
-
const PHRASE_BODY = {
|
|
273
|
-
keys: [[-260, 0], [180, 1], [900, 0.5], [1500, -0.30], [2400, 0]],
|
|
274
|
-
shoulder: 0.28,
|
|
275
|
-
lean: 0.21,
|
|
276
|
-
};
|
|
277
|
-
|
|
278
|
-
// The phrase-final settle: chin down this far from the phrase's own pose,
|
|
279
|
-
// held until the next phrase lifts it. Pitch only, and slower than a nod — it
|
|
280
|
-
// closes a phrase rather than marking a word.
|
|
281
|
-
const SETTLE = { pitch: 0.07, lead: 60, durMs: 360, gapMs: 520 };
|
|
282
|
-
|
|
283
|
-
// A turn that ends, or a clip that takes the head, sends it home this fast.
|
|
284
|
-
const HOME_MS = 700;
|
|
285
|
-
|
|
286
|
-
// Warmth: corners, and the squint that makes them a smile rather than a mask
|
|
287
|
-
// (emotions.js). A smile that never varies is discounted as insincere
|
|
288
|
-
// (research-perception.md §3), so warmth only ever comes as an episode, with
|
|
289
|
-
// an onset, a hold and an offset, and each is tied to something that
|
|
290
|
-
// happened: the reply's first words (`onset`, from the first voiced cue), the
|
|
291
|
-
// reply played out to its end (`close`), and an acknowledgement the server
|
|
292
|
-
// sent (`ack`). Felt smiles run from about two-thirds of a second to four
|
|
293
|
-
// (Ekman & Friesen 1982), and a listener's smile is itself a backchannel
|
|
294
|
-
// (Brunner 1979).
|
|
295
|
-
//
|
|
296
|
-
// `close` is the face the user sees as the floor passes to them, so it is the
|
|
297
|
-
// biggest and the longest: full for 2.6 s, gone by 3.8 s, seen while they
|
|
298
|
-
// start to talk and gone before it could read as fixed. It used to be 0.18
|
|
299
|
-
// held 1.2 s, which on the Blender faces barely lifted the corners before it
|
|
300
|
-
// went; judged at crop on tara and tushar, 0.36 is the first height that
|
|
301
|
-
// reads as a smile rather than a resting mouth, and still well under the
|
|
302
|
-
// happy states' 0.48–0.58. `onset` stays small, because the visemes are about
|
|
303
|
-
// to take the mouth.
|
|
304
|
-
// `ack` rises with the nod and outlasts it, since a smile that ends with the
|
|
305
|
-
// head's last beat reads as part of the gesture rather than as pleasure.
|
|
306
|
-
//
|
|
307
|
-
// `listen` is the fourth, and it is the one episode that is not about speech at
|
|
308
|
-
// all: the floor has arrived and the face receives it. Every other state change
|
|
309
|
-
// the avatar makes is a change of task; this one is a change of *who is
|
|
310
|
-
// talking*, and a face that takes the floor back with no expression at all is
|
|
311
|
-
// where the "not smiling enough" read comes from. Its shape is the owner's
|
|
312
|
-
// (2026-09-21): up, then a long fade to neutral rather than a held level, which
|
|
313
|
-
// is the same argument the other three are built on — a smile that stays is
|
|
314
|
-
// discounted. The full height lasts 1.8 s, inside Ekman's felt-smile window,
|
|
315
|
-
// and the remaining 3.8 s is the fade. **It is the warmest of the four, raised
|
|
316
|
-
// on the owner's read of 2026-09-21 that it was not arriving.** It started at
|
|
317
|
-
// `ack`'s height and reasoned that `close` — the face that hands the floor
|
|
318
|
-
// over — should outrank the face that takes it; on screen that had it landing
|
|
319
|
-
// as a politeness rather than as a welcome, and this is the one episode a user
|
|
320
|
-
// is looking straight at when it fires.
|
|
321
|
-
// Times in ms.
|
|
322
|
-
const WARMTH = {
|
|
323
|
-
onset: { keys: [[0, 0], [350, 1], [1500, 1], [2800, 0]], corner: 0.18, squint: 0.08 },
|
|
324
|
-
close: { keys: [[0, 0], [300, 1], [2600, 1], [3800, 0]], corner: 0.36, squint: 0.14 },
|
|
325
|
-
ack: { keys: [[0, 0], [220, 1], [1100, 1], [2000, 0]], corner: 0.30, squint: 0.12 },
|
|
326
|
-
listen: { keys: [[0, 0], [400, 1], [2200, 1], [6000, 0]], corner: 0.42, squint: 0.17 },
|
|
327
|
-
};
|
|
328
|
-
// A continuer can come every second or two. Each one smiling would hold the
|
|
329
|
-
// smile up for as long as the user talks, which is the fixed smile again, so
|
|
330
|
-
// an acknowledgement inside this long of the last smiling one nods without it.
|
|
331
|
-
const ACK_SMILE_REST_MS = 4000;
|
|
332
|
-
// The listening smile rests behind *any* warmth, not just its own, and this is
|
|
333
|
-
// the whole of what keeps it from becoming the fixed smile. A turn normally
|
|
334
|
-
// ends SPEAKING -> `close` -> LISTENING within a few hundred ms, and `close` is
|
|
335
|
-
// already that handover's smile; a state that flaps back through LISTENING —
|
|
336
|
-
// through THINKING and out again, or a barge-in and a resume — would otherwise
|
|
337
|
-
// re-arm one every time. Longer than the episodes themselves, so two can never
|
|
338
|
-
// run end to end.
|
|
339
|
-
const LISTEN_SMILE_REST_MS = 7000;
|
|
340
|
-
|
|
341
|
-
/**
|
|
342
|
-
* What a rig renders of the head amplitudes above, unless it says otherwise.
|
|
343
|
-
*
|
|
344
|
-
* **A pose unit is not a shared quantity.** On tara it is an angle; on an SVG
|
|
345
|
-
* face it is a pixel count — `peep`'s head travels about 17 px per unit of
|
|
346
|
-
* pitch on a 130 px drawing. A rig passes its own `prosodyHeadGain` (tara: 1);
|
|
347
|
-
* one that has not been measured renders this share. It was held at 0.3 while
|
|
348
|
-
* the amplitudes here were sized to the lab's ranges, which scaled peep's
|
|
349
|
-
* speaking head to 20 px; sized to the medium they are small enough to render
|
|
350
|
-
* whole, and the SVG faces now take the same hold-and-move head as tara —
|
|
351
|
-
* unjudged by eye on them, which is the open item this number stands for.
|
|
352
|
-
*/
|
|
353
|
-
export const UNCALIBRATED_HEAD_GAIN = 1;
|
|
354
|
-
|
|
355
|
-
function envelope(t, keys) {
|
|
356
|
-
if (t <= keys[0][0] || t >= keys[keys.length - 1][0]) return 0;
|
|
357
|
-
for (let k = 1; k < keys.length; k++) {
|
|
358
|
-
const [t1, v1] = keys[k];
|
|
359
|
-
if (t <= t1) {
|
|
360
|
-
const [t0, v0] = keys[k - 1];
|
|
361
|
-
return v0 + (v1 - v0) * ((t - t0) / (t1 - t0));
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
return 0;
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
const endOf = (keys) => keys[keys.length - 1][0];
|
|
368
|
-
const rand = ([a, b]) => a + Math.random() * (b - a);
|
|
369
|
-
const side = () => (Math.random() < 0.5 ? -1 : 1);
|
|
370
|
-
|
|
371
|
-
const isPause = (cues, j) =>
|
|
372
|
-
cues[j].v === SILENT && (j + 1 >= cues.length || cues[j + 1].t - cues[j].t >= PAUSE_MS);
|
|
373
|
-
|
|
374
|
-
/** Channels this layer writes, besides the `blink` flag. */
|
|
375
|
-
export const PROSODY_CHANNELS = [
|
|
376
|
-
'headPitch', 'headYaw', 'headRoll',
|
|
377
|
-
'browRaiseL', 'browRaiseR', 'browInnerL', 'browInnerR',
|
|
378
|
-
'browAngleL', 'browAngleR', 'lidL', 'lidR',
|
|
379
|
-
'mouthCornerL', 'mouthCornerR', 'squintL', 'squintR',
|
|
380
|
-
'breath',
|
|
381
|
-
'shoulderL', 'shoulderR', 'torsoLean',
|
|
382
|
-
];
|
|
383
|
-
|
|
384
|
-
export class SpeechProsody {
|
|
385
|
-
constructor(opts = {}) {
|
|
386
|
-
// Moves are timed on this layer's own clock, not the track's: a turn's
|
|
387
|
-
// clock restarts, and the tails of a settle or a closing smile outlive the
|
|
388
|
-
// track they came from.
|
|
389
|
-
this._ms = 0;
|
|
390
|
-
this._moves = [];
|
|
391
|
-
this._warm = [];
|
|
392
|
-
this._lastAckSmile = -Infinity;
|
|
393
|
-
this._lastWarm = -Infinity;
|
|
394
|
-
this._head = new HeadPose();
|
|
395
|
-
this._home = true;
|
|
396
|
-
this._corner = 0;
|
|
397
|
-
this._cornerTo = 0;
|
|
398
|
-
this._brow = 0;
|
|
399
|
-
this._browTo = 0;
|
|
400
|
-
// See `_drawBrow`. Defaulted to the range every face has always drawn from
|
|
401
|
-
// with no band skipped, so a seeded reel of an SVG or Canvas face draws the
|
|
402
|
-
// same level from the same `Math.random` call as before.
|
|
403
|
-
this._brows = { range: POSE.brow, floor: 0, forms: BEAT.browForms, ...opts.brows };
|
|
404
|
-
this._side = side();
|
|
405
|
-
this.reset();
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
/**
|
|
409
|
-
* Forget the utterance: its times are offsets into a clock that restarts.
|
|
410
|
-
* `newTurn` false is the same turn re-spoken on the same clock (the accurate
|
|
411
|
-
* leg splicing), and everything already weighed stays weighed.
|
|
412
|
-
*/
|
|
413
|
-
reset(newTurn = true) {
|
|
414
|
-
if (!newTurn) return;
|
|
415
|
-
this._blinkKey = null;
|
|
416
|
-
this._lastBlink = -Infinity;
|
|
417
|
-
this._resumeKey = null;
|
|
418
|
-
this._bookedKey = null;
|
|
419
|
-
this._scanT = -Infinity;
|
|
420
|
-
this._lastBeat = -Infinity;
|
|
421
|
-
this._lastMove = -Infinity;
|
|
422
|
-
this._phraseKey = null;
|
|
423
|
-
this._poseAt = -Infinity;
|
|
424
|
-
this._phrasePitch = 0;
|
|
425
|
-
this._onsetPending = true;
|
|
426
|
-
// The side a diagonal beat leans to, and which way roll couples to yaw,
|
|
427
|
-
// are the speaker's habit for the turn (Graf: motions repeat).
|
|
428
|
-
this._beatSide = side();
|
|
429
|
-
this._rollSide = side();
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
/** The track played out to its end: the turn's closing warmth. */
|
|
433
|
-
closeTurn() {
|
|
434
|
-
this._smile(WARMTH.close);
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
/**
|
|
438
|
-
* The avatar has entered LISTENING: the floor is the user's. A warmth
|
|
439
|
-
* episode, unless something smiled recently — see `LISTEN_SMILE_REST_MS`,
|
|
440
|
-
* which is what stands between this and a face that is always smiling.
|
|
441
|
-
*
|
|
442
|
-
* Nothing smiles when the user *starts* talking, on purpose. The client has
|
|
443
|
-
* no event for that moment, and a smile the renderer timed for itself would
|
|
444
|
-
* be an acknowledgement nobody sent. The lever for more warmth while
|
|
445
|
-
* listening is the server sending more `ACK_NOD`s, never a timer here.
|
|
446
|
-
*/
|
|
447
|
-
listen() {
|
|
448
|
-
if (this._ms - this._lastWarm < LISTEN_SMILE_REST_MS) return;
|
|
449
|
-
this._smile(WARMTH.listen);
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
/** Start a warmth episode, and remember that the face has just smiled. */
|
|
453
|
-
_smile(w) {
|
|
454
|
-
this._lastWarm = this._ms;
|
|
455
|
-
this._warm.push({ at: this._ms, w });
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
/**
|
|
459
|
-
* The server acknowledged what the user is saying: the smile that goes with
|
|
460
|
-
* it. Here rather than in the clip because it has to outlast the nod, and
|
|
461
|
-
* because it is one smile with the turn-edge warmth when the two meet.
|
|
462
|
-
*/
|
|
463
|
-
acknowledge() {
|
|
464
|
-
if (this._ms - this._lastAckSmile < ACK_SMILE_REST_MS) return;
|
|
465
|
-
this._lastAckSmile = this._ms;
|
|
466
|
-
this._smile(WARMTH.ack);
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
/** Cut off: a smile that survives being interrupted has not noticed. The
|
|
470
|
-
* rest period is deliberately *not* cleared — being interrupted is not an
|
|
471
|
-
* occasion to smile a fresh one at the state change that follows. */
|
|
472
|
-
cool() {
|
|
473
|
-
this._warm = [];
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
/**
|
|
477
|
-
* @param {{cues: {t:number, v:string, i?:number}[], now: number, index: number}} track
|
|
478
|
-
* the viseme track, already sampled this frame
|
|
479
|
-
* @param {boolean} active speech owns the mouth and no clip is gesturing
|
|
480
|
-
* @param {number} [dt] seconds since the last frame
|
|
481
|
-
* @returns {Record<string, number> & {blink: boolean}} additive deltas on
|
|
482
|
-
* PROSODY_CHANNELS, at gain 1
|
|
483
|
-
*/
|
|
484
|
-
update(track, active, dt = 1 / 60) {
|
|
485
|
-
this._ms += dt * 1000;
|
|
486
|
-
const out = { blink: false };
|
|
487
|
-
for (const c of PROSODY_CHANNELS) out[c] = 0;
|
|
488
|
-
|
|
489
|
-
if (active && track.cues.length) {
|
|
490
|
-
this._home = false;
|
|
491
|
-
this._listen(track, out);
|
|
492
|
-
} else {
|
|
493
|
-
// Beats belong to the words: an interrupted or gesture-covered turn
|
|
494
|
-
// drops the ones still to come, and the head goes home. Tails play out.
|
|
495
|
-
this._moves = this._moves.filter((m) => m.keep);
|
|
496
|
-
this._cornerTo = 0;
|
|
497
|
-
this._browTo = 0;
|
|
498
|
-
if (!this._home) {
|
|
499
|
-
this._home = true;
|
|
500
|
-
this._head.cancelStrokes(this._ms);
|
|
501
|
-
this._head.moveTo({ headYaw: 0, headPitch: 0, headRoll: 0 }, this._ms, HOME_MS);
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
const t = this._ms;
|
|
506
|
-
const h = this._head.sample(t);
|
|
507
|
-
out.headYaw += h.headYaw;
|
|
508
|
-
out.headPitch += h.headPitch;
|
|
509
|
-
out.headRoll += h.headRoll;
|
|
510
|
-
// Where the head is turned, without the strokes on it. The trunk follows
|
|
511
|
-
// it — a phrase's pose recruits the body, a nod does not — and the mixer
|
|
512
|
-
// budgets against it, for the same division: what the head *holds* is what
|
|
513
|
-
// has to stay inside a face's measured range, and a stroke is forgiven a
|
|
514
|
-
// peak because it is over before it is read as a pose.
|
|
515
|
-
out.hold = this._head.held(t);
|
|
516
|
-
out.trunkYaw = out.hold.headYaw;
|
|
517
|
-
this._corner += (this._cornerTo - this._corner) * (1 - Math.exp(-dt / POSE.cornerTauS));
|
|
518
|
-
out.mouthCornerL += this._corner;
|
|
519
|
-
out.mouthCornerR += this._corner;
|
|
520
|
-
// Eased rather than moved: a brow that steps to its phrase's level reads as
|
|
521
|
-
// a beat, and the beats are already on top of this.
|
|
522
|
-
this._brow += (this._browTo - this._brow) * (1 - Math.exp(-dt / POSE.browTauS));
|
|
523
|
-
out.browRaiseL += this._brow;
|
|
524
|
-
out.browRaiseR += this._brow * POSE.browAsym;
|
|
525
|
-
|
|
526
|
-
this._moves = this._moves.filter((m) => t - m.at < endOf(m.keys));
|
|
527
|
-
for (const m of this._moves) {
|
|
528
|
-
const e = envelope(t - m.at, m.keys);
|
|
529
|
-
if (e) for (const c in m.add) out[c] += e * m.add[c];
|
|
530
|
-
}
|
|
531
|
-
// Two warmth episodes that meet are one smile, not a bigger one.
|
|
532
|
-
this._warm = this._warm.filter((m) => t - m.at < endOf(m.w.keys));
|
|
533
|
-
let corner = 0;
|
|
534
|
-
let squint = 0;
|
|
535
|
-
for (const m of this._warm) {
|
|
536
|
-
const e = envelope(t - m.at, m.w.keys);
|
|
537
|
-
corner = Math.max(corner, e * m.w.corner);
|
|
538
|
-
squint = Math.max(squint, e * m.w.squint);
|
|
539
|
-
}
|
|
540
|
-
out.mouthCornerL += corner;
|
|
541
|
-
out.mouthCornerR += corner;
|
|
542
|
-
out.squintL += squint;
|
|
543
|
-
out.squintR += squint;
|
|
544
|
-
return out;
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
_listen(track, out) {
|
|
548
|
-
const { cues, now } = track;
|
|
549
|
-
const i = Math.min(track.index, cues.length - 1);
|
|
550
|
-
const cue = cues[i];
|
|
551
|
-
if (cue.t > now) return;
|
|
552
|
-
// Converts a time on the track's clock to one on this layer's.
|
|
553
|
-
const at = (ms) => this._ms + (ms - now);
|
|
554
|
-
|
|
555
|
-
// The pause blink.
|
|
556
|
-
if (isPause(cues, i) && cue.t !== this._blinkKey) {
|
|
557
|
-
this._blinkKey = cue.t;
|
|
558
|
-
if (now - cue.t < BLINK_FRESH_MS && now - this._lastBlink >= BLINK_MIN_GAP_MS) {
|
|
559
|
-
this._lastBlink = now;
|
|
560
|
-
out.blink = true;
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
// The next phrase's pose and the inbreath: booked inside a long pause,
|
|
565
|
-
// played through the resume.
|
|
566
|
-
if (cue.v === SILENT && i + 1 < cues.length && cue.t !== this._resumeKey
|
|
567
|
-
&& cues[i + 1].t - cue.t >= ONSET.minPauseMs) {
|
|
568
|
-
this._resumeKey = cue.t;
|
|
569
|
-
this._bookedKey = cues[i + 1].t;
|
|
570
|
-
this._pose(at(cues[i + 1].t) - ONSET.leadMs, ONSET.durMs);
|
|
571
|
-
this._moves.push({ at: at(cues[i + 1].t), keys: INHALE, add: { breath: INHALE_AMP } });
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
if (cue.v !== SILENT) {
|
|
575
|
-
if (this._onsetPending) {
|
|
576
|
-
this._onsetPending = false;
|
|
577
|
-
this._smile(WARMTH.onset);
|
|
578
|
-
}
|
|
579
|
-
let a = i;
|
|
580
|
-
while (a > 0 && !isPause(cues, a - 1)) a--;
|
|
581
|
-
if (cues[a].t !== this._phraseKey) {
|
|
582
|
-
this._phraseKey = cues[a].t;
|
|
583
|
-
// A phrase nothing was booked for: the turn's first words, or a
|
|
584
|
-
// phrase after a pause too short to book a move in. An old pose
|
|
585
|
-
// changes as it starts; a recent one just lifts out of its settle.
|
|
586
|
-
if (this._bookedKey !== cues[a].t) {
|
|
587
|
-
if (this._ms - this._poseAt >= POSE.holdMs) this._pose(this._ms, ONSET.durMs);
|
|
588
|
-
else this._head.moveTo({ headPitch: this._phrasePitch }, this._ms, ONSET.durMs);
|
|
589
|
-
}
|
|
590
|
-
// The postural lift belongs to the sentence.
|
|
591
|
-
this._moves.push({
|
|
592
|
-
at: at(cues[a].t), keys: PHRASE_BODY.keys,
|
|
593
|
-
add: {
|
|
594
|
-
shoulderL: PHRASE_BODY.shoulder, shoulderR: PHRASE_BODY.shoulder,
|
|
595
|
-
torsoLean: PHRASE_BODY.lean,
|
|
596
|
-
},
|
|
597
|
-
});
|
|
598
|
-
}
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
// Every vowel is weighed once, as it comes into the commit window. One
|
|
602
|
-
// that arrives already inside it is let go: a late stroke is worse than
|
|
603
|
-
// none.
|
|
604
|
-
for (let j = i; j + 1 < cues.length && cues[j].t - now <= BEAT.commitMs + BEAT.windowMs; j++) {
|
|
605
|
-
const c = cues[j];
|
|
606
|
-
if (c.t <= this._scanT || !BEAT.vowels.has(c.v)) continue;
|
|
607
|
-
this._scanT = c.t;
|
|
608
|
-
if (c.t - now >= BEAT.commitMs) this._weigh(cues, j, at);
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
// A new pose for the head, starting at `at` on this layer's clock. The next
|
|
613
|
-
// phrase usually turns the other way.
|
|
614
|
-
_pose(at, dur) {
|
|
615
|
-
this._poseAt = at;
|
|
616
|
-
if (Math.random() < POSE.switchP) this._side = -this._side;
|
|
617
|
-
const s = this._side;
|
|
618
|
-
this._phrasePitch = rand(POSE.headPitch);
|
|
619
|
-
this._head.moveTo({
|
|
620
|
-
headYaw: s * rand(POSE.headYaw),
|
|
621
|
-
headPitch: this._phrasePitch,
|
|
622
|
-
headRoll: s * this._rollSide * rand(POSE.headRoll),
|
|
623
|
-
}, at, dur);
|
|
624
|
-
this._cornerTo = rand(POSE.corner);
|
|
625
|
-
this._browTo = this._drawBrow();
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
/**
|
|
629
|
-
* The level this phrase holds its brows at.
|
|
630
|
-
*
|
|
631
|
-
* `floor` is a band about zero the draw skips. `browRaise` is the one channel
|
|
632
|
-
* the Blender rigs split by sign into two photographed maps — a raise above
|
|
633
|
-
* zero, a knit below it — and *both* read as nothing near zero, so a range
|
|
634
|
-
* straddling it spends most of its phrases in a dead band where the upper
|
|
635
|
-
* face recruits no light at all. That is the deadpan: not too few events, but
|
|
636
|
-
* a pose that is usually nowhere. Skipping the band makes each phrase commit
|
|
637
|
-
* to a furrow or a lift.
|
|
638
|
-
*
|
|
639
|
-
* It is a floor and not a bigger range on purpose. What the prohibitions here
|
|
640
|
-
* turn on is *hold*, not size — a held inner lift reads as worried, a held
|
|
641
|
-
* outer kink as a smirk — and this changes only how far the brows travel
|
|
642
|
-
* between one phrase and the next, never how long any shape is kept.
|
|
643
|
-
*
|
|
644
|
-
* The two sides keep their widths' share of the draw rather than collapsing
|
|
645
|
-
* onto ±floor: a skipped band must not become a pair of repeated values.
|
|
646
|
-
*/
|
|
647
|
-
_drawBrow() {
|
|
648
|
-
const [a, b] = this._brows.range;
|
|
649
|
-
const f = this._brows.floor;
|
|
650
|
-
const lo = Math.max(0, -f - a);
|
|
651
|
-
const hi = Math.max(0, b - f);
|
|
652
|
-
if (!(f > 0) || lo + hi <= 0) return rand(this._brows.range);
|
|
653
|
-
return Math.random() * (lo + hi) < lo ? rand([a, -f]) : rand([f, b]);
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
// Decides whether vowel `j` carries a beat or closes its phrase.
|
|
657
|
-
_weigh(cues, j, at) {
|
|
658
|
-
let a = j;
|
|
659
|
-
while (a > 0 && !isPause(cues, a - 1)) a--;
|
|
660
|
-
let b = j;
|
|
661
|
-
while (b + 1 < cues.length && !isPause(cues, b)) b++;
|
|
662
|
-
const complete = isPause(cues, b);
|
|
663
|
-
const span = cues[b].t - cues[a].t;
|
|
664
|
-
const vowels = [];
|
|
665
|
-
for (let k = a; k < b; k++) {
|
|
666
|
-
const c = cues[k];
|
|
667
|
-
if (!BEAT.vowels.has(c.v)) continue;
|
|
668
|
-
const dur = cues[k + 1].t - c.t;
|
|
669
|
-
// `score` ranks, `i` sizes. They are kept apart deliberately: a missing
|
|
670
|
-
// loudness still scores as a full one so selection is unchanged, but it
|
|
671
|
-
// leaves `i` null so the beat below stays exactly as it was authored.
|
|
672
|
-
vowels.push({ t: c.t, dur, i: c.i, score: dur * (c.i == null ? 1 : c.i) });
|
|
673
|
-
}
|
|
674
|
-
const v = vowels.find((x) => x.t === cues[j].t);
|
|
675
|
-
|
|
676
|
-
// The last vowel before a pause is long because the phrase is ending, not
|
|
677
|
-
// because it is stressed: final lengthening marks a boundary, and scoring
|
|
678
|
-
// it would put a beat on the last word of nearly every phrase. It gets the
|
|
679
|
-
// settle instead.
|
|
680
|
-
if (complete && v === vowels[vowels.length - 1]) {
|
|
681
|
-
if (span >= BEAT.minPhraseMs && v.t - this._lastMove >= SETTLE.gapMs) {
|
|
682
|
-
this._lastMove = v.t;
|
|
683
|
-
this._head.moveTo({ headPitch: this._phrasePitch + SETTLE.pitch }, at(v.t) - SETTLE.lead, SETTLE.durMs);
|
|
684
|
-
}
|
|
685
|
-
return;
|
|
686
|
-
}
|
|
687
|
-
if (complete) vowels.pop();
|
|
688
|
-
|
|
689
|
-
if (span < BEAT.minPhraseMs || vowels.length < BEAT.minVowels) return;
|
|
690
|
-
const mean = vowels.reduce((s, x) => s + x.score, 0) / vowels.length;
|
|
691
|
-
if (v.dur < BEAT.minDurMs || v.score < BEAT.salience * mean) return;
|
|
692
|
-
if (vowels.some((x) => x !== v && Math.abs(x.t - v.t) < BEAT.localMs && x.score > v.score)) return;
|
|
693
|
-
if (v.t - this._lastBeat < BEAT.gapMs) return;
|
|
694
|
-
this._lastBeat = this._lastMove = v.t;
|
|
695
|
-
|
|
696
|
-
let yaw = 0;
|
|
697
|
-
const vt = at(v.t);
|
|
698
|
-
if (vt - this._poseAt >= BEAT.swingAfterMs && Math.random() < BEAT.swingP) {
|
|
699
|
-
this._pose(vt - BEAT.nod.lead, BEAT.swingMs);
|
|
700
|
-
yaw = this._head.aim.headYaw;
|
|
701
|
-
} else if (Math.random() < BEAT.nodP) {
|
|
702
|
-
const add = { headPitch: BEAT.nod.pitch };
|
|
703
|
-
if (Math.random() < BEAT.yawP) add.headYaw = yaw = this._beatSide * BEAT.yaw;
|
|
704
|
-
this._head.stroke(add, vt - BEAT.nod.lead, BEAT.nod.attack, BEAT.nod.release);
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
const forms = this._brows.forms;
|
|
708
|
-
let br = Math.random() * forms.reduce((s, f) => s + f.p, 0);
|
|
709
|
-
const bf = forms.find((f) => (br -= f.p) < 0) || forms[0];
|
|
710
|
-
// The whole gesture takes the syllable's size, not the raise alone: the
|
|
711
|
-
// form's parts keep their proportions to each other, so a loud beat is the
|
|
712
|
-
// same drawing bigger rather than a different shape.
|
|
713
|
-
const g = v.i == null ? 1 : BEAT.browGain[0] + BEAT.browGain[1] * v.i;
|
|
714
|
-
this._moves.push({
|
|
715
|
-
at: vt, keys: BEAT.brow,
|
|
716
|
-
add: {
|
|
717
|
-
browRaiseL: BEAT.browAmp[0] * g, browRaiseR: BEAT.browAmp[1] * g,
|
|
718
|
-
browInnerL: bf.inner * g, browInnerR: bf.inner * g,
|
|
719
|
-
// The tilt is the one asymmetric thing here: both outer ends go up on
|
|
720
|
-
// the side the beat already leans to, which is a shade rather than the
|
|
721
|
-
// matched pair a symmetric lift would draw.
|
|
722
|
-
browAngleL: bf.angle * g * (this._beatSide < 0 ? 1 : 0.55),
|
|
723
|
-
browAngleR: bf.angle * g * (this._beatSide > 0 ? 1 : 0.55),
|
|
724
|
-
lidL: BEAT.lidAmp * g, lidR: BEAT.lidAmp * g,
|
|
725
|
-
},
|
|
726
|
-
});
|
|
727
|
-
// The trunk takes the same stroke. The two shoulders are deliberately
|
|
728
|
-
// unequal — `spread` puts more of it on the side the head is going, so a
|
|
729
|
-
// diagonal beat reads as one body turning rather than two shoulders
|
|
730
|
-
// shrugging in unison, which is the tell of a rig with no spine.
|
|
731
|
-
const lead = yaw < 0 ? -1 : 1;
|
|
732
|
-
this._moves.push({
|
|
733
|
-
at: vt, keys: BEAT_BODY.keys,
|
|
734
|
-
add: {
|
|
735
|
-
shoulderL: BEAT_BODY.shoulder * (1 + lead * BEAT_BODY.spread),
|
|
736
|
-
shoulderR: BEAT_BODY.shoulder * (1 - lead * BEAT_BODY.spread),
|
|
737
|
-
torsoLean: BEAT_BODY.lean,
|
|
738
|
-
},
|
|
739
|
-
});
|
|
740
|
-
}
|
|
741
|
-
}
|