@voqalize/avatar 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -661
- package/README.md +144 -663
- package/client/Avatar.tsx +57 -0
- package/client/AvatarClient.ts +523 -0
- package/client/arjun.ts +24 -0
- package/client/createAvatar.ts +139 -0
- package/client/createCanvasAvatar.ts +72 -0
- package/client/index.ts +39 -0
- package/client/internal.ts +80 -0
- package/client/interviewer-female.ts +4 -0
- package/client/interviewer-male.ts +4 -0
- package/client/ishita.ts +24 -0
- package/client/kabir.ts +24 -0
- package/client/meera.ts +24 -0
- package/client/naina.ts +24 -0
- package/client/professional-female-a.ts +4 -0
- package/client/professional-female-b.ts +4 -0
- package/client/professional-male-a.ts +4 -0
- package/client/professional-male-b.ts +4 -0
- package/client/react.ts +13 -0
- package/client/types.ts +146 -0
- package/client/useAvatar.ts +74 -0
- package/client/vikram.ts +24 -0
- package/dist/Avatar.d.ts +44 -0
- package/dist/Avatar.d.ts.map +1 -0
- package/{client/dist → dist}/Avatar.js +2 -2
- package/dist/Avatar.js.map +1 -0
- package/dist/AvatarClient.d.ts +196 -0
- package/dist/AvatarClient.d.ts.map +1 -0
- package/dist/AvatarClient.js +450 -0
- package/dist/AvatarClient.js.map +1 -0
- package/dist/arjun.d.ts +6 -0
- package/dist/arjun.d.ts.map +1 -0
- package/dist/arjun.js +18 -0
- package/dist/arjun.js.map +1 -0
- package/dist/createAvatar.d.ts +99 -0
- package/dist/createAvatar.d.ts.map +1 -0
- package/dist/createAvatar.js +75 -0
- package/dist/createAvatar.js.map +1 -0
- package/dist/createCanvasAvatar.d.ts +22 -0
- package/dist/createCanvasAvatar.d.ts.map +1 -0
- package/dist/createCanvasAvatar.js +47 -0
- package/dist/createCanvasAvatar.js.map +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +29 -0
- package/dist/index.js.map +1 -0
- package/dist/internal.d.ts +30 -0
- package/dist/internal.d.ts.map +1 -0
- package/dist/internal.js +33 -0
- package/dist/internal.js.map +1 -0
- package/dist/interviewer-female.d.ts +4 -0
- package/dist/interviewer-female.d.ts.map +1 -0
- package/dist/interviewer-female.js +3 -0
- package/dist/interviewer-female.js.map +1 -0
- package/dist/interviewer-male.d.ts +4 -0
- package/dist/interviewer-male.d.ts.map +1 -0
- package/dist/interviewer-male.js +3 -0
- package/dist/interviewer-male.js.map +1 -0
- package/dist/ishita.d.ts +6 -0
- package/dist/ishita.d.ts.map +1 -0
- package/dist/ishita.js +18 -0
- package/dist/ishita.js.map +1 -0
- package/dist/kabir.d.ts +6 -0
- package/dist/kabir.d.ts.map +1 -0
- package/dist/kabir.js +18 -0
- package/dist/kabir.js.map +1 -0
- package/dist/meera.d.ts +6 -0
- package/dist/meera.d.ts.map +1 -0
- package/dist/meera.js +18 -0
- package/dist/meera.js.map +1 -0
- package/dist/naina.d.ts +6 -0
- package/dist/naina.d.ts.map +1 -0
- package/dist/naina.js +18 -0
- package/dist/naina.js.map +1 -0
- package/dist/professional-female-a.d.ts +4 -0
- package/dist/professional-female-a.d.ts.map +1 -0
- package/dist/professional-female-a.js +3 -0
- package/dist/professional-female-a.js.map +1 -0
- package/dist/professional-female-b.d.ts +4 -0
- package/dist/professional-female-b.d.ts.map +1 -0
- package/dist/professional-female-b.js +3 -0
- package/dist/professional-female-b.js.map +1 -0
- package/dist/professional-male-a.d.ts +4 -0
- package/dist/professional-male-a.d.ts.map +1 -0
- package/dist/professional-male-a.js +3 -0
- package/dist/professional-male-a.js.map +1 -0
- package/dist/professional-male-b.d.ts +4 -0
- package/dist/professional-male-b.d.ts.map +1 -0
- package/dist/professional-male-b.js +3 -0
- package/dist/professional-male-b.js.map +1 -0
- package/dist/react.d.ts +13 -0
- package/dist/react.d.ts.map +1 -0
- package/dist/react.js +13 -0
- package/dist/react.js.map +1 -0
- package/dist/types.d.ts +94 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +93 -0
- package/dist/types.js.map +1 -0
- package/dist/useAvatar.d.ts +39 -0
- package/dist/useAvatar.d.ts.map +1 -0
- package/dist/useAvatar.js +35 -0
- package/dist/useAvatar.js.map +1 -0
- package/dist/vikram.d.ts +6 -0
- package/dist/vikram.d.ts.map +1 -0
- package/dist/vikram.js +18 -0
- package/dist/vikram.js.map +1 -0
- package/package.json +81 -24
- package/src/avatar.d.ts +161 -120
- package/src/avatar.js +234 -146
- package/src/behavior.d.ts +26 -0
- package/src/behavior.js +102 -0
- package/src/camera.js +29 -0
- package/src/canvas/author/parts/eye.mjs +722 -0
- package/src/canvas/author/parts/hand.mjs +1156 -0
- package/src/canvas/author/parts/mouth.mjs +741 -0
- package/src/canvas/author/parts/nose.mjs +100 -0
- package/src/canvas/author/parts/skin-detail.mjs +67 -0
- package/src/canvas/author/path.mjs +283 -0
- package/src/canvas/author/rig.mjs +405 -0
- package/src/canvas/avatars/round/face.d.mts +3 -0
- package/src/canvas/avatars/round/face.mjs +1307 -0
- package/src/canvas/create-rig.d.ts +15 -0
- package/src/canvas/create-rig.js +100 -0
- package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
- package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
- package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
- package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
- package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
- package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
- package/src/canvas/data/img/round-m3-top-body.webp +0 -0
- package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
- package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
- package/src/canvas/data/img/round-w1-top-body.webp +0 -0
- package/src/canvas/data/interviewer-female.rig.json +1 -0
- package/src/canvas/data/interviewer-male.rig.json +1 -0
- package/src/canvas/data/professional-female-a.rig.json +1 -0
- package/src/canvas/data/professional-female-b.rig.json +1 -0
- package/src/canvas/data/professional-male-a.rig.json +1 -0
- package/src/canvas/data/professional-male-b.rig.json +1 -0
- package/src/canvas/src/live.js +508 -0
- package/src/canvas/src/render2d.js +218 -0
- package/src/canvas/src/rig.js +297 -0
- package/src/canvas/src/vocab.js +96 -0
- package/src/clips.js +18 -7
- package/src/conformance.js +119 -0
- package/src/emotions.js +4 -3
- package/src/face-core.js +27 -1
- package/src/face-myna.d.ts +7 -0
- package/src/face-myna.js +192 -131
- package/src/face-peep-control-plane.js +167 -0
- package/src/face-peep.d.ts +7 -0
- package/src/face-peep.js +274 -116
- package/src/face-wren.d.ts +7 -0
- package/src/face-wren.js +17 -19
- package/src/faces.d.ts +12 -0
- package/src/faces.js +53 -0
- package/src/gaze.js +119 -4
- package/src/hand.js +120 -91
- package/src/idle.js +18 -101
- package/src/interjections.js +156 -31
- package/src/params.js +8 -4
- package/src/perform.js +7 -9
- package/src/rig.d.ts +24 -0
- package/src/rig.js +32 -0
- package/src/speech-timing.js +23 -0
- package/src/visemes.js +28 -12
- package/client/dist/Avatar.d.ts +0 -27
- package/client/dist/Avatar.d.ts.map +0 -1
- package/client/dist/Avatar.js.map +0 -1
- package/client/dist/AvatarClient.d.ts +0 -139
- package/client/dist/AvatarClient.d.ts.map +0 -1
- package/client/dist/AvatarClient.js +0 -223
- package/client/dist/AvatarClient.js.map +0 -1
- package/client/dist/index.d.ts +0 -22
- package/client/dist/index.d.ts.map +0 -1
- package/client/dist/index.js +0 -22
- package/client/dist/index.js.map +0 -1
- package/client/dist/types.d.ts +0 -86
- package/client/dist/types.d.ts.map +0 -1
- package/client/dist/types.js +0 -31
- package/client/dist/types.js.map +0 -1
- package/client/dist/useAvatar.d.ts +0 -42
- package/client/dist/useAvatar.d.ts.map +0 -1
- package/client/dist/useAvatar.js +0 -51
- package/client/dist/useAvatar.js.map +0 -1
- package/client/src/Avatar.tsx +0 -33
- package/client/src/AvatarClient.ts +0 -270
- package/client/src/index.ts +0 -22
- package/client/src/types.ts +0 -104
- package/client/src/useAvatar.ts +0 -85
- package/docs/contract-avatar.md +0 -371
- package/docs/contract-protocol.md +0 -461
|
@@ -0,0 +1,508 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// src/live.js — the face, evaluated at runtime instead of blended from poses.
|
|
3
|
+
//
|
|
4
|
+
// Everything else in src/ plays a rig back: `rig.evaluate(weights)` blends
|
|
5
|
+
// baked geometry, and the only opinions in the runtime are how to interpolate
|
|
6
|
+
// and when to snap. This file is the one exception, and it exists because of a
|
|
7
|
+
// measurement rather than a preference.
|
|
8
|
+
//
|
|
9
|
+
// WHY. voqalize's mixer hands a renderer 30 continuous channels — a mouth that
|
|
10
|
+
// is 0.37 open and 0.62 wide with the left corner up — sixty times a second.
|
|
11
|
+
// Our rigs answer that with poses, and a pose set can only reach the points it
|
|
12
|
+
// was baked at plus the straight lines between them. The linearity spike
|
|
13
|
+
// (scratchpad parts/linearity/report.md) baked the nine Rhubarb letters and
|
|
14
|
+
// then asked how close a weighted blend of them gets to the real face at
|
|
15
|
+
// in-between channel values. The geometry is close but not identical (worst
|
|
16
|
+
// mouth vertex 9.7 px at 1x), and the ALPHA ramps are categorically wrong:
|
|
17
|
+
// `seam` off by 0.848, `teeth` by 0.612. Two poses at half weight each show
|
|
18
|
+
// half a set of teeth THROUGH the lip — the grey-teeth artefact anyone who has
|
|
19
|
+
// scrubbed the intensity slider on this rig has seen. Alpha is where a blend
|
|
20
|
+
// of two shapes stops being a shape.
|
|
21
|
+
//
|
|
22
|
+
// So: re-run the builder. `avatars/round/face.mjs` is the pure half of that
|
|
23
|
+
// avatar's generator — parameters, palette, landmarks, shape tables, both
|
|
24
|
+
// parts, `buildDraws` — and it is pure precisely so a browser can import it.
|
|
25
|
+
// This module calls it once per frame with the channels the mixer sent and
|
|
26
|
+
// writes the result over the evaluated display list. 60 us a frame at the time
|
|
27
|
+
// of writing, which is under 4% of a 16 ms budget.
|
|
28
|
+
//
|
|
29
|
+
// WHAT IT DOES NOT DO. It does not replace `rig.evaluate`: it calls it. Poses
|
|
30
|
+
// still own everything live geometry does not — the iris hue ladder (paint,
|
|
31
|
+
// which no channel of ours moves), the wardrobe's bitmap layers, any track a
|
|
32
|
+
// host still has running. Live draws are overwritten AFTER that blend, and
|
|
33
|
+
// only their `cmds` and `a`; paint, clip, blend mode and matrix come out of
|
|
34
|
+
// the normal path untouched. Nothing in rig.js, render2d.js or drivers.js
|
|
35
|
+
// changed to make this work.
|
|
36
|
+
//
|
|
37
|
+
// THE CONTRACT WITH rig.js, in three lines, because they are subtle:
|
|
38
|
+
// * write into `d._cmds`, never into `d.cmds` — the latter may be pointing
|
|
39
|
+
// at a POSE's array after a topology snap, and writing there corrupts the
|
|
40
|
+
// pose for every future frame.
|
|
41
|
+
// * set `d.gen = rig.frame` or render2d's Path2D cache serves last frame's
|
|
42
|
+
// shape, and `d._geoDirty = true` or `restore` never puts the base back.
|
|
43
|
+
// * `rig.dirty.add(i)` for every draw touched. `evaluate` restores what is
|
|
44
|
+
// in that set and nothing else, so a draw we wrote and did not register
|
|
45
|
+
// accumulates our writes forever.
|
|
46
|
+
//
|
|
47
|
+
// THE PERSONA RULE. A live face MUST be built from the same persona the rig
|
|
48
|
+
// was baked from, or every vertex belongs to a different character — a wider
|
|
49
|
+
// brow, a heavier lash, a different iris solve. The persona is not derivable
|
|
50
|
+
// from the JSON, so it is baked into `meta.live.persona` by the generator and
|
|
51
|
+
// read back here. `opts.persona` overrides it, which is only ever right if you
|
|
52
|
+
// are deliberately drawing somebody else.
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
import { MORPH_AXES } from './vocab.js';
|
|
56
|
+
|
|
57
|
+
// The six identity axes as the driver names them, against the control channel
|
|
58
|
+
// each one spends. The poses are `morph/<axis>_100` / `_-100` and drivers.js
|
|
59
|
+
// sends a weight for one of the pair; live geometry has to read the same dial
|
|
60
|
+
// or a morph slider moves the baked half of the face and not the live half.
|
|
61
|
+
// The order is MORPH_AXES', so this is a lookup rather than a second list.
|
|
62
|
+
const MORPH_CHANNEL = { head: 'headW', lips: 'lipFull', nose: 'noseW', brows: 'browH', eyes: 'eyeSize', distance: 'eyeSpace' };
|
|
63
|
+
|
|
64
|
+
// A 2x3 affine compose, same convention and same six numbers as render2d's.
|
|
65
|
+
// Duplicated rather than exported from there: this file may not modify the
|
|
66
|
+
// three runtime modules, and six lines of arithmetic is a cheaper coupling
|
|
67
|
+
// than a new export in a file that is under a byte-identity gate.
|
|
68
|
+
const mul = (A, B, o) => {
|
|
69
|
+
const a = A[0] * B[0] + A[2] * B[1];
|
|
70
|
+
const b = A[1] * B[0] + A[3] * B[1];
|
|
71
|
+
const c = A[0] * B[2] + A[2] * B[3];
|
|
72
|
+
const d = A[1] * B[2] + A[3] * B[3];
|
|
73
|
+
const e = A[0] * B[4] + A[2] * B[5] + A[4];
|
|
74
|
+
const f = A[1] * B[4] + A[3] * B[5] + A[5];
|
|
75
|
+
o[0] = a; o[1] = b; o[2] = c; o[3] = d; o[4] = e; o[5] = f;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const num = (v, d) => (typeof v === 'number' && v === v ? v : d);
|
|
79
|
+
|
|
80
|
+
// ---------------------------------------------------------------------------
|
|
81
|
+
// THE CHANNEL MAP. 30 in, one control vector out. Everything here is a rename
|
|
82
|
+
// except the head block, and the renames are 1:1 because both ends were
|
|
83
|
+
// written against the same table (author/parts/{mouth,eye}.mjs headers quote
|
|
84
|
+
// voqalize's params.js line by line, ranges included). In particular
|
|
85
|
+
// `mouthCornerL/R` are NOT rescaled: our channel is the same -1.4..1.4
|
|
86
|
+
// expression scalar, spent at MAP.CORNER_PX = 32 px per unit at the corner.
|
|
87
|
+
//
|
|
88
|
+
// SIDEDNESS. voqalize draws `lidL` at `CX - EYE.dx` (face-peep.js:736), i.e.
|
|
89
|
+
// the VIEWER's left. This rig's `side < 0` is the viewer's left, wears the
|
|
90
|
+
// `L` slot suffix and reads `c.eyeL` through `eyeSide(c, side)`. So L is L and
|
|
91
|
+
// R is R with no flip. `pupilX/Y` are shared and unmirrored on both sides,
|
|
92
|
+
// which is what makes a pair of eyes look at one point instead of crossing.
|
|
93
|
+
//
|
|
94
|
+
// PART-LOCAL CHANNELS STAY AT REST. `tongueUp`, `curveUp` and `outerDroop`
|
|
95
|
+
// are this rig's own inventions; no mixer channel means them and inferring
|
|
96
|
+
// them (a closed happy eye implies a curve) would be this file quietly
|
|
97
|
+
// authoring expression on top of a pose it was handed. The eye STATES bake
|
|
98
|
+
// those; the live path leaves them where `face.ctrl()` put them.
|
|
99
|
+
//
|
|
100
|
+
// …WITH ONE EXCEPTION, AND IT IS NOT INFERENCE. `cheekRaise` is the one
|
|
101
|
+
// part-local channel that is not an opinion about what the pose MEANS: it is
|
|
102
|
+
// where the mesh has to move for the pose it was already sent to be drawable.
|
|
103
|
+
// `squint` is defined as the lower lid raised — and the thing that raises a
|
|
104
|
+
// lower lid is a cheek, so a squint whose cheek does not move reads as the lid
|
|
105
|
+
// having been trimmed rather than lifted (author/parts/eye.mjs says the same
|
|
106
|
+
// thing about `squintShade`). A lifted mouth corner is the same fact from the
|
|
107
|
+
// other end: the corner is pulled by zygomaticus, which passes over the cheek.
|
|
108
|
+
// So the cheek follows both, at less than either, and nothing here decides
|
|
109
|
+
// that a smile is HAPPY or that a lifted corner means anything at all. An
|
|
110
|
+
// explicit `controls.cheekRaise` still wins, because `controls` is folded in
|
|
111
|
+
// after this runs.
|
|
112
|
+
//
|
|
113
|
+
// THE TRUNK. `torsoLean`, `shoulderL`, `shoulderR` and `torsoTurn` were
|
|
114
|
+
// ignored here until now, on the grounds that this character's body is two
|
|
115
|
+
// flat shapes with no arm and no shoulder landmark. That was the wrong reason:
|
|
116
|
+
// voqalize's own peep has no arm either, and it spends all four — because none
|
|
117
|
+
// of them needs one. A lean is a change of SCALE (a figure that leans toward a
|
|
118
|
+
// webcam gets bigger, and that is nearly all of what the viewer sees); a shrug
|
|
119
|
+
// is the torso translating up; a one-sided shrug is the same shape rotated a
|
|
120
|
+
// degree and a half about the sternum, which is exactly what you get when the
|
|
121
|
+
// torso is one filled path. So the four are spent the way `poseTransforms`
|
|
122
|
+
// spends them, with each avatar's own travels in `face.mjs BODY_LIVE`, and the
|
|
123
|
+
// state that made this visible — STRAINING, which is voqalize's `CANT_HEAR`
|
|
124
|
+
// and holds `torsoLean` at 0.70 — leans in instead of standing straight.
|
|
125
|
+
//
|
|
126
|
+
// AND THE TRUNK'S SHARE OF THE HEAD'S OWN CHANNELS, which is the rest of what
|
|
127
|
+
// `poseTransforms` does and was left out the first time round. Three things,
|
|
128
|
+
// all peep's and all converted rather than re-tuned:
|
|
129
|
+
//
|
|
130
|
+
// * a YAW PARALLAX. peep's body layer sits at `parallax 0.1` against a head
|
|
131
|
+
// at 1.0, so a head turn drags the trunk a tenth as far. That is on top of
|
|
132
|
+
// `torsoTurn`, which the mixer retargets to `headYaw*0.45` on a much
|
|
133
|
+
// slower tau: the parallax is the part of the trunk's answer that is
|
|
134
|
+
// instant, `torsoTurn` is the part that lags.
|
|
135
|
+
// * a ROLL, about the SAME pivot the head rolls about, at peep's
|
|
136
|
+
// `ROLL_TORSO/ROLL_HEAD` = 1.5/5.5 of the head's angle. A head tips and
|
|
137
|
+
// the shoulders under it tip a little, or the neck reads as a hinge.
|
|
138
|
+
// * BREATH AS A SWELL rather than a slide — see the block that builds it.
|
|
139
|
+
//
|
|
140
|
+
// `breath` IS spent, on the same cycle the `breathing` track drives — a host
|
|
141
|
+
// that sends `breath` should stop that track or the two add up.
|
|
142
|
+
// ---------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
function writeChannels(c, pose, HL) {
|
|
145
|
+
const m = c.mouth;
|
|
146
|
+
m.open = num(pose.mouthOpen, m.open);
|
|
147
|
+
m.width = num(pose.mouthWidth, m.width);
|
|
148
|
+
m.round = num(pose.mouthRound, m.round);
|
|
149
|
+
m.press = num(pose.mouthPress, m.press);
|
|
150
|
+
m.tuck = num(pose.mouthTuck, m.tuck);
|
|
151
|
+
m.cornerL = num(pose.mouthCornerL, m.cornerL);
|
|
152
|
+
m.cornerR = num(pose.mouthCornerR, m.cornerR);
|
|
153
|
+
m.teeth = num(pose.teethUpper, m.teeth);
|
|
154
|
+
m.tongue = num(pose.tongue, m.tongue);
|
|
155
|
+
c.jaw = num(pose.jaw, c.jaw);
|
|
156
|
+
|
|
157
|
+
// Shared gaze on `eye`, everything else per side on `eyeL`/`eyeR`. The two
|
|
158
|
+
// side blocks are written in full rather than patched, so a channel that
|
|
159
|
+
// goes back to rest actually goes back to rest.
|
|
160
|
+
const e = c.eye;
|
|
161
|
+
e.pupilX = num(pose.pupilX, e.pupilX);
|
|
162
|
+
e.pupilY = num(pose.pupilY, e.pupilY);
|
|
163
|
+
const L = c.eyeL, R = c.eyeR;
|
|
164
|
+
L.lid = num(pose.lidL, e.lid); R.lid = num(pose.lidR, e.lid);
|
|
165
|
+
L.squint = num(pose.squintL, e.squint); R.squint = num(pose.squintR, e.squint);
|
|
166
|
+
L.browRaise = num(pose.browRaiseL, e.browRaise); R.browRaise = num(pose.browRaiseR, e.browRaise);
|
|
167
|
+
L.browAngle = num(pose.browAngleL, e.browAngle); R.browAngle = num(pose.browAngleR, e.browAngle);
|
|
168
|
+
L.browInner = num(pose.browInnerL, e.browInner); R.browInner = num(pose.browInnerR, e.browInner);
|
|
169
|
+
|
|
170
|
+
// The cheek. Zero at rest — `face.ctrl()`'s own value — and both terms are
|
|
171
|
+
// one-sided: a NEGATIVE corner (a frown) does not push a cheek up, and a
|
|
172
|
+
// negative squint (a lid pushed down, which the range allows) does not
|
|
173
|
+
// either. Coefficients under 1 because the cheek is downstream of both and
|
|
174
|
+
// moves less than the thing pulling it; the avatars spend it at 13 px
|
|
175
|
+
// (round) and 12 px (ink) of mesh at 1.0.
|
|
176
|
+
c.cheekRaise = Math.max(0, Math.min(1,
|
|
177
|
+
0.55 * ((L.squint + R.squint) / 2) + 0.45 * Math.max(0, (m.cornerL + m.cornerR) / 2)));
|
|
178
|
+
|
|
179
|
+
// The head block is not geometry: this rig has no head poses and never had
|
|
180
|
+
// any — head motion has always been six numbers written onto every head
|
|
181
|
+
// draw's matrix (author/rig.mjs, headMatFactory). So the four channels that
|
|
182
|
+
// move the head become exactly that matrix, in the same units the tracks are
|
|
183
|
+
// keyed in. `HEAD_LIVE` in face.mjs is where those units live, because how
|
|
184
|
+
// far a head turns is the character's business and not this file's.
|
|
185
|
+
const yaw = num(pose.headYaw, 0), pitch = num(pose.headPitch, 0);
|
|
186
|
+
const roll = num(pose.headRoll, 0), breath = num(pose.breath, 0);
|
|
187
|
+
//
|
|
188
|
+
// The trunk is four more, and they are NOT geometry either: same story, one
|
|
189
|
+
// matrix down. `shrug` and `tilt` are the mean and the half-difference of
|
|
190
|
+
// the two shoulders, which is voqalize's decomposition (face-core.js) and
|
|
191
|
+
// the only one a single filled torso can draw.
|
|
192
|
+
const shL = num(pose.shoulderL, 0), shR = num(pose.shoulderR, 0);
|
|
193
|
+
return {
|
|
194
|
+
deg: roll * HL.rollDeg,
|
|
195
|
+
tx: yaw * HL.yawPx,
|
|
196
|
+
ty: pitch * HL.pitchPx,
|
|
197
|
+
// The three channels the TRUNK reads raw, because what it does with them
|
|
198
|
+
// is a fraction of what the head does and the fraction is `BODY_LIVE`'s
|
|
199
|
+
// to state: `yaw` is a parallax, `roll` a shallower roll about the same
|
|
200
|
+
// pivot, `breath` a swell about the hem.
|
|
201
|
+
yaw, roll, breath,
|
|
202
|
+
lean: num(pose.torsoLean, 0),
|
|
203
|
+
turn: num(pose.torsoTurn, 0),
|
|
204
|
+
shrug: (shL + shR) / 2,
|
|
205
|
+
tilt: (shR - shL) / 2,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// The channel vector at rest, as a plain object — what a host that has no
|
|
210
|
+
// mixer yet (the player's live panel) initialises its sliders from, and what
|
|
211
|
+
// `apply({})` is equivalent to.
|
|
212
|
+
export const REST_POSE = Object.freeze({
|
|
213
|
+
mouthOpen: 0.02, mouthWidth: 0.42, mouthRound: 0.10, mouthPress: 0.15, mouthTuck: 0,
|
|
214
|
+
mouthCornerL: 0, mouthCornerR: 0, teethUpper: 0, tongue: 0, jaw: 0,
|
|
215
|
+
lidL: 0.12, lidR: 0.12, squintL: 0, squintR: 0, pupilX: 0, pupilY: 0.05,
|
|
216
|
+
browRaiseL: 0, browRaiseR: 0, browAngleL: 0, browAngleR: 0, browInnerL: 0, browInnerR: 0,
|
|
217
|
+
headYaw: 0, headPitch: 0, headRoll: 0, breath: 0,
|
|
218
|
+
shoulderL: 0, shoulderR: 0, torsoLean: 0, torsoTurn: 0,
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
export const LIVE_CHANNELS = Object.keys(REST_POSE);
|
|
222
|
+
|
|
223
|
+
// ---------------------------------------------------------------------------
|
|
224
|
+
// createLive(rig, face, opts) -> { apply, destroy, ... }
|
|
225
|
+
//
|
|
226
|
+
// rig a loaded Rig whose `meta.live` says it can do this
|
|
227
|
+
// face the MODULE `meta.live.face` names, imported by the host (a rig
|
|
228
|
+
// cannot import; only the host knows where it is serving from)
|
|
229
|
+
// opts { persona } to override the baked one
|
|
230
|
+
//
|
|
231
|
+
// `apply(pose, weights, controls, hand)` is the whole of the API:
|
|
232
|
+
// pose the 30 channels, already smoothed and clamped by the mixer
|
|
233
|
+
// weights the pose weights this frame, exactly as you would have passed
|
|
234
|
+
// them to `rig.evaluate` — live calls it for you, because the
|
|
235
|
+
// order matters and owning it here is one less rule for a host
|
|
236
|
+
// controls extra control-vector fields (identity morphs, mostly) to fold
|
|
237
|
+
// in. The six `morph/*` weights are picked up automatically, so
|
|
238
|
+
// this is only for a host that drives morphs some other way.
|
|
239
|
+
// hand voqalize's `HandFrame` — `{ gesture, progress, side }` — or
|
|
240
|
+
// nothing. OPTIONAL at both ends: a rig whose face has no hand
|
|
241
|
+
// ignores it, and a face that has one draws nothing without it.
|
|
242
|
+
// The mixer owns the clock; `progress` is the only thing that
|
|
243
|
+
// crosses this seam, and it is clamped here because a host that
|
|
244
|
+
// overshoots its own duration should get a parked hand rather
|
|
245
|
+
// than a sampled table read off the end.
|
|
246
|
+
// ---------------------------------------------------------------------------
|
|
247
|
+
|
|
248
|
+
export function createLive(rig, face, opts = {}) {
|
|
249
|
+
const meta = rig.meta && rig.meta.live;
|
|
250
|
+
if (!meta) throw new Error('createLive: this rig has no meta.live');
|
|
251
|
+
const persona = opts.persona || meta.persona || {};
|
|
252
|
+
const kit = face.makeKit(persona);
|
|
253
|
+
// THE REST RULE, the persona rule's other half. A face with a `sex` axis is
|
|
254
|
+
// BUILT at a rest vector of its own (face.mjs SEX_GEO / ctrlFor), so live
|
|
255
|
+
// evaluation has to start from the same one or a male rig would animate off
|
|
256
|
+
// a woman's skull and every frame would fight the baked rest. Families that
|
|
257
|
+
// have no sex axis do not export `ctrlFor`, and `face.ctrl` is the answer.
|
|
258
|
+
const mkCtrl = face.ctrlFor ? face.ctrlFor(persona) : face.ctrl;
|
|
259
|
+
// This persona's baseline for the six morph sliders. They are an EXCURSION
|
|
260
|
+
// from it, not an absolute channel value — otherwise a slider at 0 would
|
|
261
|
+
// reset a male rig's browH/headW to the family neutral and rest would stop
|
|
262
|
+
// matching the baked face. author/finish.mjs writes the baked pair the same
|
|
263
|
+
// way. All-zero for a family with no sex axis, so nothing else changes.
|
|
264
|
+
const REST_C = mkCtrl();
|
|
265
|
+
const HL = face.HEAD_LIVE;
|
|
266
|
+
|
|
267
|
+
// slot -> draw index, over the FINISHED rig: a dressed variant has bitmap
|
|
268
|
+
// layers inserted into the middle of the list, so every index shifts and
|
|
269
|
+
// nothing but the slot name survives the wardrobe. Which is why this looks
|
|
270
|
+
// the draws up by name rather than baking a table at finish time.
|
|
271
|
+
const idx = new Map();
|
|
272
|
+
rig.data.draws.forEach((d, i) => { if (d.slot != null && !idx.has(d.slot)) idx.set(d.slot, i); });
|
|
273
|
+
|
|
274
|
+
// What the face draws at rest, used three ways: to resolve slot -> index
|
|
275
|
+
// once, to check that the runtime's topology matches the baked one, and to
|
|
276
|
+
// spot the draws a wardrobe has hidden.
|
|
277
|
+
const restDraws = face.buildDraws(mkCtrl(), kit);
|
|
278
|
+
const own = []; // [drawIndex, buildIndex] pairs, in draw order
|
|
279
|
+
const hidden = [];
|
|
280
|
+
restDraws.forEach((s, j) => {
|
|
281
|
+
const i = idx.get(s.slot);
|
|
282
|
+
if (i === undefined) throw new Error(`createLive: the rig has no draw "${s.slot}" — face.mjs and this rig were built from different code`);
|
|
283
|
+
const b = rig.base[i];
|
|
284
|
+
if (!b.cmds || b.cmds.length !== s.cmds.length) {
|
|
285
|
+
throw new Error(`createLive: draw "${s.slot}" has ${b.cmds ? b.cmds.length : 0} opcodes baked and ${s.cmds.length} live`);
|
|
286
|
+
}
|
|
287
|
+
// A draw the sidecar hid is at alpha 0 in the rig and non-zero in the
|
|
288
|
+
// builder, because the builder has never heard of the wardrobe. Anything
|
|
289
|
+
// that is zero at rest in BOTH (teeth, tongue, the squint shadows) is a
|
|
290
|
+
// channel waiting to be spent and is very much ours to write.
|
|
291
|
+
if (b.a === 0 && s.a > 0) { hidden.push(s.slot); return; }
|
|
292
|
+
own.push(i, j);
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
// Head and body, for the head matrix. Body is `meta.live.body`, the slots
|
|
296
|
+
// the generator's own head matrix exempts, plus any wardrobe layer riding
|
|
297
|
+
// one of them; everything else in the rig — including the bitmap hair and
|
|
298
|
+
// the glasses, which follow `face` — turns with the head.
|
|
299
|
+
const bodySlots = new Set(meta.body || []);
|
|
300
|
+
// ...and the HAND, which is on neither. `meta.live.hand.slots` is the third
|
|
301
|
+
// group: a secondary-gesture hand (author/parts/hand.mjs) is not on the
|
|
302
|
+
// character at all, it is the nearest object in the FRAME, placed by the
|
|
303
|
+
// camera window's own numbers. A head that turns must not carry it round and
|
|
304
|
+
// a breath must not lift it, so it is excluded from both lists rather than
|
|
305
|
+
// falling into `head` by default. A rig without a hand has an empty set here
|
|
306
|
+
// and the split is the one it always was.
|
|
307
|
+
const handSlots = new Set((meta.hand && meta.hand.slots) || []);
|
|
308
|
+
const head = [], body = [];
|
|
309
|
+
rig.data.draws.forEach((d, i) => {
|
|
310
|
+
if (handSlots.has(d.slot)) return;
|
|
311
|
+
(bodySlots.has(d.slot) ? body : head).push(i);
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
// One control vector, mutated in place forever. `mkCtrl()` clones a nested
|
|
315
|
+
// rest object through JSON, which is not something to do 60 times a second,
|
|
316
|
+
// and `buildDraws` never writes to what it is given.
|
|
317
|
+
const c = mkCtrl();
|
|
318
|
+
const M = new Float32Array(6);
|
|
319
|
+
const T = new Float32Array(6);
|
|
320
|
+
const B = new Float32Array(6);
|
|
321
|
+
// Two more scratch matrices for the trunk. They are folded into `T` before
|
|
322
|
+
// anything is written to a draw, so the per-draw cost is still one compose.
|
|
323
|
+
const S = new Float32Array(6);
|
|
324
|
+
const D = new Float32Array(6);
|
|
325
|
+
// The trunk's travels, or nothing. A face that predates `BODY_LIVE` (facet,
|
|
326
|
+
// which has no `meta.live` at all, is the only one in the tree) keeps the
|
|
327
|
+
// behaviour it had: the four body channels arrive and are dropped.
|
|
328
|
+
const BL = face.BODY_LIVE || null;
|
|
329
|
+
let alive = true;
|
|
330
|
+
|
|
331
|
+
const touched = new Set();
|
|
332
|
+
for (let k = 0; k < own.length; k += 2) touched.add(own[k]);
|
|
333
|
+
|
|
334
|
+
function apply(pose = REST_POSE, weights = {}, controls = null, hand = null) {
|
|
335
|
+
if (!alive) throw new Error('createLive: apply() after destroy()');
|
|
336
|
+
const H = writeChannels(c, pose, HL);
|
|
337
|
+
|
|
338
|
+
// The hand block, guarded by the face having one rather than by the rig's
|
|
339
|
+
// meta: `c.hand` exists exactly when this character can gesture. Written
|
|
340
|
+
// in full every frame, so a gesture that ends actually ends, and an
|
|
341
|
+
// unknown name lands as `null` — which the part draws as no hand.
|
|
342
|
+
if (c.hand) {
|
|
343
|
+
c.hand.gesture = hand && typeof hand.gesture === 'string' ? hand.gesture : null;
|
|
344
|
+
c.hand.progress = hand ? Math.min(Math.max(num(hand.progress, 0), 0), 1) : 0;
|
|
345
|
+
c.hand.side = hand && hand.side === 'left' ? 'left' : 'right';
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// Identity morphs off the pose weights, so the player's morph sliders
|
|
349
|
+
// move the live face and the baked one together. drivers.js sends one of
|
|
350
|
+
// each +/-100 pair; taking the difference costs nothing and is right even
|
|
351
|
+
// if some future host sends both.
|
|
352
|
+
for (const axis of MORPH_AXES) {
|
|
353
|
+
const ch = MORPH_CHANNEL[axis];
|
|
354
|
+
c[ch] = (REST_C[ch] || 0)
|
|
355
|
+
+ (weights[`morph/${axis}_100`] || 0) - (weights[`morph/${axis}_-100`] || 0);
|
|
356
|
+
}
|
|
357
|
+
if (controls) for (const k of Object.keys(controls)) {
|
|
358
|
+
const v = controls[k];
|
|
359
|
+
if (v && typeof v === 'object' && !Array.isArray(v)) Object.assign(c[k], v);
|
|
360
|
+
else c[k] = v;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const out = rig.evaluate(weights);
|
|
364
|
+
|
|
365
|
+
// ---- the live half of the display list -------------------------------
|
|
366
|
+
const dl = face.buildDraws(c, kit);
|
|
367
|
+
const frame = rig.frame;
|
|
368
|
+
for (let k = 0; k < own.length; k += 2) {
|
|
369
|
+
const d = out[own[k]], s = dl[own[k + 1]];
|
|
370
|
+
d.cmds = d._cmds;
|
|
371
|
+
d._cmds.set(s.cmds);
|
|
372
|
+
d.a = s.a;
|
|
373
|
+
d.gen = frame;
|
|
374
|
+
d._geoDirty = true;
|
|
375
|
+
rig.dirty.add(own[k]);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// ---- breath ----------------------------------------------------------
|
|
379
|
+
// A SWELL, not a slide. peep's own note is the argument: a rigid vertical
|
|
380
|
+
// bob of the whole shirt "moved the hem — and the hem is the one part of a
|
|
381
|
+
// seated torso that does not move, so the result read as the figure being
|
|
382
|
+
// nudged up and down rather than as breath" (face-core.js). So the torso
|
|
383
|
+
// SCALES about its hem, which raises the shoulder line and widens the
|
|
384
|
+
// chest, and the head then rides the displacement that scale produces AT
|
|
385
|
+
// THE NECK PIVOT. That lift is arithmetic and not a second tuned number:
|
|
386
|
+
// author one swell and the two layers cannot drift apart. A face with no
|
|
387
|
+
// `breathSwell` keeps the flat pair of translates it was keyed with.
|
|
388
|
+
const swell = BL && BL.breathSwell ? H.breath * BL.breathSwell : 0;
|
|
389
|
+
const bodyTy = swell ? 0 : H.breath * HL.breathBodyTy;
|
|
390
|
+
const headTy = H.ty + (swell
|
|
391
|
+
? -swell * (BL.swellPivot[1] - HL.pivot[1])
|
|
392
|
+
: H.breath * HL.breathTy);
|
|
393
|
+
|
|
394
|
+
// ---- and the head it is on -------------------------------------------
|
|
395
|
+
if (H.deg || H.tx || headTy) {
|
|
396
|
+
const r = (H.deg * Math.PI) / 180, cs = Math.cos(r), sn = Math.sin(r);
|
|
397
|
+
const p = HL.pivot;
|
|
398
|
+
M[0] = cs; M[1] = sn; M[2] = -sn; M[3] = cs;
|
|
399
|
+
M[4] = p[0] - (cs * p[0] - sn * p[1]) + H.tx;
|
|
400
|
+
M[5] = p[1] - (sn * p[0] + cs * p[1]) + headTy;
|
|
401
|
+
// Composed onto whatever the blend produced rather than replacing it:
|
|
402
|
+
// a bitmap layer's matrix is `head . fit` and a track may already have
|
|
403
|
+
// turned the head, and both have to survive.
|
|
404
|
+
for (let k = 0; k < head.length; k++) {
|
|
405
|
+
const i = head[k];
|
|
406
|
+
mul(M, out[i].m, out[i].m);
|
|
407
|
+
rig.dirty.add(i); touched.add(i);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
if (!BL && bodyTy) {
|
|
411
|
+
for (let k = 0; k < body.length; k++) {
|
|
412
|
+
const i = body[k];
|
|
413
|
+
out[i].m[5] += bodyTy;
|
|
414
|
+
rig.dirty.add(i); touched.add(i);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// ---- the torso --------------------------------------------------------
|
|
419
|
+
// Torso only, which is `meta.live.body` — peep's `torsoLayers` is `['body']`
|
|
420
|
+
// and nothing above its collar sees any of this. Composed OUTSIDE the head
|
|
421
|
+
// matrix, so a head that has already turned is carried by the torso rather
|
|
422
|
+
// than fighting it. peep writes the same product as a transform list:
|
|
423
|
+
//
|
|
424
|
+
// torsoT( turn, shrug, tilt, swell ) · translate(parallax) · rotate(roll)
|
|
425
|
+
//
|
|
426
|
+
// and the two halves stay in that order here. The whole product is folded
|
|
427
|
+
// into `T` first, so however many pieces are live a body draw still pays
|
|
428
|
+
// exactly one 2x3 compose.
|
|
429
|
+
const rollT = BL ? H.roll * (BL.rollDeg || 0) : 0;
|
|
430
|
+
const yawT = BL ? H.yaw * (BL.yawPx || 0) : 0;
|
|
431
|
+
if (BL && (H.turn || H.shrug || H.tilt || swell || rollT || yawT || bodyTy)) {
|
|
432
|
+
const r = (-H.tilt * BL.shrugTiltDeg * Math.PI) / 180, cs = Math.cos(r), sn = Math.sin(r);
|
|
433
|
+
const q = BL.shrugPivot;
|
|
434
|
+
T[0] = cs; T[1] = sn; T[2] = -sn; T[3] = cs;
|
|
435
|
+
T[4] = q[0] - (cs * q[0] - sn * q[1]) + H.turn * BL.turnPx;
|
|
436
|
+
T[5] = q[1] - (sn * q[0] + cs * q[1]) - H.shrug * BL.shrugLift + bodyTy;
|
|
437
|
+
if (swell) {
|
|
438
|
+
const g = 1 + swell, h = BL.swellPivot;
|
|
439
|
+
S[0] = g; S[1] = 0; S[2] = 0; S[3] = g;
|
|
440
|
+
S[4] = h[0] - g * h[0]; S[5] = h[1] - g * h[1];
|
|
441
|
+
mul(T, S, T);
|
|
442
|
+
}
|
|
443
|
+
// The trunk's share of the head's own two channels: ~10% of the yaw as
|
|
444
|
+
// parallax, and a roll about the SAME pivot the head rolls about, at a
|
|
445
|
+
// fraction of the angle. Both are peep's, and both are inside the shrug
|
|
446
|
+
// and the swell for the same reason they are in peep — they belong to
|
|
447
|
+
// the trunk's own pose, not to what the shoulders are doing to it.
|
|
448
|
+
if (rollT || yawT) {
|
|
449
|
+
const rr = (rollT * Math.PI) / 180, c2 = Math.cos(rr), s2 = Math.sin(rr);
|
|
450
|
+
const p = HL.pivot;
|
|
451
|
+
D[0] = c2; D[1] = s2; D[2] = -s2; D[3] = c2;
|
|
452
|
+
D[4] = p[0] - (c2 * p[0] - s2 * p[1]) + yawT;
|
|
453
|
+
D[5] = p[1] - (s2 * p[0] + c2 * p[1]);
|
|
454
|
+
mul(T, D, T);
|
|
455
|
+
}
|
|
456
|
+
for (let k = 0; k < body.length; k++) {
|
|
457
|
+
const i = body[k];
|
|
458
|
+
mul(T, out[i].m, out[i].m);
|
|
459
|
+
rig.dirty.add(i); touched.add(i);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// ---- and the lean, which the whole figure takes ----------------------
|
|
464
|
+
// Head AND body, in that order and outermost of everything, because peep
|
|
465
|
+
// prefixes its lean transform to every layer it draws. The hand is on
|
|
466
|
+
// neither list and stays out of it: a secondary-gesture hand is the
|
|
467
|
+
// nearest object in the FRAME rather than a limb of this character, and
|
|
468
|
+
// voqalize's hand overlay reads no body channel either.
|
|
469
|
+
if (BL && H.lean) {
|
|
470
|
+
const sc = 1 + H.lean * BL.leanScale;
|
|
471
|
+
const q = BL.leanPivot;
|
|
472
|
+
B[0] = sc; B[1] = 0; B[2] = 0; B[3] = sc;
|
|
473
|
+
B[4] = q[0] - sc * q[0];
|
|
474
|
+
B[5] = q[1] - sc * q[1] + H.lean * BL.leanTravel;
|
|
475
|
+
for (let k = 0; k < head.length; k++) {
|
|
476
|
+
const i = head[k];
|
|
477
|
+
mul(B, out[i].m, out[i].m);
|
|
478
|
+
rig.dirty.add(i); touched.add(i);
|
|
479
|
+
}
|
|
480
|
+
for (let k = 0; k < body.length; k++) {
|
|
481
|
+
const i = body[k];
|
|
482
|
+
mul(B, out[i].m, out[i].m);
|
|
483
|
+
rig.dirty.add(i); touched.add(i);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
return out;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
// Hand the rig back. Everything live ever wrote is registered dirty, so the
|
|
490
|
+
// next `rig.evaluate` restores it; a host that stops driving live and keeps
|
|
491
|
+
// playing gets the baked face back on the following frame.
|
|
492
|
+
function destroy() {
|
|
493
|
+
alive = false;
|
|
494
|
+
for (const i of touched) rig.dirty.add(i);
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
return {
|
|
498
|
+
apply,
|
|
499
|
+
destroy,
|
|
500
|
+
// Diagnostics, and what the player's live panel reads.
|
|
501
|
+
rest: REST_POSE,
|
|
502
|
+
channels: LIVE_CHANNELS,
|
|
503
|
+
persona,
|
|
504
|
+
// Draw indices this evaluator owns, and the slots a wardrobe took off it.
|
|
505
|
+
draws: own.filter((_, k) => k % 2 === 0),
|
|
506
|
+
hidden,
|
|
507
|
+
};
|
|
508
|
+
}
|