@voqalize/avatar 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -154
- package/dist/index.d.ts +37 -26
- package/dist/index.js +48 -25
- package/dist/pin.d.ts +2 -0
- package/dist/pin.js +3 -0
- package/dist/react.d.ts +21 -6
- package/dist/react.js +24 -6
- package/package.json +8 -67
- package/LICENSE-APACHE-2.0 +0 -201
- package/LICENSE-CC-BY-4.0 +0 -396
- package/assets/README.md +0 -60
- package/assets/tanvi.glb +0 -0
- package/assets/tanya.glb +0 -0
- package/assets/tara.glb +0 -0
- package/assets/tess.glb +0 -0
- package/assets/tushar.glb +0 -0
- package/client/Avatar.tsx +0 -57
- package/client/AvatarClient.ts +0 -818
- package/client/createAvatar.ts +0 -177
- package/client/index.ts +0 -44
- package/client/internal.ts +0 -132
- package/client/playout.ts +0 -95
- package/client/react.ts +0 -13
- package/client/supports.ts +0 -27
- package/client/three/assets.ts +0 -26
- package/client/three/budgets.ts +0 -27
- package/client/three/character-rig.ts +0 -1746
- package/client/three/createCharacter.ts +0 -109
- package/client/three/holds.ts +0 -69
- package/client/three/internal.ts +0 -55
- package/client/three/motion-limits.json +0 -124
- package/client/three/sequences.ts +0 -281
- package/client/three/tanvi-asset.ts +0 -6
- package/client/three/tanvi.ts +0 -30
- package/client/three/tanya-asset.ts +0 -6
- package/client/three/tanya.ts +0 -29
- package/client/three/tara-asset.ts +0 -25
- package/client/three/tara.ts +0 -36
- package/client/three/tess-asset.ts +0 -6
- package/client/three/tess.ts +0 -30
- package/client/three/tushar-asset.ts +0 -6
- package/client/three/tushar.ts +0 -28
- package/client/types.ts +0 -235
- package/client/useAvatar.ts +0 -74
- package/dist/Avatar.d.ts +0 -44
- package/dist/Avatar.d.ts.map +0 -1
- package/dist/Avatar.js +0 -7
- package/dist/Avatar.js.map +0 -1
- package/dist/AvatarClient.d.ts +0 -274
- package/dist/AvatarClient.d.ts.map +0 -1
- package/dist/AvatarClient.js +0 -712
- package/dist/AvatarClient.js.map +0 -1
- package/dist/createAvatar.d.ts +0 -135
- package/dist/createAvatar.d.ts.map +0 -1
- package/dist/createAvatar.js +0 -75
- package/dist/createAvatar.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/internal.d.ts +0 -34
- package/dist/internal.d.ts.map +0 -1
- package/dist/internal.js +0 -56
- package/dist/internal.js.map +0 -1
- package/dist/playout.d.ts +0 -42
- package/dist/playout.d.ts.map +0 -1
- package/dist/playout.js +0 -85
- package/dist/playout.js.map +0 -1
- package/dist/react.d.ts.map +0 -1
- package/dist/react.js.map +0 -1
- package/dist/supports.d.ts +0 -19
- package/dist/supports.d.ts.map +0 -1
- package/dist/supports.js +0 -25
- package/dist/supports.js.map +0 -1
- package/dist/three/assets.d.ts +0 -10
- package/dist/three/assets.d.ts.map +0 -1
- package/dist/three/assets.js +0 -23
- package/dist/three/assets.js.map +0 -1
- package/dist/three/budgets.d.ts +0 -17
- package/dist/three/budgets.d.ts.map +0 -1
- package/dist/three/budgets.js +0 -21
- package/dist/three/budgets.js.map +0 -1
- package/dist/three/character-rig.d.ts +0 -366
- package/dist/three/character-rig.d.ts.map +0 -1
- package/dist/three/character-rig.js +0 -1644
- package/dist/three/character-rig.js.map +0 -1
- package/dist/three/createCharacter.d.ts +0 -60
- package/dist/three/createCharacter.d.ts.map +0 -1
- package/dist/three/createCharacter.js +0 -84
- package/dist/three/createCharacter.js.map +0 -1
- package/dist/three/holds.d.ts +0 -38
- package/dist/three/holds.d.ts.map +0 -1
- package/dist/three/holds.js +0 -61
- package/dist/three/holds.js.map +0 -1
- package/dist/three/internal.d.ts +0 -52
- package/dist/three/internal.d.ts.map +0 -1
- package/dist/three/internal.js +0 -49
- package/dist/three/internal.js.map +0 -1
- package/dist/three/motion-limits.json +0 -124
- package/dist/three/sequences.d.ts +0 -71
- package/dist/three/sequences.d.ts.map +0 -1
- package/dist/three/sequences.js +0 -262
- package/dist/three/sequences.js.map +0 -1
- package/dist/three/tanvi-asset.d.ts +0 -7
- package/dist/three/tanvi-asset.d.ts.map +0 -1
- package/dist/three/tanvi-asset.js +0 -7
- package/dist/three/tanvi-asset.js.map +0 -1
- package/dist/three/tanvi.d.ts +0 -24
- package/dist/three/tanvi.d.ts.map +0 -1
- package/dist/three/tanvi.js +0 -24
- package/dist/three/tanvi.js.map +0 -1
- package/dist/three/tanya-asset.d.ts +0 -7
- package/dist/three/tanya-asset.d.ts.map +0 -1
- package/dist/three/tanya-asset.js +0 -7
- package/dist/three/tanya-asset.js.map +0 -1
- package/dist/three/tanya.d.ts +0 -23
- package/dist/three/tanya.d.ts.map +0 -1
- package/dist/three/tanya.js +0 -23
- package/dist/three/tanya.js.map +0 -1
- package/dist/three/tara-asset.d.ts +0 -26
- package/dist/three/tara-asset.d.ts.map +0 -1
- package/dist/three/tara-asset.js +0 -26
- package/dist/three/tara-asset.js.map +0 -1
- package/dist/three/tara.d.ts +0 -30
- package/dist/three/tara.d.ts.map +0 -1
- package/dist/three/tara.js +0 -30
- package/dist/three/tara.js.map +0 -1
- package/dist/three/tess-asset.d.ts +0 -7
- package/dist/three/tess-asset.d.ts.map +0 -1
- package/dist/three/tess-asset.js +0 -7
- package/dist/three/tess-asset.js.map +0 -1
- package/dist/three/tess.d.ts +0 -24
- package/dist/three/tess.d.ts.map +0 -1
- package/dist/three/tess.js +0 -24
- package/dist/three/tess.js.map +0 -1
- package/dist/three/tushar-asset.d.ts +0 -7
- package/dist/three/tushar-asset.d.ts.map +0 -1
- package/dist/three/tushar-asset.js +0 -7
- package/dist/three/tushar-asset.js.map +0 -1
- package/dist/three/tushar.d.ts +0 -22
- package/dist/three/tushar.d.ts.map +0 -1
- package/dist/three/tushar.js +0 -22
- package/dist/three/tushar.js.map +0 -1
- package/dist/types.d.ts +0 -149
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -140
- package/dist/types.js.map +0 -1
- package/dist/useAvatar.d.ts +0 -39
- package/dist/useAvatar.d.ts.map +0 -1
- package/dist/useAvatar.js +0 -35
- package/dist/useAvatar.js.map +0 -1
- package/src/avatar.d.ts +0 -436
- package/src/avatar.js +0 -1378
- package/src/behavior.d.ts +0 -42
- package/src/behavior.js +0 -114
- package/src/camera.js +0 -29
- package/src/clips.js +0 -155
- package/src/conformance.js +0 -119
- package/src/emotions.js +0 -58
- package/src/face-core.js +0 -180
- package/src/face-myna.d.ts +0 -7
- package/src/face-myna.js +0 -786
- package/src/face-peep-control-plane.js +0 -167
- package/src/face-peep.d.ts +0 -7
- package/src/face-peep.js +0 -925
- package/src/face-wren.d.ts +0 -7
- package/src/face-wren.js +0 -468
- package/src/faces.d.ts +0 -12
- package/src/faces.js +0 -53
- package/src/gaze.js +0 -504
- package/src/hand.js +0 -709
- package/src/head.js +0 -134
- package/src/idle.js +0 -668
- package/src/interjections.js +0 -747
- package/src/line-art.js +0 -137
- package/src/params.js +0 -180
- package/src/perform.js +0 -107
- package/src/prosody.js +0 -741
- package/src/rig.d.ts +0 -31
- package/src/rig.js +0 -32
- package/src/speech-timing.js +0 -51
- package/src/visemes.js +0 -410
package/README.md
CHANGED
|
@@ -1,187 +1,98 @@
|
|
|
1
1
|
# @voqalize/avatar
|
|
2
2
|
|
|
3
|
-
A
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
This is the browser half. The pipeline half is
|
|
13
|
-
[`voqalize-avatar`](https://pypi.org/project/voqalize-avatar/) on PyPI; they are
|
|
14
|
-
two ends of one wire format and release independently, kept compatible by the
|
|
15
|
-
wire contract rather than a shared version number
|
|
16
|
-
([RELEASING.md § Compatibility](https://github.com/voqalize/avatar/blob/main/RELEASING.md#compatibility)).
|
|
17
|
-
|
|
18
|
-
**Licence: MIT for the code and the SVG avatars, CC-BY 4.0 for the
|
|
19
|
-
2.5-D character binaries** (`assets/*.glb` — see `assets/README.md` for the
|
|
20
|
-
credit line). Which applies is decided by the kind of avatar, so adding a
|
|
21
|
-
character never moves the line. The manifest declares the pair as
|
|
22
|
-
`MIT AND CC-BY-4.0`.
|
|
3
|
+
A talking head for AI voice calls. Give it your live `PipecatClient` and an
|
|
4
|
+
element, and a character in that element speaks the bot's audio, lip-synced,
|
|
5
|
+
and shows the state of the call: listening, thinking, being interrupted.
|
|
6
|
+
|
|
7
|
+
This package is the loader. The avatar itself (the characters, the renderer and
|
|
8
|
+
everything that follows the call) is the Voqalize avatar runtime, which this
|
|
9
|
+
package imports from `https://avatar.voqalize.com` when the first avatar mounts.
|
|
10
|
+
Each release of this package is pinned to the runtime it was released with, so
|
|
11
|
+
upgrading the package is how you upgrade the avatar.
|
|
23
12
|
|
|
24
13
|
## Install
|
|
25
14
|
|
|
26
15
|
```sh
|
|
27
|
-
npm install @voqalize/avatar
|
|
28
|
-
pip install voqalize-avatar # the pipecat half
|
|
16
|
+
npm install @voqalize/avatar
|
|
29
17
|
```
|
|
30
18
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
19
|
+
The package is ESM, has no runtime dependencies and ships its own types.
|
|
20
|
+
`@pipecat-ai/client-js` (`>=1.4 <2`) and React (`>=18`) are optional peers. The
|
|
21
|
+
pipecat import is used only for types, and React is used only by
|
|
22
|
+
`@voqalize/avatar/react`.
|
|
35
23
|
|
|
36
|
-
##
|
|
24
|
+
## Mount a character
|
|
37
25
|
|
|
38
|
-
|
|
26
|
+
Mount it wherever your app renders the bot's tile:
|
|
39
27
|
|
|
40
28
|
```js
|
|
41
|
-
import { createAvatar } from
|
|
29
|
+
import { createAvatar } from "@voqalize/avatar";
|
|
42
30
|
|
|
43
|
-
const avatar = createAvatar({
|
|
44
|
-
|
|
31
|
+
const avatar = createAvatar({
|
|
32
|
+
mount: document.querySelector("#bot-tile"),
|
|
33
|
+
client: pipecatClient,
|
|
34
|
+
character: "<name>",
|
|
35
|
+
});
|
|
45
36
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
import { wren } from '@voqalize/avatar/faces/wren'; // `peep` is the default
|
|
49
|
-
|
|
50
|
-
<Avatar client={pipecatClient} options={{ face: wren }} className="call-tile" />
|
|
37
|
+
// when the call ends, or the tile goes away
|
|
38
|
+
avatar.destroy();
|
|
51
39
|
```
|
|
52
40
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
from voqalize_avatar import AvatarProcessor
|
|
58
|
-
|
|
59
|
-
pipeline = Pipeline([..., tts, AvatarProcessor(), transport.output()])
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
That is the integration, both halves of it. Neither takes an argument.
|
|
63
|
-
`createAvatar` returns `{ destroy() }` and nothing else — the avatar is an
|
|
64
|
-
embodiment of your `PipecatClient` and reacts to it, so there is no avatar to
|
|
65
|
-
drive and no state to read back.
|
|
66
|
-
|
|
67
|
-
## What you get for free
|
|
68
|
-
|
|
69
|
-
Most of the avatar works on any pipecat application without customization — not
|
|
70
|
-
because integrations were enumerated, but because the behaviour is derived from
|
|
71
|
-
frames and events a pipecat pipeline already emits. `SPEAKING`, `LISTENING`,
|
|
72
|
-
`MUTED`, `OFFLINE` and `DEGRADED` come from your `PipecatClient` with no backend
|
|
73
|
-
involvement at all; `THINKING`, `WORKING` and `CANT_HEAR` come from
|
|
74
|
-
`AvatarProcessor` watching turn boundaries, LLM response boundaries and
|
|
75
|
-
function-call frames; lipsync comes from the same karaoke frames pipecat already
|
|
76
|
-
pushes for word-level captions; blink, breath, gaze aversion and idle motion are
|
|
77
|
-
always the renderer's.
|
|
78
|
-
|
|
79
|
-
What is left over is small, specific, and each item is a case the library
|
|
80
|
-
refuses to guess at — a deliberate nod or greeting, a tool whose calls never
|
|
81
|
-
enter your pipeline, a pose richer than the core states, a backend that is not
|
|
82
|
-
ours. [The wire
|
|
83
|
-
contract](https://github.com/voqalize/avatar/blob/main/docs/contract-wire.md) is
|
|
84
|
-
the canonical reference for all of it.
|
|
85
|
-
|
|
86
|
-
## The wire protocol
|
|
87
|
-
|
|
88
|
-
`state`, `action` and `cues`, one envelope
|
|
89
|
-
([contract-wire.md](https://github.com/voqalize/avatar/blob/main/docs/contract-wire.md)):
|
|
90
|
-
|
|
91
|
-
```json
|
|
92
|
-
{ "type": "avatar", "cmd": "state", "state": "WORKING" }
|
|
93
|
-
{ "type": "avatar", "cmd": "action", "id": "ACKNOWLEDGE" }
|
|
94
|
-
{ "type": "avatar", "cmd": "cues", "ctx": "tts-context", "from_ms": 0, "cues": [] }
|
|
95
|
-
```
|
|
41
|
+
`createAvatar` returns at once, and the face appears when the runtime and the
|
|
42
|
+
character have loaded. The avatar owns the contents of `mount` and sizes itself
|
|
43
|
+
to it. `destroy()` is safe to call at any time, including before the face has
|
|
44
|
+
appeared.
|
|
96
45
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
point-in-time animations** that land on top of whatever state is effective at
|
|
102
|
-
the time; they are finite, they complete on their own, and they never establish
|
|
103
|
-
state.
|
|
46
|
+
`character` names one of the characters you can meet at
|
|
47
|
+
https://voqalize.com/demos/avatar. Each release of this package can mount a
|
|
48
|
+
fixed set of them, and a name it does not have throws, listing the ones it
|
|
49
|
+
does.
|
|
104
50
|
|
|
105
|
-
|
|
106
|
-
queued at or after `from_ms`, then append these". The server decides; the client
|
|
107
|
-
has no say and no way to refuse.
|
|
51
|
+
The gains are optional, from `0` to `2`, and `1` is as authored:
|
|
108
52
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
`
|
|
112
|
-
the text against the audio and map ARPAbet through `ARPABET_TO_VISEME`. With no
|
|
113
|
-
server work at all, `textToCues(text)` is a crude grapheme guesser, fit for
|
|
114
|
-
previews only. Each is exported from `@voqalize/avatar/internal`.
|
|
53
|
+
- `mouthGain` sets how far the mouth moves.
|
|
54
|
+
- `gestureGain` sets the size of nods and other gestures.
|
|
55
|
+
- `motionGain` sets the amount of idle motion.
|
|
115
56
|
|
|
116
|
-
|
|
57
|
+
The client must be connected to a Voqalize session that has the avatar turned on.
|
|
117
58
|
|
|
118
|
-
|
|
119
|
-
**`wren`**, **`myna`**. Each is its own entry point, and you pass the value
|
|
120
|
-
rather than a name:
|
|
59
|
+
### React
|
|
121
60
|
|
|
122
|
-
```
|
|
123
|
-
import {
|
|
61
|
+
```jsx
|
|
62
|
+
import { Avatar } from "@voqalize/avatar/react";
|
|
124
63
|
|
|
125
|
-
|
|
64
|
+
<Avatar client={client} character="<name>" className="bot-tile" />
|
|
126
65
|
```
|
|
127
66
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
## The 2.5-D characters
|
|
132
|
-
|
|
133
|
-
The characters ship as compiled binaries, each its own `createAvatar` module:
|
|
67
|
+
Nothing mounts while `client` is `null`. A new `client` or `character` rebuilds
|
|
68
|
+
the avatar. The gains are read when the avatar mounts. Every other prop goes to
|
|
69
|
+
the `<div>` the avatar renders into.
|
|
134
70
|
|
|
135
|
-
|
|
136
|
-
import { createAvatar } from '@voqalize/avatar/avatars/tara';
|
|
137
|
-
// or: @voqalize/avatar/avatars/tushar
|
|
138
|
-
// @voqalize/avatar/avatars/tanya
|
|
139
|
-
// @voqalize/avatar/avatars/tess
|
|
140
|
-
// @voqalize/avatar/avatars/tanvi
|
|
141
|
-
|
|
142
|
-
const avatar = createAvatar({ mount, client: pipecatClient });
|
|
143
|
-
```
|
|
71
|
+
## Content Security Policy
|
|
144
72
|
|
|
145
|
-
|
|
146
|
-
to move — eyes, teeth, the lip line — built as geometry rather than painted.
|
|
147
|
-
Three.js is an *optional* peer (`three`, `>=0.180 <0.187`) behind those
|
|
148
|
-
entry points only, so an SVG consumer never downloads it, and the `.glb`
|
|
149
|
-
is fetched when the avatar mounts.
|
|
73
|
+
If your page sets a Content-Security-Policy, add these sources:
|
|
150
74
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
`
|
|
75
|
+
- `script-src https://avatar.voqalize.com`, because the runtime is a module
|
|
76
|
+
imported from there.
|
|
77
|
+
- `connect-src https://avatar.voqalize.com blob:`, because the runtime fetches
|
|
78
|
+
the character from there and decodes the character's textures from `blob:`
|
|
79
|
+
URLs.
|
|
80
|
+
- `img-src https://avatar.voqalize.com blob:`. The asset host serves the
|
|
81
|
+
still photograph shown where a browser has no WebGL 2, as a call whose other
|
|
82
|
+
side has turned their camera off; without it that tile shows an empty circle.
|
|
83
|
+
`blob:` is for Safari before 17 and Firefox before 98, which decode the
|
|
84
|
+
character's textures through an image element.
|
|
157
85
|
|
|
158
|
-
|
|
159
|
-
around them; the credit line is in `assets/README.md`. Mounting, sizing, the asset
|
|
160
|
-
budget and what the characters can be asked to do:
|
|
161
|
-
[characters.md](https://github.com/voqalize/avatar/blob/main/docs/characters.md).
|
|
86
|
+
The runtime needs nothing else: no workers, no inline styles and no `eval`.
|
|
162
87
|
|
|
163
|
-
##
|
|
88
|
+
## Bundlers
|
|
164
89
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
90
|
+
The runtime is imported with a literal `import()` of its URL, marked so that
|
|
91
|
+
Vite and webpack leave it alone. Your bundle carries only this package, a few
|
|
92
|
+
kilobytes, and the browser fetches the runtime once per page however many
|
|
93
|
+
avatars mount.
|
|
168
94
|
|
|
169
|
-
|
|
170
|
-
createAvatar({ mount, client, ...yourOptions }) -> { destroy() }
|
|
171
|
-
```
|
|
95
|
+
## Licence
|
|
172
96
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
`@voqalize/avatar/internal` turns a cue array plus a clock into the mouth shape
|
|
176
|
-
for the current frame; every renderer needs that and none should write it twice.
|
|
177
|
-
|
|
178
|
-
An implementation may also export `supports` — one object naming the action ids
|
|
179
|
-
it answers to. Nothing in the library reads it; it is there because the wire's
|
|
180
|
-
action id is open and an unknown one is ignored in silence, so a page that
|
|
181
|
-
drives an avatar otherwise cannot tell a face that has no such motion from one
|
|
182
|
-
that did nothing ([design-avatar-interface.md](https://github.com/voqalize/avatar/blob/main/docs/design-avatar-interface.md)).
|
|
183
|
-
Omitting it is conforming.
|
|
184
|
-
|
|
185
|
-
**There is deliberately no renderer interface.** The pose channels our SVG mixer
|
|
186
|
-
uses to talk to our faces are internal, and a second public contract stays
|
|
187
|
-
premature until a second renderer says what it needs.
|
|
97
|
+
This package is MIT; see [LICENSE](./LICENSE). The runtime it loads is not open
|
|
98
|
+
source. Its terms are at https://avatar.voqalize.com/LICENSE.
|
package/dist/index.d.ts
CHANGED
|
@@ -3,32 +3,43 @@
|
|
|
3
3
|
*
|
|
4
4
|
* import { createAvatar } from "@voqalize/avatar";
|
|
5
5
|
*
|
|
6
|
-
* const avatar = createAvatar({ mount: el, client: pipecatClient });
|
|
6
|
+
* const avatar = createAvatar({ mount: el, client: pipecatClient, character: "tara" });
|
|
7
|
+
* // …
|
|
8
|
+
* avatar.destroy();
|
|
7
9
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
10
|
+
* This package is the loader and nothing else. The avatar itself — the
|
|
11
|
+
* characters, the renderer and everything that reads the call — is the
|
|
12
|
+
* Voqalize avatar runtime, which this module imports from
|
|
13
|
+
* `avatar.voqalize.com` at the version it was released with and which is
|
|
14
|
+
* licensed separately. `createAvatar` returns at once; the face appears when
|
|
15
|
+
* the runtime and the character have arrived.
|
|
11
16
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* -
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* - `@voqalize/avatar/faces/{peep,wren,myna}` — one drawing each. Import the
|
|
18
|
-
* one you want and pass it as `face`; the others never enter your bundle.
|
|
19
|
-
* - `@voqalize/avatar/react` — `<Avatar>`. Pulls React; nothing here does.
|
|
20
|
-
* - `@voqalize/avatar/internal` — the SVG widget, the behavior catalog and the
|
|
21
|
-
* viseme clock. **No semver promise**: these move in any minor. They are
|
|
22
|
-
* exported because an avatar author building on our renderer needs them, not
|
|
23
|
-
* because they are an interface.
|
|
24
|
-
*
|
|
25
|
-
* To ship your own avatar, publish a module exporting `createAvatar` and import
|
|
26
|
-
* that instead — see `createAvatar.ts` and docs/design-avatar-interface.md. A
|
|
27
|
-
* module may also export `supports`, which is how a page that *drives* an
|
|
28
|
-
* avatar learns which action ids this one answers to; every consumer that only
|
|
29
|
-
* mounts a face can ignore it.
|
|
17
|
+
* A page with a Content-Security-Policy needs `https://avatar.voqalize.com` in
|
|
18
|
+
* `script-src` (the runtime), `connect-src` (the character it fetches) and
|
|
19
|
+
* `img-src` (the camera-off still, where there is no WebGL 2), and `blob:` in
|
|
20
|
+
* `connect-src` (the character's textures are decoded from blob URLs;
|
|
21
|
+
* `img-src` too on Safari before 17 and Firefox before 98).
|
|
30
22
|
*/
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
export type
|
|
34
|
-
|
|
23
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
24
|
+
/** Amplitude, 0..2. `1` is as authored. */
|
|
25
|
+
export type Gain = number;
|
|
26
|
+
export interface AvatarOptions {
|
|
27
|
+
/** The element to render into. The avatar owns its contents. */
|
|
28
|
+
readonly mount: HTMLElement;
|
|
29
|
+
/** The live `PipecatClient` the avatar embodies. */
|
|
30
|
+
readonly client: PipecatClient;
|
|
31
|
+
/** Which character, by name — `"tara"`, for example. */
|
|
32
|
+
readonly character: string;
|
|
33
|
+
/** Mouth movement. */
|
|
34
|
+
readonly mouthGain?: Gain;
|
|
35
|
+
/** Nods and other gestures. */
|
|
36
|
+
readonly gestureGain?: Gain;
|
|
37
|
+
/** Idle motion. */
|
|
38
|
+
readonly motionGain?: Gain;
|
|
39
|
+
}
|
|
40
|
+
export interface AvatarInstance {
|
|
41
|
+
/** Stop embodying the client and empty the mount. Safe to call at any time,
|
|
42
|
+
* including before the avatar has appeared. */
|
|
43
|
+
destroy(): void;
|
|
44
|
+
}
|
|
45
|
+
export declare function createAvatar(options: AvatarOptions): AvatarInstance;
|
package/dist/index.js
CHANGED
|
@@ -3,31 +3,54 @@
|
|
|
3
3
|
*
|
|
4
4
|
* import { createAvatar } from "@voqalize/avatar";
|
|
5
5
|
*
|
|
6
|
-
* const avatar = createAvatar({ mount: el, client: pipecatClient });
|
|
6
|
+
* const avatar = createAvatar({ mount: el, client: pipecatClient, character: "tara" });
|
|
7
|
+
* // …
|
|
8
|
+
* avatar.destroy();
|
|
7
9
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
10
|
+
* This package is the loader and nothing else. The avatar itself — the
|
|
11
|
+
* characters, the renderer and everything that reads the call — is the
|
|
12
|
+
* Voqalize avatar runtime, which this module imports from
|
|
13
|
+
* `avatar.voqalize.com` at the version it was released with and which is
|
|
14
|
+
* licensed separately. `createAvatar` returns at once; the face appears when
|
|
15
|
+
* the runtime and the character have arrived.
|
|
11
16
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* -
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* - `@voqalize/avatar/faces/{peep,wren,myna}` — one drawing each. Import the
|
|
18
|
-
* one you want and pass it as `face`; the others never enter your bundle.
|
|
19
|
-
* - `@voqalize/avatar/react` — `<Avatar>`. Pulls React; nothing here does.
|
|
20
|
-
* - `@voqalize/avatar/internal` — the SVG widget, the behavior catalog and the
|
|
21
|
-
* viseme clock. **No semver promise**: these move in any minor. They are
|
|
22
|
-
* exported because an avatar author building on our renderer needs them, not
|
|
23
|
-
* because they are an interface.
|
|
24
|
-
*
|
|
25
|
-
* To ship your own avatar, publish a module exporting `createAvatar` and import
|
|
26
|
-
* that instead — see `createAvatar.ts` and docs/design-avatar-interface.md. A
|
|
27
|
-
* module may also export `supports`, which is how a page that *drives* an
|
|
28
|
-
* avatar learns which action ids this one answers to; every consumer that only
|
|
29
|
-
* mounts a face can ignore it.
|
|
17
|
+
* A page with a Content-Security-Policy needs `https://avatar.voqalize.com` in
|
|
18
|
+
* `script-src` (the runtime), `connect-src` (the character it fetches) and
|
|
19
|
+
* `img-src` (the camera-off still, where there is no WebGL 2), and `blob:` in
|
|
20
|
+
* `connect-src` (the character's textures are decoded from blob URLs;
|
|
21
|
+
* `img-src` too on Safari before 17 and Firefox before 98).
|
|
30
22
|
*/
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
23
|
+
import { PROTOCOL, RUNTIME_URL } from "./pin.js";
|
|
24
|
+
let runtime;
|
|
25
|
+
function load() {
|
|
26
|
+
// One import per page, however many avatars mount; a failure is forgotten so
|
|
27
|
+
// the next mount tries again rather than inheriting a dead promise.
|
|
28
|
+
runtime ??= import(/* @vite-ignore */ /* webpackIgnore: true */ RUNTIME_URL).catch((error) => {
|
|
29
|
+
runtime = undefined;
|
|
30
|
+
throw error;
|
|
31
|
+
});
|
|
32
|
+
return runtime;
|
|
33
|
+
}
|
|
34
|
+
export function createAvatar(options) {
|
|
35
|
+
if (!options?.mount)
|
|
36
|
+
throw new TypeError("createAvatar: `mount` is required");
|
|
37
|
+
if (!options.client)
|
|
38
|
+
throw new TypeError("createAvatar: `client` is required");
|
|
39
|
+
if (!options.character)
|
|
40
|
+
throw new TypeError("createAvatar: `character` is required");
|
|
41
|
+
let instance;
|
|
42
|
+
let destroyed = false;
|
|
43
|
+
load()
|
|
44
|
+
.then((rt) => {
|
|
45
|
+
if (!destroyed)
|
|
46
|
+
instance = rt.mount({ ...options, protocol: PROTOCOL });
|
|
47
|
+
})
|
|
48
|
+
.catch((error) => console.error(`[avatar] could not start the avatar from ${RUNTIME_URL}`, error));
|
|
49
|
+
return {
|
|
50
|
+
destroy() {
|
|
51
|
+
destroyed = true;
|
|
52
|
+
instance?.destroy();
|
|
53
|
+
instance = undefined;
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
package/dist/pin.d.ts
ADDED
package/dist/pin.js
ADDED
package/dist/react.d.ts
CHANGED
|
@@ -3,11 +3,26 @@
|
|
|
3
3
|
*
|
|
4
4
|
* import { Avatar } from "@voqalize/avatar/react";
|
|
5
5
|
*
|
|
6
|
-
* <Avatar client={session.client} className="avatar-tile" />
|
|
6
|
+
* <Avatar client={session.client} character="tara" className="avatar-tile" />
|
|
7
7
|
*
|
|
8
|
-
* Separate from the main entry so that `createAvatar` costs nothing to a
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* Separate from the main entry so that `createAvatar` costs nothing to a caller
|
|
9
|
+
* who is not on React. Everything but the avatar's own props is forwarded to
|
|
10
|
+
* the mount `<div>`, so it sizes and styles like the tile it lives in.
|
|
11
11
|
*/
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
import { type HTMLAttributes } from "react";
|
|
13
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
14
|
+
import { type Gain } from "./index.js";
|
|
15
|
+
export type AvatarProps = Omit<HTMLAttributes<HTMLDivElement>, "children"> & {
|
|
16
|
+
/** The live `PipecatClient`, or `null` before connect. Nothing mounts until
|
|
17
|
+
* it is non-null. */
|
|
18
|
+
client?: PipecatClient | null;
|
|
19
|
+
/** Which character, by name. */
|
|
20
|
+
character: string;
|
|
21
|
+
/** Read at mount only. */
|
|
22
|
+
mouthGain?: Gain;
|
|
23
|
+
/** Read at mount only. */
|
|
24
|
+
gestureGain?: Gain;
|
|
25
|
+
/** Read at mount only. */
|
|
26
|
+
motionGain?: Gain;
|
|
27
|
+
};
|
|
28
|
+
export declare function Avatar({ client, character, mouthGain, gestureGain, motionGain, ...rest }: AvatarProps): import("react").JSX.Element;
|
package/dist/react.js
CHANGED
|
@@ -1,13 +1,31 @@
|
|
|
1
|
+
import { jsx as _jsx } from "react/jsx-runtime";
|
|
1
2
|
/**
|
|
2
3
|
* `@voqalize/avatar/react` — the avatar as one component.
|
|
3
4
|
*
|
|
4
5
|
* import { Avatar } from "@voqalize/avatar/react";
|
|
5
6
|
*
|
|
6
|
-
* <Avatar client={session.client} className="avatar-tile" />
|
|
7
|
+
* <Avatar client={session.client} character="tara" className="avatar-tile" />
|
|
7
8
|
*
|
|
8
|
-
* Separate from the main entry so that `createAvatar` costs nothing to a
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
* Separate from the main entry so that `createAvatar` costs nothing to a caller
|
|
10
|
+
* who is not on React. Everything but the avatar's own props is forwarded to
|
|
11
|
+
* the mount `<div>`, so it sizes and styles like the tile it lives in.
|
|
11
12
|
*/
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
import { useEffect, useRef } from "react";
|
|
14
|
+
import { createAvatar } from "./index.js";
|
|
15
|
+
export function Avatar({ client, character, mouthGain, gestureGain, motionGain, ...rest }) {
|
|
16
|
+
const ref = useRef(null);
|
|
17
|
+
// The gains are read at mount through a ref, so a fresh literal on every
|
|
18
|
+
// render does not rebuild the face.
|
|
19
|
+
const gains = useRef({ mouthGain, gestureGain, motionGain });
|
|
20
|
+
gains.current = { mouthGain, gestureGain, motionGain };
|
|
21
|
+
useEffect(() => {
|
|
22
|
+
const mount = ref.current;
|
|
23
|
+
if (!mount || !client)
|
|
24
|
+
return;
|
|
25
|
+
const avatar = createAvatar({ mount, client, character, ...gains.current });
|
|
26
|
+
return () => avatar.destroy();
|
|
27
|
+
// A new client or character is a new thing to embody, so the avatar is
|
|
28
|
+
// rebuilt rather than re-pointed.
|
|
29
|
+
}, [client, character]);
|
|
30
|
+
return _jsx("div", { role: "img", "aria-label": "avatar", ...rest, ref: ref });
|
|
31
|
+
}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voqalize/avatar",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "A
|
|
5
|
-
"license": "MIT
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "A talking head for AI voice calls that embodies your pipecat client. This package loads the Voqalize avatar runtime.",
|
|
5
|
+
"license": "MIT",
|
|
6
6
|
"author": "Voqalize",
|
|
7
7
|
"homepage": "https://github.com/voqalize/avatar#readme",
|
|
8
8
|
"repository": {
|
|
@@ -17,14 +17,8 @@
|
|
|
17
17
|
"avatar",
|
|
18
18
|
"talking-head",
|
|
19
19
|
"lipsync",
|
|
20
|
-
"viseme",
|
|
21
|
-
"rhubarb",
|
|
22
20
|
"pipecat",
|
|
23
|
-
"voice-agent"
|
|
24
|
-
"svg",
|
|
25
|
-
"animation",
|
|
26
|
-
"3d",
|
|
27
|
-
"threejs"
|
|
21
|
+
"voice-agent"
|
|
28
22
|
],
|
|
29
23
|
"engines": {
|
|
30
24
|
"node": ">=20"
|
|
@@ -45,66 +39,20 @@
|
|
|
45
39
|
"types": "./dist/react.d.ts",
|
|
46
40
|
"default": "./dist/react.js"
|
|
47
41
|
},
|
|
48
|
-
"./internal": {
|
|
49
|
-
"types": "./dist/internal.d.ts",
|
|
50
|
-
"default": "./dist/internal.js"
|
|
51
|
-
},
|
|
52
|
-
"./internal/three": {
|
|
53
|
-
"types": "./dist/three/internal.d.ts",
|
|
54
|
-
"default": "./dist/three/internal.js"
|
|
55
|
-
},
|
|
56
|
-
"./faces/peep": {
|
|
57
|
-
"types": "./src/face-peep.d.ts",
|
|
58
|
-
"default": "./src/face-peep.js"
|
|
59
|
-
},
|
|
60
|
-
"./faces/wren": {
|
|
61
|
-
"types": "./src/face-wren.d.ts",
|
|
62
|
-
"default": "./src/face-wren.js"
|
|
63
|
-
},
|
|
64
|
-
"./faces/myna": {
|
|
65
|
-
"types": "./src/face-myna.d.ts",
|
|
66
|
-
"default": "./src/face-myna.js"
|
|
67
|
-
},
|
|
68
|
-
"./avatars/tara": {
|
|
69
|
-
"types": "./dist/three/tara.d.ts",
|
|
70
|
-
"default": "./dist/three/tara.js"
|
|
71
|
-
},
|
|
72
|
-
"./avatars/tushar": {
|
|
73
|
-
"types": "./dist/three/tushar.d.ts",
|
|
74
|
-
"default": "./dist/three/tushar.js"
|
|
75
|
-
},
|
|
76
|
-
"./avatars/tanya": {
|
|
77
|
-
"types": "./dist/three/tanya.d.ts",
|
|
78
|
-
"default": "./dist/three/tanya.js"
|
|
79
|
-
},
|
|
80
|
-
"./avatars/tess": {
|
|
81
|
-
"types": "./dist/three/tess.d.ts",
|
|
82
|
-
"default": "./dist/three/tess.js"
|
|
83
|
-
},
|
|
84
|
-
"./avatars/tanvi": {
|
|
85
|
-
"types": "./dist/three/tanvi.d.ts",
|
|
86
|
-
"default": "./dist/three/tanvi.js"
|
|
87
|
-
},
|
|
88
42
|
"./package.json": "./package.json"
|
|
89
43
|
},
|
|
90
44
|
"files": [
|
|
91
|
-
"
|
|
92
|
-
"client",
|
|
93
|
-
"dist",
|
|
94
|
-
"assets",
|
|
95
|
-
"LICENSE-CC-BY-4.0",
|
|
96
|
-
"LICENSE-APACHE-2.0"
|
|
45
|
+
"dist"
|
|
97
46
|
],
|
|
98
47
|
"scripts": {
|
|
99
|
-
"build": "tsc -p tsconfig.json",
|
|
48
|
+
"build": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json",
|
|
100
49
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
101
|
-
"prepare": "tsc -p tsconfig.json"
|
|
50
|
+
"prepare": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json"
|
|
102
51
|
},
|
|
103
52
|
"peerDependencies": {
|
|
104
53
|
"@pipecat-ai/client-js": ">=1.4 <2",
|
|
105
54
|
"react": ">=18",
|
|
106
|
-
"react-dom": ">=18"
|
|
107
|
-
"three": ">=0.180 <0.187"
|
|
55
|
+
"react-dom": ">=18"
|
|
108
56
|
},
|
|
109
57
|
"peerDependenciesMeta": {
|
|
110
58
|
"@pipecat-ai/client-js": {
|
|
@@ -115,13 +63,6 @@
|
|
|
115
63
|
},
|
|
116
64
|
"react-dom": {
|
|
117
65
|
"optional": true
|
|
118
|
-
},
|
|
119
|
-
"three": {
|
|
120
|
-
"optional": true
|
|
121
66
|
}
|
|
122
|
-
},
|
|
123
|
-
"devDependencies": {
|
|
124
|
-
"@types/three": "^0.185.4",
|
|
125
|
-
"three": "^0.180.0"
|
|
126
67
|
}
|
|
127
68
|
}
|